From 6949c372c4f403fab31459cf0d713770d289c758 Mon Sep 17 00:00:00 2001 From: Nicolas Boulanger-Lewandowski Date: Wed, 2 Jan 2013 23:07:17 -0500 Subject: [PATCH 001/442] first version of RNN-RBM tutorial --- code/rnnrbm.py | 232 ++++++ doc/contents.txt | 1 + doc/images/rnnrbm.png | Bin 0 -> 21397 bytes doc/images/rnnrbm.svg | 1758 +++++++++++++++++++++++++++++++++++++++++ doc/intro.txt | 3 + doc/references.txt | 6 +- doc/rnnrbm.txt | 350 ++++++++ 7 files changed, 2348 insertions(+), 2 deletions(-) create mode 100644 code/rnnrbm.py create mode 100644 doc/images/rnnrbm.png create mode 100644 doc/images/rnnrbm.svg create mode 100644 doc/rnnrbm.txt diff --git a/code/rnnrbm.py b/code/rnnrbm.py new file mode 100644 index 00000000..5a7b7cb9 --- /dev/null +++ b/code/rnnrbm.py @@ -0,0 +1,232 @@ +# Author: Nicolas Boulanger-Lewandowski +# University of Montreal (2012) +# RNN-RBM deep learning tutorial +# More information at http://deeplearning.net/tutorial/rnnrbm.html + +import numpy +import pylab +import glob +import sys + +from midi.utils import midiread, midiwrite +import theano +import theano.tensor as T +from theano.tensor.shared_randomstreams import RandomStreams + +numpy.random.seed(0xdeadbeef) +rng = RandomStreams(seed=numpy.random.randint(1 << 30)) +theano.config.warn.subtensor_merge_bug = False + + +def build_rbm(v, W, bv, bh, k): + '''Construct a k-step Gibbs chain starting at v with RBM parameters W, bv, bh. + +v : Theano vector or matrix + If a matrix, multiple chains will be run in parallel (batch). +W : Theano matrix + Weight matrix of the RBM. +bv : Theano vector + Visible bias vector of the RBM. +bh : Theano vector + Hidden bias vector of the RBM. +k : scalar or Theano scalar + Length of the Gibbs chain. + +Return a (v_sample, cost, monitor, updates) tuple: + +v_sample : Theano vector or matrix with the same shape as `v` + Corresponds to the generated sample(s). +cost : Theano scalar + Expression whose gradient with respect to W, bv, bh is the CD-k approximation + to the log-likelihood of `v` (training example) under the RBM. + The cost is averaged in the batch case. +monitor: Theano scalar + Pseudo log-likelihood (also averaged in the batch case). +updates: dictionary of Theano variable -> Theano variable + The `updates` object returned by scan.''' + + def gibbs_step(v): + mean_h = T.nnet.sigmoid(T.dot(v, W) + bh) + h = rng.binomial(size=mean_h.shape, n=1, p=mean_h, dtype=theano.config.floatX) + mean_v = T.nnet.sigmoid(T.dot(h, W.T) + bv) + v = rng.binomial(size=mean_v.shape, n=1, p=mean_v, dtype=theano.config.floatX) + return mean_v, v + + chain, updates = theano.scan(lambda v: gibbs_step(v)[1], outputs_info=[v], n_steps=k) + v_sample = chain[-1] + + mean_v = gibbs_step(v_sample)[0] + monitor = T.xlogx.xlogy0(v, mean_v) + T.xlogx.xlogy0(1-v, 1-mean_v) + monitor = monitor.sum() / v.shape[0] + + free_energy = lambda v: -(v * bv).sum() - T.log(1 + T.exp(T.dot(v, W) + bh)).sum() + cost = (free_energy(v) - free_energy(v_sample)) / v.shape[0] + + return v_sample, cost, monitor, updates + + +def shared_normal(num_rows, num_cols, scale=1): + '''Initialize a matrix shared variable with normally distributed elements.''' + return theano.shared(numpy.random.normal(scale=scale, size=(num_rows, num_cols)).astype(theano.config.floatX)) + + +def shared_zeros(*shape): + '''Initialize a vector shared variable with zero elements.''' + return theano.shared(numpy.zeros(shape, dtype=theano.config.floatX)) + + +def build_rnnrbm(n_visible, n_hidden, n_hidden_recurrent): + '''Construct a symbolic RNN-RBM, including initialized parameters in shared variables and +symbolic variables for the training cost and sequence generation. + +n_visible : integer + Number of visible units. +n_hidden : integer + Number of hidden units of the conditional RBMs. +n_hidden_recurrent : integer + Number of hidden units of the RNN. + +Return a (v, v_sample, cost, monitor, params, updates_train, v_t, updates_generate) tuple: + +v : Theano matrix + Symbolic variable holding an input sequence (used during training) +v_sample : Theano matrix + Symbolic variable holding the negative particles for CD log-likelihood gradient estimation + (used during training) +cost : Theano scalar + Expression whose gradient (considering v_sample constant) corresponds to the LL gradient of the RNN-RBM. + (used during training) +monitor : Theano scalar + Frame-level pseudo-likelihood (useful for monitoring during training) +params : tuple of Theano shared variables + The parameters of the model to be optimized during training. +updates_train : dictionary of Theano variable -> Theano variable + Update object that should be passed to theano.function when compiling the training function. +v_t : Theano matrix + Symbolic variable holding a generate sequence (used during sampling) +updates_generate : dictionary of Theano variable -> Theano variable + Update object that should be passed to theano.function when compiling the generation function.''' + + W = shared_normal(n_visible, n_hidden, 0.01) + bv = shared_zeros(n_visible) + bh = shared_zeros(n_hidden) + Wuh = shared_normal(n_hidden_recurrent, n_hidden, 0.0001) + Wuv = shared_normal(n_hidden_recurrent, n_visible, 0.0001) + Wvu = shared_normal(n_visible, n_hidden_recurrent, 0.0001) + Wuu = shared_normal(n_hidden_recurrent, n_hidden_recurrent, 0.0001) + bu = shared_zeros(n_hidden_recurrent) + + params = W, bv, bh, Wuh, Wuv, Wvu, Wuu, bu # learned parameters as shared variables + + v = T.matrix() # a training sequence + u0 = T.zeros((n_hidden_recurrent,)) # initial value for the RNN hidden units + + # if `v_t` is given, deterministic recurrence to compute the variable biases bv_t, bh_t at each time step + # if `v_t` is None, same recurrence but with a separate Gibbs chain at each time step to sample (generate) from the RNN-RBM + # the resulting sample v_t is returned in order to be passed down to the sequence history + def recurrence(v_t, u_tm1): + bv_t = bv + T.dot(u_tm1, Wuv) + bh_t = bh + T.dot(u_tm1, Wuh) + generate = v_t is None + if generate: + v_t, _, _, updates = build_rbm(T.zeros((n_visible,)), W, bv_t, bh_t, k=25) + u_t = T.tanh(bu + T.dot(v_t, Wvu) + T.dot(u_tm1, Wuu)) + return ([v_t, u_t], updates) if generate else [u_t, bv_t, bh_t] + + # for training, the deterministic recurrence is used to compute all the {bv_t, bh_t, 1 <= t <= T} given v + # conditional RBMs can then be trained in batches using those parameters + (u_t, bv_t, bh_t), updates_train = theano.scan(lambda v_t, u_tm1, *_: recurrence(v_t, u_tm1), sequences=v, outputs_info=[u0, None, None], non_sequences=params) + v_sample, cost, monitor, updates_rbm = build_rbm(v, W, bv_t[:], bh_t[:], k=15) + updates_train.update(updates_rbm) + + # symbolic loop for sequence generation + (v_t, u_t), updates_generate = theano.scan(lambda u_tm1, *_: recurrence(None, u_tm1), outputs_info=[None, u0], non_sequences=params, n_steps=200) + + return v, v_sample, cost, monitor, params, updates_train, v_t, updates_generate + + +class RnnRbm: + '''Simple class to build and train an RNN-RBM from MIDI files and to generate sample sequences.''' + + def __init__(self, n_hidden=150, n_hidden_recurrent=100, lr=0.001, r=(21, 109), dt=0.3): + '''Constructs and compiles Theano functions for training and sequence generation. + + n_hidden : integer + Number of hidden units of the conditional RBMs. + n_hidden_recurrent : integer + Number of hidden units of the RNN. + lr : float + Learning rate + r : (integer, integer) tuple + Specifies the pitch range of the piano-roll in MIDI note numbers, including r[0] but not r[1], + such that r[1]-r[0] is the number of visible units of the RBM at a given time step. + The default (21, 109) corresponds to the full range of piano (88 notes). + dt : float + Sampling period when converting the MIDI files into piano-rolls, or equivalently the time difference + between consecutive time steps.''' + + self.r = r + self.dt = dt + v, v_sample, cost, monitor, params, updates_train, v_t, updates_generate = build_rnnrbm(r[1]-r[0], n_hidden, n_hidden_recurrent) + + gradient = T.grad(cost, params, consider_constant=[v_sample]) + updates_train.update(dict((p, p - lr*g) for p, g in zip(params, gradient))) + self.train_function = theano.function([v], monitor, updates=updates_train) + self.generate_function = theano.function([], v_t, updates=updates_generate) + + + def train(self, files, batch_size=100, num_epochs=150): + '''Train the RNN-RBM via stochastic gradient descent (SGD) using MIDI files converted to piano-rolls. + + files : list of strings + List of MIDI files that will be loaded as piano-rolls for training. + batch_size : integer + Training sequences will be split into subsequences of at most this size + before applying the SGD updates. + num_epochs : integer + Number of epochs (pass over the training set) performed. The user can + safely interrupt training with Ctrl+C at any time.''' + + dataset = [midiread(f, self.r, self.dt).piano_roll for f in files] + try: + for epoch in xrange(num_epochs): + numpy.random.shuffle(dataset) + costs = [] + + for s, sequence in enumerate(dataset): + for i in xrange(0, len(sequence), batch_size): + cost = self.train_function(sequence[i:i+batch_size]) + costs.append(cost) + + print 'Epoch %i/%i' % (epoch + 1, num_epochs), numpy.mean(costs) + sys.stdout.flush() + + except KeyboardInterrupt: + print 'Interrupted by user.' + + + def generate(self, filename, show=True): + '''Generate a sample sequence, plot the resulting piano-roll and save it as a MIDI file. + + filename : string + A MIDI file will be created at this location. + show : boolean + If True, a piano-roll of the generated sequence will be shown.''' + + piano_roll = self.generate_function() + midiwrite(filename, piano_roll, self.r, self.dt) + if show: + extent = (0, self.dt * len(piano_roll)) + self.r + pylab.imshow(piano_roll.T, origin='lower', aspect='auto', interpolation='nearest', cmap=pylab.cm.gray_r, extent=extent) + pylab.xlabel('time (s)') + pylab.ylabel('MIDI note number') + pylab.title('generated piano-roll') + pylab.show() + + +if __name__ == '__main__': + model = RnnRbm() + model.train(glob.glob('Nottingham/train/*.mid')) + model.generate('sample1.mid') + model.generate('sample2.mid') + diff --git a/doc/contents.txt b/doc/contents.txt index cfcf30e6..381043d9 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -19,5 +19,6 @@ Contents rbm DBN hmc + rnnrbm utilities references diff --git a/doc/images/rnnrbm.png b/doc/images/rnnrbm.png new file mode 100644 index 0000000000000000000000000000000000000000..b318fff624535d8f981a77af40f4954cef784b19 GIT binary patch literal 21397 zcma&Oby!qg)CW3*q_i|hcS=Zu!qDB_4HD7~3eqVcNOwp{cb9}9E#2MSaJOIH@BVe? zd3=F+X3m^BXYaLswcao#1xYj%A`}P&f+j5mRe?ZY0>SUoNU-3)EgYV;;0KJeiliu{ ze1v2d{DEL1D+vWZ!Q)n}*&qBsc9ha~hConp{{4f2q@)pmClOtw<;4*fVc_7Q3s)GFautx#)awW3)2FuXh!9_oCly+K@- zawaUhpam7RoG(>*l(SXlq(kiY4^ITr)1AWv{G%e=nbkhqe8<;m$B*BxH*?0U0w6H3 z@M0*hM8OXvOd8RDe?!AC$=?OgWBq%ETqfv$o`EP+{?9Y;q1Y7f0{-h6D?}bqn9s%` z-@biQ%oIlRDZW-7zNZ~^n}fL$(j~(> z*2MaON~5$&r|OLr&&}5-&YZXufzB)h*61Jhd3O)mPmUjnr_UhdZ61M5n4-uzkD)K9 zWVpC8w}V?EWgywN5))9&x0r{L%mm6g+3ycae$`X2Oud=L+TWGu!SyMpop_#C(T_pQ zS{poT$H6+%nsS;Wq?^XuOyX98WT)}r$7bU-}Yaz;E}ZsKV4WiyQ0pJ$m~QZLBsxrY#lU9Ye6ofQ2Vo*Z#jjB z&SRQwwwN{>VLUeR?6hcy#Ni)fL|Jn%N9lTc(|{XdXB&q}D7fq}pIC zQbiOYZ3fmD`LHAhOB&Wqkw=t&Jau%wzIwd>d9u5N1a7XK)$Oat25h78rp@(6n5<3P z$fVlU!6^-VmHi^fpOeX3!4adlix{5ITkk(U@+N9ZE7JYuKH*$@!(P4lV3wvhVATEU zB?N_2!Si!?4grtoEjdfFp|<}m{Lv3ftmT@>LgN5MY4={iKm%m(Y5L@^dPL}93F>tW z!(#OC%mpk{;$%BopG_)e50P2P#fA!tpy5p_7$teP{YA1Hiwec>7hY`Drr8tpw|OgB zuH~?OQW|-MS}u<^s{Upg&-_;~7L5tcuYAR$O<1W4t$Pul@yr5zkF@H&c5&u)7ziU8 zM9MWk%8J6z1*6azH9YGlZj@BDk-^aJFmFEp=^Q5uhPS+*o038=u3(Jk8I~;JyIJ;1 zqYpY-TS33i=O5RW{++c@mKwFV1xzTb4%6%Pl@}xRt!Q+P>VB*=Fj5Mcw7=1OzRKhT z%~?2MY)|NnEL)a*7n%!Ad%*3{d zb+7$akGX-5ic;O=F(=dS<8ctBOb?!ttqSEh19_GZ)w$WhL4>_5wbHv{kJss_0pFYb z7!d*tUAu{f3m*!|mbWkH*dkloUlRTJ@q>?# zua#8!xvoE}|$2s?GZf9(0C$7iWHo`Q#%|3MIHb}OC!QtjlP~{{w9(aW=>%gmEk1SC z>yeo-@mj>M#btLKMg%s$aYbBjjgHwIVqV(ujYU^fOibINvlWZqW|E!+J>(+n@27%D z0X8C-tgk<>5~iL#U?5pf>OI8RS(YE3-gFL^)l7{3U=(Dm$e)-g5`NTZnUrP|`nX~K zCOWtttKtNL-5}tw)tt<94hyMe{NPjqs$iJe@CMpQ=H~TiZ^(!si?1*W+Tiavs~8Ur^Zv^u8NN5vE9X zW86ruq1)T+VZOM?=XZKQl02~Eo34Zjy>iZYLFHg()xjpx|L%1*%$Q%#XDraVYDnNo z;6U?wDVva#w7s$M()fjwlM@m=>=`8{jS2p*zh7gtwGFLRHCYn251%jvUmNP^Ffua> zs=1bymp_#)A(N4jiHNjUR-R%`XIEB!>Ps$GvoZbmpt@)J(!Y*Hvv-)E*qbhJ z?l7FL%fzHS*Vkvp zLHNdkmXd;^b9nf}Vv{Qz!F#cUA|A{LoYxz0Qmp!%+5w%aeg()jL)eq9NOp*zCe^7+ezpFal%c6W6hEw__lV}I^m-5I9pTflE8#YVt7z5J3n)J(-I zBI19(^}F6-d24%nxz<)Qb##i24(+`n>8+?gDJ#Ud8__i~J>c1C;#Teha+n0Yua8qk z2f;lTNt8q`S6jQ&@cHhw{d(Qa{e29XNXAn3<>{%ik`jT|wi|rF$*e3Kgqj7L_+IQS z#dY{>%XghY0s>Q?+|WtR`QqZ@&!0bIkO}8KGqbTJef@fNc_|**=x$mbu~?pK>*_c| z@_yJSbv>GRvViFDY6t2lCKmYP$6m2&p%s^oo!wzi7|v)K-$cb7h0LTZdr6&?(oi=e zVYup4%5I&!_Qxb#ZBFX>hPaGj?rFO(YSwjG9_a zEBu?SqsYcf*@IkB*v4Oqx+7fKalxZ8{&$;?Z1C{|X1>0@>gwu@i85@B?#CIiv46nL ziTy}vk@fa+8q}%U$n8+$<&fs%x8sHdwhiYS>59J>k1`GtXp2n0W!XDVgM~>*NXVjH z1s~BnF5A%1z{td;Oh5UKzrLGv^2bJ~JJ-s@o0Nff8S78xNH%tM`t|nd-@gOYw!mhJ zL$79Kwd1h*3S%L;nf9elY3UhiXMYSC8+b4?e?WCb4d`CA_?=>IZ~yf4=xvr}^|PZu zl6DrYGo43a=KLw(*vaF09G;tpr$WEkZ8SqDo>qR(g9IOc z2H-|j6%|1tA*{vY%Hj(;G0Z55>Z+=*2?-Y`Cp%s=;3`;JZcmkH#vE{K2Xb#2;vl$E z^D;9Z++FTtVPPF|KtsT_O-h61c!&Jrf37h>GS(f<1Vn?*$A zdD;J2JiBSze>_h*E-nra4=>W)-oK)t%P}}BIhjCFadcDgJd zR-p$t4H{$2KL9B}-vmYNPp6Cc`;DaW2`$bJ4av&L3@$9B$>dYYaA9J$_4Ot|LLG$d z1j+(2I(nHpP$3zYYG&F}IhX)5Fq@#E_6T&*V3Q>8c;clou9WmVL=&H|_$vD?dNqn-5_Y zW*)6beE{pYsH4QRlxn&%P4A4IHwqzusDG?2o-@Q2fZY z2SZzS-$;=be($kpv*WSve|)g^-%r= zg|`_o=Q!Qo-k44E0x~t(X^TlyrOoqFnawuXZCuyNXmfqf zp+w`kXY{Uo=Aqr}@BUcTtgK-E*vX@@je1L1KDoOU5fv+>MNLNs%41#OqIg;d+c@1_ z*z~l9sp)#9Y5(2*{WZ%f2Oh&RGteaZHA+cnSc6e$NWdWa~58+oFO5^?Cse& zJD-D2boCV=9;zC4EG{Azdir0}({)1Zu~}J0?E?2wnW1)>=451<))NJ_Lzs}Ay*)~| zVHIjxT7`7}+T52>WFl?zyt+sT!CPBf;3E$IR_2=eaFCLcBDj}O`cl^A6czpI>Y`*| zC|12a+Z@7DwZQsiR+XBX+T842Xk{i95*qevaFBz8V-S>kHDUK7J@krOheGH@^!_I} z#FUg2{zqwmfn;S-D=jp#^76nQ=HlaP+|LAMEaas7m7vd!Mil&tLh>}+vKiTmBec)qNpA5t`hsIBe(>3V;9T3S=I2DvY##d}4?9vjh5znYsDLj~~M#BA)M$+d=OJhav*U0X>FE zvm8Dk42RCk)AN#PgzK~L*xX!`X+JtZ8(wv~uU}_4Z;ycYQOWd#6%`fztu&QXQi_d? zeA#N&`T0d_$h`!+2UJ3$&F7Ybj;^q^XLvZW8^7_T09a{}pb_;)lOED=S_~85aX%j( z1jyv4q^1%R5|$aXHnqk^M97MZzo;~w&TeX&>+1s@L+T$mbDS(vM&gv~0e1@y78b0Q z;PpTAV3oABwSi95O*Q7+O>9`7MWPqt8bIInM)oTC465g$C9&s7jvsDlvy_obyDrEK_6!p>H%I@F4<=?+I))MY;NK|^m z#Dp5y+2Va2?fhfXdrMsAeCECmXV*bSYliMp;8O-3M(Xol-;eW84Fl+v;tgbkXGn?C z{RI$6q`@loBcbe8sAY@uc64d!2T&GbXY#IxBf5#o%gd`RM``0jwi2&1Jmat)B&u&b z1ebbxdS+)m=W49G^GE34_ipN<^}#@-2>G;se_hMJHWN?yraeEr1_Md1bf`G;_(XOb zr|Gl?{pc5mo0K8hNc5l>1UGFxJw2_XtBVK^@1kcL63}>avKm-le|@<>6LLJhu>4Cv zz-c2iAt9l)^^u+UZx;_X45V*VR9UN|P_2KEdSRGsAwDtE;JTadIM>HcfAj!0zqu4+#n~v#_{&^)BA|@pP%H zpVEb)w@=utifDY$5Wwl&j+cBvnK7=+Fi&0^N@BCOw+E1Adv%p?8iV7kKg^12E}B$8 zqfDpPxI5Tb%U4wWJqsit=<_R~ZS;JO!PC(-yX_gLb;vu9Qn`5otT#LzbNJ_lqqLq1 zw@oCxj{jho?M%7&#b@;lfkYc~^IC^xVR`xAt53{S?TW7m2yQ^3P33VwAKk+#LEVQK zB}AsN6G7SSVIW$wmU)W{m50F_yY~{3MRJA=(A)!o>TQ9OSMELh^I(d5(BLo)P(IM|d zl~XVRlKov{cLgVL zKp!5nmXVcJ=+2A%J`%UXIdfy<&f8BRm^7ckiK3#QyykN<<1Mjv1@AwRipH__+G=^; z{L!&7O8_yk`-qDSp-#weY(qHY6B9gH9+4)=5d(DyJVFioRDyznHa0gG+kCxy1jj;(@ox>VOdy+Fy{6p0Z_aP7oCd@87h1C!1W$L`S zA)QF zvo^=q*EaBsSc_i_YOLbI!Ui1u@kF`x!@?a3wS|1zwOZ`PUJ&Z4NJ9d?Y1XGKE*)uZ z$6ijJo}L0=|L^NAwE5nH%}EurfmHnvOae11TEy513;FfoTBKDJ_Od#e zRf)ZKH~dizCc4N5(4LWW0XF?6zbv~Tq9*L2CY-UpS^Lqll)5OF? zqNjvX4PQ+ahFYF2Q=ehh7>hA(NnJ(A>$h@*sssuE@3pnHQBhG?twy8=2M0iyn|@P+ z2e6h8K?qk4I*>OwIYG%?mXgjEFJ<@v_i??wRYTUD#I2BxBW1(mF!XIm=~u6dow3Dc zchJLPPi{)Z^z`(A8Uik>K7*v4ZkPv5!OI~*2u~_j*X$6I+{fc%V z9){d&bYy0oDbzKu)+?8D(k+bN8PqEbufDoI|C22SkkEXUIqJBbF)2EkaA|%%_vds& zjYu4{_t5mn{URQJ1G%xFKRC)oDZ^1t|#ydd!A8qaD2^}KmhH4f`MUo zx=fcU)l7qa3==w8SzDXV=UmX($Y)Y}0993;0F?w(E^;QOh@YL5fL>{7YkPWo#{l02 zUW67kA@mT-)qbhvdvdarqCqeLtH8&NUs$fLuHfT?P^LR)XSJoJt$X~i?;*-dK zFFmx?c(7bEUNYt=C@aT-B?hi)oTJ_c32-O1v;4FKWD4@t&W@5wQlSy&X) z`7Z@Fq%GU9U5S;q9VA(fQo72qT%U;apy`~JBj7FZbc&Q3Ll(Sze8oS0fSrPt{n_>8 zOWR2Z}p5ys^NV`T_=<6&ceO;3-yF=y`Wiy{VV zIanVZNkN&iKzuU$t*`I7;NiW8Zy-Z5>v2wFzT~nyl$3ShT&($tWxp;O5uHRyU460E7T=`eLmyZs35kh- z2V*i~Iyz;BdSbaM)r8kaIS*TyG(BvBy;50ODfY%rvG1GD@Zg%hPO z*mJ)=Bg3aH)9qm@Tb7Rw*i-F2VEvJYk~-ar;Zs!)dBS)2zx;VE(+$H)brzK~_O~!Z zgH7Zlul%uNe0&_JCof*SsN`fMZ)83iUiy9BcM=+v+4QI5kn|JQQjz?N%#r~JWb)gd zmmoyxqxtP}Sof5Mj*bqv3qZ>NTz;1RQG3+%Na$VyI>(p;eN@f)BtA)@=4;F9{e%*+ zWk2wA4`mKLT|dvsI4My#M;UpYeqPn^8?#t#@9J<~US3|aTyJ2yJU8tLYx%@VM92@4 zT;ry_%#?+6YfSg_#7y5O&n)X`NMF30F##JuM6>YF@tdhKoS(#Q^X7CNRKNPFsx^<@ z|MJ9wTd$rKx<)?G5LrPv6g}UTe=UcLR=@xAW# z_QUNtK%bVQ8KfxXCp5c;Cy;fGU53j|owaJg=Y|rCHkAzG|Ms;o5oJ0|I4*fq9JKTgTo1kCirQT=tkoaiGRgNqTeFHM-fengxZ6DiV8qx?o-E zc()^evv~cr&#S8E>pw2oe4L?oy#u(Q`I;HT1EBKM)OoNvZ*M~=^%yI8SKkWacq z--l;T{+IXVLfmZidUAAns3fm2ev9bFlh?Nhz-<;57NCa>Ej6sb#rX}^>s5AtbQpJX zcK9LW62qK>u6wv0EA0D!48EmB&%(|vxO?sF13Ar{A1aaA%eR+RI|?J36b8SK<|=2C zGnjXEHQx6&|Jgw}?X%%{OV`eOsu4fd=1GaRtL&N3l*VBqD;rTxMj&%jUvKx}hPFfIM6(t+&kTWF$w}=wUugPn zD|8Yo+4WQx4Vj7p{@WK@32BoCip_a+NWf(!E|X*!YBo@7z+SF*KVG8niuzxva@JCh zTt(lS%8SxpvDvHfg}Bn+a0$jpLOj`NgH|tGOw5j_GR4Ple*MwoF}~MXBR%4VIr{gC zg_FD`k0YX!hQ>U@i4rg;O#5tzHofh@Q#=5kB1UdA!O0rGb>F8gF(Cl3AYbh=|7o1?k<)p#{*n#s5E-VLwHYQM{>7`SK!VfL?|y6}4E|92Ju z2!5|$zy7d%H|em8f5-5cu-9WdS|fPCeJf!7>=4;`v1_Z^?W`VIE3Q3TLWW?tIXEaI zD?3!8S%GejN%FzVj04^1jAO}mf6K1*f@nybS-Qo2%5&XC=osT>S|+R7q5Wnf`7j(urrN?k(2;`(T@Tk@{>FHk1V65S+FtUf3!!r ziy)#1yk%i_?%KK)vCIPZGOZlSMfAI?tLx7n$KNU3Vp>sKr2=7EkfQJ?{Esq&Qhu1P z<4#zo)ZrkRK666I^Sh4OjU#k@LS8BoFCaB}(XPy&zWG?ev{pzB#|-JYc!iv?Tj2p5 za(R8-4!|9zi#Q_vquPh?^RJE5uR~naH_Y-18vU}fL%v@*8)UZqm&u9#rsqG;C5crZ z7(N%El#lDw_(XbLkV5hsw+C_WZ&faM$hoFmn)x9{)trn17{N|VhZog9P+|sccQ4F; zD7NFWxsJ;!rmYL)!ozkhEG#T8>MJXMypa--af!YcfX%Ket*I%ksyoU20t@Y{&CG}} zhN6tUyTAB9ekWNu=JfW_-gYAB5Tz8lC2seX^N4zW zNQYUBVi0R#>=2n}W^-!9zLJ`zCXaQF%J6Nj=ETp0HYaZ-`h+NA9t1?h2?_F)3~gyR zh?K3}sozt&eVM#h|l`Ld59f3e+L^vb$5{d!P0JR*W{^e`?pb!cnh z^)Lw&_nS9-ZU=uab|*sWXnWQqug@Vpm&4l+*mzZTiw(&sDbVXoUHdoM!xcDIEjcQw z)1_^0heTqgk*@{;o&zS?-{)V}*4BV2&r}$ar7*yD%`_xqLr#Ae+7-GcW3I2^uDpP( z5cba3YFN!~Uftsx)05IB_4f34rQ(VAP&k!c-?(L=F;c&_&-1SYI#{o#=K69jpg+?Sy9pqQi<~rAbw}nHg;jycSk>U zYygyR=2^k0RUZ-hS9?IjK}SbtGi*CJUkOCS(9qOuZfXKhp!(-eUeB`)B^nvyoQvkn zQw;((2W#uu=6PHqA}=7uAY!~WVo#2vgUVsM-8E+ql+`b1(^Sp=)smI99s0)dN`?$I z0;nM`f&TIM2xPC2Ol%>q3wqvv509O0sB+0yswL%5n*CamrKWt18^(@AxUby#S>pn} zJ=~A%&!l<9gnZ*Emq5(CJj?$xJ`QNq@=s$iX=!P;dbF1>TYx&Jrj`%@X+B@dHx(+^iy$0ptYKkV(>w?L4wJVrTNr9Xf?tZ|6aLsr(-n!GM;K+0w; zSJE5UW2WnSQzf9dF1qszqPy6Xl$4m8nE_phH|H()^)wPpmL($IGsnin!(A+P!0;8K zx{D=IQVSZ*-otmfk|tYwEdBWzbs%aL$fpK(uQrj=d7W*5xX!O;_v63M4ZFL$1TSgX z*c^T#6(%feb)j}NbJeUReMC|D^h{z8=?$z%vF2Jx^6sA@Y-qR5Uo~j{;3BmrL}u^j z*A^7i18m6VXdGNzTxnTZj0pHTIQYh$Gv-$Tf9CGF(+A-!vLVp7(3zJvbKoLZRaF5I zYB3ro?myfzVCYIn5c4U76{+okKL7joZ@xU?%TwK+e}-;SbnbWqbIkXa1)FGT*TYwu zFXV%1#?)V6Sw7!}9yd2C(*xR<`qr+hx_Unk3aRwM$$y9gTIwHv~s-8Tf?z}Wf2iW0L2cNo&AdUBZ|73G4u!Gn27gPseTyX zvQd$d!B&2;xk0U&O;1cux7%de+S-7jMa$&n=5o3nX!Tk8@yUh>`#&)=GpD7b48HfB zpPTvQjmI=pT(RW5X40{o5mL|c{mt?uRa;*5X??-Lw+xpXnmQ4U$IT&pW8+m+eAYky z*kL^YJ)|ZlD;29!|;%r}9XtEtS z;`q5g@rc{Z(o(f}GF`xZBvXVeroZ|bCJdTWP*4C2z+d0|U_Y?~Jy%zE9teuR1_sck zbXd?|UHR>M71m480=WzuP z-h%vmG^g+Ppn|KYssbNectyVrT;?=E&$Et^--ClGJPu3yGZnMntBP}S)`0g31Y{4v zwt@l)DXGv?XJR5E5TQdLVXr7IMvm#n;6Uep9g_*(42;u|Zm2M>VMgNFl`QuQPUAgh zmi!~PNUT(1D_BaOKpn1lkk-q|sdC+)URhar>Wf`PqWMu#0W1Pu`&!<57066B|Nl`q zwJcw?Pqqk2%J(G&XYyyNU78x>CY@XemHQW=zLdbi#=&WAX<1uehY1J?DSW)2Mb*>S ze|&g|jEqF64GIHpVGbeCX2yBs_JB51w701%&c%K@a*giUbJ13_D4wx4K;P}w{w6OK^=*{90U?f z9Vp0eYp1z5ZK`ub~_#>;KKfZK|ZrBYH;Z-Dy-ym&o5tN+wx_4y59RU{NNs`GXz zRvZ3PndOg5e)pO1{%S88|7PsVMq(PTa+;x8Z{xP|+aGXyKX|yg^;^9zyS|{NYY3j7 zocx`g?Ud>d!5{+`aXi@d3CG5Y=~;lNgDd^_@85C(IAE~!4&4h_M4OTVNq00|0LYj? zh#wpsb+ES|`K1jI67YP*RE+O@L$t!${2rSCY6FciG(5bHuO$Z!~xW^GnIrwxqeaIWyC+zWD}J9}pkqtdjpGN2{x)MZ)I< zm5)+4R^JsfDpW|nJKL0SXy37UgdtZHk11VxFSp9Qu^lFzVkVpdJJCL4$*T2Vw018A)HdT$k=XkJO z7_iR)5CgiE|EYq1UU;HXY!01hwweUrlXlDDgN{831SuxbnwW}m3qkANd^-N>9 z&XchkUw(SD*bMp`z-0xBP`9A4FywnN?TZw;Lpg^8^nngPpdkh!MVo0F*PZ$=AV#0piT?$Iu$+98==Tg+2p)V!!J5s>&u?vQm8J#( ziFn96Ma2q3O{0+oZQFwRwWxZ=-T2RWG37E&Zq|15m`G%Tc*Kj$RGc4=h9@SRfxU4d zkMW_S(RrJin)+*eyfs%!b~Xj1RIkyA9p7wvL0e$|=3!@kHj0ZV|6$81GJFPKWx3RO zWMLK{sekbxR|*3Q3)(=<(CBD`S{vM`u%L0B_8wtI>ez^fnoT(zBI}!nn?l1Scg5ns zz`(-7?LQY`S4WEi;2r{!rm%UTRtZ@{$@PROnu>|$_hWyWSB_{d_!%uMNBtY|-EezT zf9m{cMHx!q0fxDyC7X6t50HtLmNI~4+4keh3sOpsx)s+CC$}PH^<)j|3l!Iwl^aLd zQTx|=12z_x<9~tQ04hBP5Sr1PY19~}Iy+Z^4-3q=WC3?p2*96zBt!qQqtD_WnzwG) zl1S_l<>%!Anet(8iU}(&^J6MdzjCs(JGW9?*iHS+A2)FtN;1TKCO9&JVT5TGdj}a? zmHFYFEmv1p0sH}VR$pNI|A%ysX0Jfb$>|L8>cD;iu;=0{kVhn;P!|GWO9j)OwzfT- z#Pg%A{A81WSFZb)h|iK0e6?Exp2q@g#mcS|=Fc;H&PIq2awNPrYEfWs0m9kYX$%T& zsoxlsatzcW0RaJSZgpTk8}M3=qygaQuG7?uf_9L4Wi#+wTWH(XjCOx<3PUqCO^erS zDQ7^0E|8FA|Ed<)e}Dk!iHr9xE|Nmz4b-WuNq7B-yiwhTU@%;Ut6*d93eQbRdxTHOCzGw=82pwluW$JpT zR6owh!d`bV@m#!j@zl?D(won0=Rg9y zJ2x>~4Ux)X=6NkN-i26EFRJpxFA;tboo#Y0$1 zLk;n7lm7-z!xllFe2EYAgl`Z->n&J>W^8AO8a=7#L(HwCZku+U&xV*=~unQ_8Vu zs^19ii|d)VwUluJAIeG%#%EyeIN1@VR$;1t%6bnvg$;Ge$TsS?y&;6xMF()t&pq&8jjF zCnUUj^XARgR*i|rQR(Tx)fj}_e??svh0W>q@NljGYtu}p0=WQbw(!!eZ&C$;&wXos z9e7$JHth$%P5S!vE3nNhtbf)&f$WsTy^+>-P-FGy&#=`lhh=9}b z8kH$6*I?|cwF0aN7$M-a;MQ{9ABE$oY7nDA68SdE?1YBH@B4jTU^Fx1iM@&akE+1s zZsFh{Vm(d`A|FfW|HlK%ZtL0mtOHT{XM&Y9@6!C2%zczpChOROk~-d>`yvRx#7OLH z59QPlkM9V-(A{LFT;Chm${==u7PomKBNw{@+o8^b33I~dSJ#QtdR&qCxGJ7&o-}`4 zQDK!2aY?487~hzlw?#(#@gDFo(04#s)J?B-z~$k5R0OyeNT{e26BFxs2=fAmrKahd zV&#A|fqwl#F+@`HDn()NwcX!7tLq?+cMJjvJ0JxK{K7?r`DcDPHB~jlwLX9EvTsK> zm>1#_@gxdF^djhW6n8X{W*=SXZuYVu{)Bv=bxy{`MBoEm9G` zH1SdK+%d2**4Nk1cgCo|1d)Pmjqmsf(X=)yh-Vz0T{fX84fW8#V*LantdJa1BYo^`u?S7AU z+nN5f)YK=Zl^bqAuzP-b1a30eco*V(X~7iP>G?O{1t3QWmJS$?`V~et`Kgumn?{OE zzCA$splILS-T9IY3(Ot`FRL_;pf)t*U$)xIX9BpQ_(>bTeJYk6ThvIq!6Jb=m>5#i z;#3U}y}^Dh#PRkmP_GPt&Hjf6nL_$KgnkGMNy}$<1mPK>@V^pa4J=qivZw zx1PSfl;q?fV7!h%7>~8OWy37^`Ku1T(}DyY$bA627mEq+5;$&UdW|X|!i~019fE&Y zDh9G-AoTk$S^c(!BqAYQ-FFHE=5TQ0ap)9&8vCniLM0`EJ^_V79j67Ov&MZ$wXbS) znhfh~XIM3S1Vw;y12FZ!oJ?%2F(!6wR8958mz3|{85kLX?(!8bLFm>&jBkrkprj2a z1f3LM$N6@Dk^ApWAddaz3z!#h2YM$$I}`0#HBDGUjhmH~6;K~w;y#vw*unhZ;2zj+5%_FRc4c)94L85ys6eFM ztjUlB7q`}K{;j{iKlKw=>Kos?i^<8!>?DeBpKfI?9`D^zQ6qrN@l%BIu}rrf)VjW&=BKW%>;FL7;6mVpGqiwT&+Fk7b zB-0vAQIfy?Pcm&5yQi_Vba-j$2~b@*!a-mF)>KtB0m}-Ql~^$_oj(cbz+XB*8USMG zyv@PprYkUXK&TQp)^3aOTDmHTkgoL&_<8)KxOQ=pzhv`B5b|vLTUBSIOx9zJ)Sd4& z#xDil`b}8X0Gl7^9?Uv5RwtV0mzN;K;d`3FFihV#W+tw&u>c%YgCjivfIzGXH zIRVfO0~>+T3R<_<{goXUPiXkK9;@-mh?gO*q=X498?f5{t_P+8=@iJDz!C>zAtpT1 zJ$qgJ5CR%1x<~|wjQYVURikbft90*jp4$#1);a989>HdU5r0OiP};##>-^up9zdK4 zY*cV|u8xaSR#jDXa;gG@#^cnOFX?E!o*o)AGnWCTL#SR3I;GtP;{@2)X9u&KVAo$c zLos$S$jjMN@B?%ncZ${#*D*y{j$EGk|A|P}!Vr_EDWRo>Xk=a7zc_JBpoCLVfvZOx z*eC@8)*!jNv9WQq)G9y?bBIY!(12}Rsl~}z-TrhJfz}+mvyNYHzvy*!FspRve!P>W zyq`f*@7>HwudRz@Q-ZP&^G?dIls)B!`_Ekb*2yoGFcykNFs^fbIFJAOwQ#b0GC0Ae zrY3NsTU~asiQDLrFfU*yA$gs1b6x-@>DJqu2fifbccJIta6&Q*MSO5mFbP2)GyDW-j!}*?$eh}PLtPk&g}$U9 z@bk_2^XCs}>_FkF;vb+BlD7l8H$Zei6%UD~wAsHNM}S~vc4Qva@qGNgEbEnlWimCx z4+$7U335zT<(>Di#l707Xms7z@bbC?3J!wL7f@!1iHW^EJ%NTmVF&3vQOyB80RYvd zc7H=TxyUIFsA4`C{ONfLwM}n=+KmwUce~yvq52q(I1C*@K_2V{1SVrL_eg8Mp*4Os zM{K#>zrMB>h^nL~gJFOEC=nB<#>Qd-kt`2}oWO61@Vnew5GQN(=26X&yPQVORsq2R zY;0`acfpW=cZ$Jiz4a4YcWHVLRAngto0)q?`dhJNF61ivlEO z;_`;0IZYDs&=2vh;LLO5cGVo*?6hGx2$AiLm=3X|Ng%KWZ6I9Q6c2j zi_aG*$djWcP47Bi6*66^HqSn@X0?cF@?I@GnuG^*sb%ju(EJ}d^u~|l`@zZzn3|uE zb^*=>2;14(*fbdc6aqH!)cm}+QP14`{PX3EA%LpgJQ*ag3LQX``zQaDmxBoCDd0N! zg@wp~zG0r~)f|KDrpmlzU2yvxeUN=;q;;~=`oLr6R$gxuPz z{VYlU@;`LFEUyb!)tGIK|Chh^_hCT&{gESHIirnG=D{fjH5mGxZ+2&cJb;ol-{?$- z@@1~gx4EEz7P7Ou>*`$jgAp)CAWDA{gntcyrv(~X6sQ}^j!9dD&q{G7IPb!MJxjVgIGn}W3T+CF7C0BkU8SDB6Fh-)!lU4M8>jcpC4HUX#y*cZ$Z#mB`-%gX~FU<2H7fSfS3 z3ID0fGiCEVb4BD}ZUGD&nOYYS-H}g%7XTo@7jc$?q)*fRb9_WO*wF!_xOld8r`7tu zowskIxvMJf$e-#izufaoqYi#(KdbvD8pFf|K0%>ERZFV@n4uGifLjPz3@3v+@ubn- z(xL|vgTH5Yja|npg?XlGxfKTue)4BQV&-L((5kz+;SCEc0F4N zDFH}iD3E}FG8$v*pseC1^91Ux{!PHB0g_!fC1GMJ0kFe`8CXA{BsT;81j^-|_y>FY zqrpU`nNn>)^Ej)VCojT=hvk~K=du3X>^5Jq+_7JOMj=@h70+O<^(qroKY1xBf=M4w zHhKmIEPQ-_zzp_#5~Zk#h>7=icG$NBelmQfoLAD(nX@*Z2HT6Dv>Hfc03Kgm5B>`c z0?~iA+EUVBq4!g^t%(cZJwe}KsK7iM)py`T1~w919KB)DYxpxV5}nUc0}|VO&RhTT z20qvVl=M|NfIvC;r-QAP9<&ZH0N7WLt~#bBCV*ieSd&n_)ZZonGY?og2$29kUy0iR zhZqFrN>y@z#0IuZORt(S_XiFsbyX@B7E)Z?sR~qsc0YY6bOTHrOWE$4TbbTn z>>|+xLhp$lVJ>C~)pNk$&r5{!)ow&QA-LBuJnZaR4UWdo`88KZC>40H*0CClgL(m5bHf-Q5AR20M`Q zFpmk?jdO`Rz+FP32!wj?tp^3cf1_6hlnf~JKEA%%7Z70Zf*m3IjhRpi(_?%FRTTz8 z%iGrZctSN~Psn6jo*o4Q`O|1HEi7?y+hwwO$s;3`_9OwIHsqbIZnLQ|$UA`Czq~wJ zAbvKTt-6}p!`UY4xKxw4x1%grIWXI7@BW+)(NR)0h`f9xcVi z)}VIC$b^H7$5B1OGF_?-_OXV!`;B)H=(l%Z$-+sU%aUV4jM%*=L1h6=tq}zI zfWte#>i|{*_nbOJ1;M<6Ut1)dIKs2F#QqnjK50c!gK%{fk>#2 z8qvGAy9)|n^h+6FZ;xaM*_xP)FXMnHiWK#@5iI}2^*1LTF0Owr+5G(Z`FZvBcfj>D z%XAU}4yrr>K0})$VT26=q zq^P5bxNVF|Gy$|GAt3?H>*fXlR}f?{jALOq`B1=AwZ^|kT{|86N`gWbBC4HjA@K4{ zQn4>5(Nj}W?)K9@O2La87%YK_Ac}>>r+za3qLPwlS}?_%m$%Wu&+UDEM1zSGsHmJ@ zRCJ?@HDdF!-N;v;k{*H9o$URo$Q4v+10JB&hb`G6Ld5^4i!%?0a___V(-~X#tc8@a zFEL~}NHqr0G-QUcWoMlf{fnIM&x!=9^wrkHsiPb`BT=(SOr+fo{p z)&9&9Grei2E2-waA3v3N&JNQy=c0VrsxY|Y5B=TLg3(jbewudbAN?$#C}F!AK3T$g zwc_TiTHz6W6LD-XP2y(mrKvi!7~k)g@mp`u2Zxp>al#MqM_1V-RKF2{%J<*Cmg~n< zoblaae^tM5uQ0UO@uC=ba+mjPHhC2SPxR{$Q{9}BtPKfvW zo0NEjY`+k1PUEvDtYETImyNnOQ>FTvjEaFn;&RlND4Lzp*-@k|jO!Yb@VAM&wx4`6 z^zcH+JWO}ue-LaK&i%`Me3i}iyp(j@Y^_2{a&6rQnFpoS?`_BdxVm$LBhQOuHQh0{ z;QU{Hwpl5)2%z5NDUP@#>QUl*@TfUMf{_K=Koyl%EAI($-Q*lUkVj2w=1Hj`dtNI(OIvRmxnF}i8@zY+3yg(y!9-EiEYxXuiuc!aS#{u+|scu z#iPM7dTK0xW_Ao4I4^36)T?QwGojq(?b{AM>~~Au#a)QqE5vJaz=Dh#u2>Tt$f%#c8mF{^!(ak8r@LPFJQjn!Pt>$l`WqHqiI8s2REkkG zn-oaq&3Vb)3Gs=fMJ5T(IiQ^o4g=mz3|@?oEj8+1FoV|&Zz?we_6+WpWUWPCJ@V`0 z76yVjKyCpP&q$152fpnkU|qPOhaDMgCxoyw*$Vq zZdU8k$o1iKaAO@u8JM&%1ONlT0?ZS%!9E2ddw2ptc`z)eno->unQ!#M?mSb^h2RSVBr#DU{AR>Vhx{T=UgDVIMP@>)XLYm)YLbf>tv!XA|@{ z*xW55JUczYqY%&~GCFj#s=j8@&}wFYQF#>F^T_aVktEWF|1`I;SzcNK66~8p*b05E zHZh6rr+~~cLG|jjxC|Bp;T(cejpBhqLLwzSdNd3sB2Z?qSS-`&!$UX@l2udCs1GX0 zBL602jeRR|@iIe?(MplIzI)$Sk=UuBhQzI)i@}Oe9v&86R<;U;!vej$ynGkLzjk&8 z`}(TD?1aOQ@$;XoIRdlVDPqJ_WPc9^bmNAoBK1DToEv#LTk<1!FYI|t{a8TV2($J< z$#s&D&RiPl#|>W4>I1+9Ak;nVNWLO|Ch5u|TeC=ih$SdKxV&gZ;qd&G<_E+CaAyGS zQ(dhBgNbmgt>sTZDU=2DEfpyjvxD2?+R$T{8WSOo)8{GFxAl3(ALKtyn}nqeR3=DR z(7k0hr<4V=T5?7yO91i%q!KVefwm{kLpp#3|7`7rNvGJge2D87h!ENZ@m+MUshOD% zc#VKd2UNujkPz0?tnEm4s%l`h$2?7!{~;@?5HQKzkm`ruqlHQ;6(7R%GE<12Is0}) zYzZR2Z-ROUAAHk@#WBD}f**)CW&((bDI&*|908w55Qq~nyWw-dxZ!UQ-N405u-TyD z`1#8F5)!q1@GrGPEu22Ht4adR9#e#d&!_?rg*B@EfjvCe^#Z3pHwi#E&3>IYqKgb;_8jnfW zTquwlD^g7WdYq+*Z$XOZ4dJjcAJfJJAijewDGL}1VBxX6Sw~E`m@6&Jiv5>T#7?3|Nx2VjO>jJ09g{X7S*vwDaVf5MSo!JPTk#e%K(6crB z!5&4oX~=%LC8bt6Hl`{M{-E*nIRCQn0P>PlHmw~{vKJaI38oh5@7|?0J7Hj&EgnMCejE`?Xb>ZrNTBi27 zpUB1Q2OiLWte(y%Yb3z-dZ~|^)k@+&ES%9ZbVB=b_xY1yCpdV0glB1I7uqmL#_;Iv z(^29^kF*#Z!)=k07;p)k>x#X>hsrD+Shqia{FaGFW_JT;c;WMp4Qg|=VE3lwLu&Il zoi--wL8pJUaBy&_NufAaF3Tuh>j!86*jTNFCyycS1qSMVeogaPpHx8gpYQPemwt=$ zM$f!o*j1q{QwFGFA&4?O(;)E1Qmho4*_cJCBt04KaJ@LH6Bc@^*Z8ORHyr>0iQmwdSuZuN-ZMwnKC3S@h9xOC?Hr{2_|0=IgZcXaAs z^T|y(byOGR-a9m7-2;5z2lu)XtSo}1y9!7Y9WNXRowKsBvABOhQ%kEZEtRdWajJh)^Yw}J;F=PLy`AWfYDBZ}m7gz|Ps`jcZ7mxK zC{ONyGeB~3a$pJWuxn<|3$^2nenY74n%PVqeI1_1F7Y0JYQ;rb(S>KA`Gt{1)u($< zzM<=(PjA+f_I`ic0#mYW`9*W~BaqL=y~g7cz23xY-RTE7nDJJ@Z>^LS#xReQipze( z*`wNAuL}DY5Q4e~Zj=ZRd1pMOJLzJ-$f8T#4{U5GeD`@seSQ8ua^2Ngns8Nae6G*I zk$3IMY?+1gtJ_p+Mqg=6M@jse0eeEewoh70J5Q8Att+3JQWQ~!oxvI7o|?aWj*JXw z!jcNw<~x{lJ`m45y~^Uuxu30GW${~NV((lDIDqclVW;PObKgbtSe6cTt}25D12R^B ziQkn-OrFw6i2cf1b1psBL&e$M9i3x>ybA%<0+h8C75<^0S4k$zm>LMOM)6(i2X-uA znv%Y)pH@{(b$Bd&Ar=^&k%{)cL!38wW@cwMrS5|(_>P+O+h{Q!68f4UUO59L&{g-` z5z(a0k6>Xuq8@@^ao`b) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + v(2) + + v(T) + h(2) + h(T) + ... + ... + h(1) + + + + + + + W + bh(1) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + bv(1) + bv(2) + bv(T) + + v(1) + bh(2) + bh(T) + u(2) + u(T) + ... + + + + u(0) + u(1) + + + + Wuu + + + + + Wvu + + + + + + Wuh + Wuv + + diff --git a/doc/intro.txt b/doc/intro.txt index a6b91acc..19ab4bc7 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -49,6 +49,9 @@ from energy models: Building towards including the Contractive auto-encoders tutorial, we have the code for now: * `Contractive auto-encoders`_ code - There is some basic doc in the code. +Energy-based recurrent neural network (RNN-RBM): + * :ref:`Modeling and generating sequences of polyphonic music ` + .. _Theano: http://deeplearning.net/software/theano .. _Theano basic tutorial: http://deeplearning.net/software/theano/tutorial diff --git a/doc/references.txt b/doc/references.txt index d1265bda..879903e0 100644 --- a/doc/references.txt +++ b/doc/references.txt @@ -10,6 +10,8 @@ References .. [BengioDelalleau09] Y. Bengio, O. Delalleau, Justifying and Generalizing Contrastive Divergence (2009), Neural Computation, 21(6): 1601-1621. +.. [BoulangerLewandowski12] N Boulanger-Lewandowski, Y. Bengio and P. Vincent, `Modeling Temporal Dependencies in High-Dimensional Sequences: Application to Polyphonic Music Generation and Transcription `_, in Proceedings of the 29th International Conference on Machine Learning (ICML), 2012. + .. [Fukushima] Fukushima, K. (1980). Neocognitron: A self-organizing neural network model for a mechanism of pattern recognition unaffected by shift in position. Biological Cybernetics, 36, 193–202. .. [Hinton06] G.E. Hinton and R.R. Salakhutdinov, `Reducing the Dimensionality of Data with Neural Networks `_, Science, 28 July 2006, Vol. 313. no. 5786, pp. 504 - 507. @@ -32,6 +34,6 @@ References .. [Vincent08] P. Vincent, H. Larochelle Y. Bengio and P.A. Manzagol, `Extracting and Composing Robust Features with Denoising Autoencoders `_, Proceedings of the Twenty-fifth International Conference on Machine Learning (ICML'08), pages 1096 - 1103, ACM, 2008. -.. [Tieleman08] T. Tieleman, `Training restricted boltzmann machines using approximations to the likelihood gradient`, ICML 2008. +.. [Tieleman08] T. Tieleman, Training restricted boltzmann machines using approximations to the likelihood gradient, ICML 2008. -.. [Xavier10] Y. Bengio, X. Glorot, `Understanding the difficulty of training deep feedforward neuralnetworks` AISTATS 2010 +.. [Xavier10] Y. Bengio, X. Glorot, Understanding the difficulty of training deep feedforward neuralnetworks, AISTATS 2010 diff --git a/doc/rnnrbm.txt b/doc/rnnrbm.txt new file mode 100644 index 00000000..eba68721 --- /dev/null +++ b/doc/rnnrbm.txt @@ -0,0 +1,350 @@ +.. _rnnrbm: + +Modeling and generating sequences of polyphonic music with the RNN-RBM +======================================================================== + +.. note:: + This tutorial demonstrates a basic implementation of the RNN-RBM as described in [BoulangerLewandowski12]_ + (`pdf `_). + We assume the reader is familiar with + `recurrent neural networks using the scan op `_ + and `restricted Boltzmann machines (RBM) `_. + +.. note:: + The code for this section is available for download here: `rnnrbm.py <../code/rnnrbm.py>`_. + + You will need the modified `Python MIDI package `_ in your ``$PYTHONPATH`` or in the working directory in order to convert MIDI files to and from piano-rolls. + The script also assumes that the content of the `Nottingham Database of folk tunes `_ has been extracted in the working directory. + Alternative MIDI datasets are available `here `_. + + +The RNN-RBM ++++++++++++++++++++++++++ + +The RNN-RBM is an energy-based model for density estimation of temporal sequences, where the feature vector :math:`v^{(t)}` at time step :math:`t` may be high-dimensional. +It allows to describe multimodal conditional distributions of :math:`v^{(t)}|\mathcal A^{(t)}`, where :math:`\mathcal A^{(t)}\equiv \{v_\tau|\tau