diff --git a/examples/ladder_nets/LadderNets.ipynb b/examples/ladder_nets/LadderNets.ipynb new file mode 100644 index 0000000..8cc3830 --- /dev/null +++ b/examples/ladder_nets/LadderNets.ipynb @@ -0,0 +1,1047 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ladder Networks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References: \n", + "[1] original (Gaussian) denoising function - called here Curious AI combinator \n", + "Rasmus, Antti et al.: \"Semi-Supervised Learning with Ladder Networks\" - http://arxiv.org/pdf/1507.02672v2.pdf \n", + "[2] \"vanilla\" denoising function - called here MILA UDEM combinator \n", + " Pezeshki, et al.: \"Deconstructing the Ladder Network Architecture\" - http://arxiv.org/pdf/1511.06430.pdf \n", + "[3] for a deeper dive into theory behind: \n", + " Valpola, Harri: \"From Neural PCA to Deep Unsupervised Learning\" - http://arxiv.org/pdf/1411.7783.pdf " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bit of description" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from PIL import Image" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcUAAAFLCAYAAACne5csAADI3klEQVR4nOydd1hUR9fAf7ssHaRZ\nKKKCnQ4qakyMJrZojIktJqaY3uub90tiiqaYvOm9mB4TuybGbqKxxK70KqIgCAhI78vu3u+PZS8s\nsLAYEJX5PU+eyC1zZ3fPPWfOmTNnFJIkSQgEAoFAIEDZ2R0QCAQCgeBSQRhFgUAgEAjqEEZRIBAI\nBII6hFEUCAQCgaAOYRQFAoFAIKhDGEWBQCAQCOoQRlEgEAgEgjqEURTIVCZ/y7wB1igc+hA26Q6e\neuk5bpt4P3/k6jq7awKBQHBRUIjF+wIAdHlsfvYp/u4zFo/zkRzYd4DIcz2Y/f6vvHuzN6rO7p9A\nIBBcBIRRFAgEAoFpyg/z6pyH+X5fArm1Dgy++R1+/+VBBmmOs2jyDbwTbcXgG5ew4dcF+FwBo2dh\nFAUCgUDQMroCNs8fxPRVNdywPIVNt3tigZqk9ybxsO4L/nzeH+vO7mM7IeYUBZQffpUbQrywt1Rh\n5eLP3G9SqAEqji/i6h42OHgFM++nNDSd3VGBQNA5KN2Y8MLjDKGCbW9+Q0I1ULyfT9a48+SCoVeM\nQQThKQrq0BVsZv6g6ayquYHlKZu43dMC1Em8N+lhdF/8yfP+V5LYCwSCNqPNZsX0QczfZs1tmxNZ\nkr2A29PfYNeS4dh1dt/aEeEpCgBQuk3ghceHQMU23vwmAf1A8BPWuD/JgqHCIAoEXR4LT256+T68\nKWTNyy/z0lILHnwo9IoyiCCMokDGhoCHXuEGe0j67H/syk1j7XsxTHruBnoJKREIBIBD+FO8OMYS\nbfR37Bn2HDP7WHR2l9odET4VNKCcg0/5MebTbELun4tFzkR+++MerkC5FwgEF0jRjgUMnZHA/8Uc\n4NnBVp3dnXZH+ACCBjgQ/tSLjLHUEv3dHoY9N1MYRIGgy6OjMvsE6SVaQIekU+E95xXuHHjlGUQQ\nRlHQCJXvPF65vRfWw5/j2aucOrs7AoGgsyn5i3v9h+Az9kOSi6P45M2TzHx+Ej2uUOtxhX4sQZvQ\nVZJ9Ih39QFBCp/Jmzit3coUOBAUCQVuwHcSU8R44qA/xxt2LyH/0B54NsOnsXnUYYk5RQMmOefSb\nspo+7yax97qVTHvWhqU7XuQKlnuBQNAWdNUUnM2DXr1xs76yfSlhFAWo035i3pgnSBp1E4PUTsz9\n5CPm9xfLMAQCQddDGEUBALrqAvQDQTeu8IGgQCAQmEQYRYFAIBAI6hA+gaAJGRkZaDSi0qlAIGie\n559/nvLy8s7uRocgjKKgCS+++CK//vprZ3dDIBBcgqSlpfHhhx/y2WefdXZXOgQRPhUYceLECfz8\n/PD19SU5ORkLC7F6XyAQ1HPffffxww8/4ObmRnp6Og4ODp3dpXZFeIoCI15//XV0Oh2pqanCWxQI\nBEakpaWxbNkyAAoKCvj88887uUftj/AUBTLJycn4+/uj0+kAGDBggPAWBQKBjMFLNNC9e3fS0tKu\nKG9ReIoCGYOXaCA1NZXly5d3Yo8EAsGlQkMv0cD58+evOG9ReIoCoKmXaGDgwIEkJSUJb1Eg6OI0\n9hINdO/enfT0dOzt7TuhV+2P8BQFQFMv0cDJkyeFtygQdHFOnz7dxEs0cKV5i8JTFJj0Eg0Ib1Eg\n6Nrce++9/PjjjybPX0neovAUBSa9RAMnT55kxYoVF7FHAoHgUuH06dP88ssvLV5z/vx5vvjii4vU\no45FeIpdnNa8RAODBg0iMTFReIsCQRejNS/RQI8ePUhLS7vsvUXhKXZxWvMSDaSkpAhvUSDoYpjj\nJRrIz8+/IrxF4Sl2Ycz1Eg0Ib1Eg6FqY6yUauBK8RVVnd0DQeZjrJRoweIt33nlnB/ZKIBBcCqjV\napydnXn88cflY1qtlq+++kr++/rrr2fw4MFG9yUmJjJixIiL1s/2RniKXZi0tDS0Wq3RsSlTpnDq\n1CkA7rnnHl588UWj846Ojri7u1+0PgoEgksHtVqNtXX9BuQ///wzd911Vyf2qP0RnmIXxsfHp8kx\nKysr+d8uLi4MHDjwYnZJIBAIOhWRaCMwQgQOBAJBV0YYRYFJFApFZ3dBIBBcQnSFQbMwigKBQCC4\nIK7EgbOYUxQY0TAbVavVolarAdMjxI4+3pnPvtyPt2dbbm5u2NjYmHyOQHClIIyiwIiioiL53x9/\n/DEff/xx53VGcMnw4Ycf8swzz3R2NwSCDkeETwVGdIU5A0Hbabx0R9A1aawfrsTwqTCKAoGgVYRR\nFHQVRPhUYIS9vT3nz58H4IYbbmDevHnyOVOjwo4+3pnPvtyPX2hbOTk53HffffIxsV5V0FUQRlFg\nRMPF+wEBAVdctQqBeaSnpxv97ejo2DkdEVzSiPCpQCDoEjQu+i7CpwLoGjkHwigKjGgo9FfiKFBg\nHiqVcRBJGEVBV0EYRYFA0AThKQq6KsIoCgSCJjQ2ihqNppN6IriUuRKjScIoCowQ4VMBiPCpoHnE\nnKJAIOiSiPCpoKsijKJAIGhCY09RhE8FzXElRpOEURQYIcKnAhCeoqDrIoyiQCBogjCKguYQc4qC\nLkdXEPqORFNwiuxK0+d1JRlkl+tMX3CJoFQqjSIFwigKmuNKjCYJoygwyZUo8B2JOvVnnl60l7I6\nJ0udn8DeLZvZHZ1Ddd01SrJZ/n+vsSPv0jeMDb1FMaco6CoIoygQtAfqeP43fy0jFy5gsLWO4iNf\n8c6KZCzcncj8YTajbl9GmhpwGsVTj1qw+L6V5FzidrGhURSeoqCrIIyiwAgRPr0wijc9wQf9nmWm\npxIo5fBnX3DYNoTwYddw1+svMfDvN/gyRh9XtQp4mEfz/8vrR1uIs14CNMxAFUZRAF1DPwijKDCJ\nCJ+aSwn7lx4hfP4w7AHoxtWvfc+SGd5YAbrKAgp1zni5GIxMTybcac+a9amom2lNVxzBj5/v4lw7\n2KGSiO/5ctc5LqQpET4VtMaVqCOEURQI/jXFxByoJqi/fd3fShz6jySklxVo8/nz7a+wfvYL7vGt\n35bLcYAf5XsSKGvcVHUin96xkPxrx+Bu0fhk23EKmYTrsrt45WBJm+8V4VNBV0QYRYERYp3iBaCp\norRKibWq0eukK+TAhy+yMfBr1vxfOE4NT1sqUZdUNvLg1Jz44iFWX/spTwfatE/fLLyZ+84Coh9e\nzOE2Rmsbhk+FpyjoKgijKBD8W1Qu9OutJfN8df0xXSGHv/mUY8Pe5NMHg6jZ9SZL4+vP1+ZnofL1\nwLphO8V/89IH1jx110CsaD+U7jN4MXwNT6/Ioi25PcJTFDRGzCkKBBcbTSa/vfE+B4pNXVBN7NeL\n+SW52tQFnYAb4+5wZtfB/DqjU03M29O47pHXeOZ6DywVCrrf9A9uHgbvr4bUvekE3xqGU4NWCv/+\ngL8GPcykXu39WtoTctsY4t9bR2YbrOIlk2hzWcpE16DTokkdKBPCKAqM6NzwaQ1Ri29lfdi9jHHW\nkLHuGWbe9iiPzh3L0EHX8NiqNGqwIejOSRy74wn2lF/k3qX9zuInn+etTz7j48+X8dnCt9lToANU\nDLxvEZ7LfiVFA2BD8EuHqJQkJMN/VTuY7VbXUGUUP+0M56Wb3Ru0XkHcb4dwviHUyFC2F46BNzHk\n5CoOF5t/z6WRaHNpy4SgM+hgmZAEggb06dNHAiRAevnllzvmIbVlUlG5pu4PjVRr+GfBCmlkj2el\nmFpJkqQCacP0blK3ab9J56Vy6fAzfSRcbpf+KtZfmv5uX6n/Rxkd079mqEn9Rrpp4LXSOzEVkiRJ\n0vnfb5LsbadI688brqiQji+aIM39KV2qbbGlainhwxulmd+dbnRdlvTVAKTrfi/sgN5LklRxSHrA\nzln6b2y12bf4+vrKsvDCCy90TL+MaCALBi5hmeiKlJSUyDIBSKtXr774nehgmRCeouCiUnP6V+4J\n9cbNwRrXAcMZe811PLw+Ew1QcuB7YsbPYJAKwJUZGwoo2HgLbroyMtJKsR0yAm9bfTu9b5xD1tKd\n5F+MTmszWf3kc+wf/iIPBtkBWirzc1H63USYs+EiO4YtXsszlRtb9MY0OfvY5fwGP9zng9E+FOoi\nMoqs6OlsaXxDdRzv3RDIhDejaD1Pppq4924gcMKbRDW+WOWEh10pp/OaWwTSPBczfKorPsq7E1yw\n9HuI9Rn1fbxkZUIAdE74tKNlQtX6JYKuhNTB4VOFfQj/tyef76xziDtynDN2I5ky2hMVUHU6imof\nL2SzoFSh0pVw9IMH+cDyFbb/Op++dRJr4eyOKuWMGYbi36PN2MjHWxVM2j4SZwBKiNx0AvepV+Fh\ntGzCmVGPPNFiWyqPiTxxTzMnJB06SYFK2eg715ZyJimeJM8SM9Yaaik9k0R8kiclTS6WkLQ6dCby\nJB5//HFOnDiBJEnodDokSSIzM1M+v3z5cvbv3y+Hg3U6HY888gj33NPch2kj6iQ+mnk/hyat4aD3\nVv7v5oex+es7prkpL1mZEHQeHS0TwigKLipWvQIYCoA3Idd7E9LgnGU3N4gvRgNYAGhy2PrSHSzJ\nmsgTc2xZ8cBrSBs+51pHQF1KuZVdu2ZpmqL6XBLnVIN4LLCb/kB5Ilsj7bj62f7G2aP/BksnPB1r\niCqtNT5uP4bP0yU+N6sRe8Z8no7U3MWaUnKqHfBysWzmJIwcOZIvvvjCZMvZ2dlkZ2fXP8nenunT\np5vVq1axGsozf0XzHwslMIV/btOhU+qDWJeqTAg6j46WCRE+FRjR0Z5i+YHnGD1kODfe+yT//c89\njB/qy8Cxr3C0Apyvmovrvgj0y8xL2PPU1Ux7928OLn+RO+Y+ytKiIfSx07dTErkPp9vH4druPWyK\njVcwvi4O2FsqAQ2Zf3zAmqIApvo7tN9DlE4MDrHiZGpJq8sm1GlreG7+E/xyqsb89qszSKjyZbh3\n8+sfb7/9dgYPHmx2c4888gjdu3c3//ktUH7gOcb4h9fLhP8ABl/iMmEaDZm/vcH7clqkloqSinov\nvzqWrxf/wuWaKCtdpCUZnaon2m/2U3Al0Lt3b3kS/dVXX2339s+vnSYNf+GoVK4tlg6+GCApcJZm\n/JQm1UiSJGnSpM9Gj5a+SG+cbdGYAmntNH9pYYz5SSP/jnIp5vN7pdv/+6H07XefSa/O9ZQswz6X\nTrfWzTaS89MIyXHqJqmolevK9j0i+WIpBS6KlqrMbLvi0IOSs/cb0okW+rx8+XKjJApT/9na2kq5\nublmPrl1Lk+ZaJ7qyJek0bdvlgokSarN2Ci9/cZH0tKvXpPuHDddeutYqSRJklR+4Alp2P27pbJO\n7emFUVxcbCQLa9eu7ZDndKZMCKMoMKKjjWJp5C/SqsQyKeePeyQvkPo9tE3K09afr07+Urr7wZ+l\nUyZTODVS7qYnpNs/ipc6R/2dl9bf4CQNeNl8g2Q251ZI1/dZIO0xR1uWHpA++jbWzD5UScee8pKG\nvHtSakmNaLVayc/Pr1Wj+Mwzz5j1VHO5/GXCQIG0YmQP6dmYWkmStNKZr66Wwt/Q9ynnpxGS24wt\nkj4pMl16t29/6XJMlL1YRrEzZUIYRYERXl5essAvWrSoQ55Rc3KpNMURSTVssXSsrEw68uH/pL1F\n9eers+Ol9HJTd1dLOYln298gmaIqUfr+mcelL2L1T9Rk/Srd2DNAejW6I3pQIR357zDp5pU5kraV\nK4sPfiJ9fKjUvGaLtkhzB90mbTFjtceqVataNIg2NjZSTk6Oec9tA5eVTJiieKN0vc1caa+hI9pa\nqVYrSZJUJh14zEcKXZKg93QkjZT4nI005Ie8zurpBXOxjKIkdZ5MiDlFwcWl/Dj/m/cY25nCl6tf\nZLginrWrj1HcYKrC2sOfvvamGrDGfagX7VQZtFV0RZGs+nkbkbHH+WfDZzz98K/0fn89C4M7ogd2\nhC/8EN+vnmV1Vgu5ppXxbDjmy8zhjq03qStkxwtLsHn7Qya7tH753LlzCQgIMHn+gQcewN3d3eT5\nC+Iykwk0xaTHHGTf0RQK1ABqinPLoeo0UdU+eBlymZQqVEoN2Rtf5UPLD/jj//zqEj4scHZXkXLm\n8suTlS5WmbdOlAlhFAUmaf9EmxriP7mPRREaHAY7ceTNO5k4/Hrez+yBa/NJkZ2O0mM+27OOsDDA\nAkvf6byxbhtf3Tmo/bJOG+M8lreXzSZ1+W5yTdlFuwDufvJGvM3IHS859jP7Rn3DVzPdMWfTDYVC\nwaJFi5o9Z21tzfPPP29GK23hMpIJXRFHv3yImfOf59u/0ynNO8ayJa/x5QcPcsf3p8GyG24UUCwX\n/9GQs/MTlubM49t3RnJqcwIVdWfUpeVY2V7+ebIds06xk2XiArxLwRWMp6enHBpZvHhxZ3dH0Ano\ndDopODi4Sej00Ucf7eyudR6lx6UPbgmTJr68XcppOI9VvFu638tDumd3mSRpkqWXXAdJS3MlSZK0\nUsFfD0l9sJZce3SX3Lp1kwLfiK8LnxZLv41zku493LkzoBdCUVGRkUysW7eus7vU7ghPUSAQGNGc\nt2hlZcULL7zQST3qZLR5bHz0Jv7Hf/lp8WTcG3roTv5Mumo80/wdwGIA97/mxk/rzqBFieuErzkj\nVVOQl8/5khJiX/bXh08L/+L7/Md4KqTD4g2Cf4EwigKTiP0Uuy633HILYWFh8t8LFizA29u7E3vU\neVRGvMtTKx15/LVb8GwSg3Yk7JFnGOsKYEG/B37kzqNvsfy0iQLq2jw2L/6dCatfJegytImS2DpK\n0NXoCkIvMI/FixcDYGlpycKFCzu3M51GJbHLV5Le/w5mDW7OitnQf/xwehiMpfVgHlm6mGEWJgor\naHT0f/57nva/DC1iM1yJA2dR5k0gEDTL9OnTGT58OMHBwfTt27ezu9NJVJEZk0+34OH0biYvRlt+\njkJVT3rYNPAvrD3wN/V1Wbsz1KtDOipoJ4RR7ATUajVVVdXodG3ZB/3i0LBP1dXVFBUVd15nWsDS\nUoWdnR1K5eUb7NDpdFRWVlFbW9v6xZ3E88+/QP/+/S9ZOVAoFdja2GJt3VGZnCocujtgY2/dTFit\nkoQ1aymb+QQ9GuT+a/MOsXpNDM7T72Vq37b3Sy8XldTWdtYelqYpKS3F1dVN/lutrr0kZUOpVGJr\na4OVVdu/f4Uk4mUXBUmSOJORSURkNCknT12yYcoNv62lqqoKgMCgYAICgzu5R6axt7dneFgIoSFB\n2NnZdnZ3zKaiopLo2Dgio2IoKxO74rYHgwb2J3zEMPp4927nlnUU//0YIx6x5afjHzLGsf540ZFl\nrCyayINTvIy9i5oEFvuOpXRLNh+2IZmmqqqKyOhYjkdEU1FR0foNghZRKBQMGtifsNAQ+vX1NjvU\nK4ziRUCSJPbs28+hw8c6uytXJN26OXLn/Ftx6tats7vSKvn551mxeh0VFZffwu3LgevGjWXUyOHt\n26iuhGOfPMbCg32Zf/+NDLXKIT7qFDWDbmHBjQOwM7pWi7Z4Ezf4r+L5lFVcb0Z9BYDy8nJ+/nUV\nJSWl7dt3AQDhw8O4/rprzTKMwiheBHbv/Uc2iL169iB8RBienp5YWOgDMvnnz5OTkytfb2Vlha9P\n3za5/hUVFaSnZ6LV6Vd8K5VK+vbxxtHR/J0cams1pKWfobq6voR/r5496Nmzh9mjLEmSOHs2m6Li\nYvmYg4M9ffv0kT+vORQWFpGVlYOEXjwtLS3x8emLjbU1SFBVXU1CQhIRUTFotVpcXJy5+87bsLO9\ndD3G/PMF/Lp8NVXV1ahUKsJCgvAbOhg7OztQ6MPqp9POoFbXb7Lr4dGLHm3YjUKn05GZmUVJab1y\n7dbNkT7evdsUaj5/voDsnHPy3xcmk5Wkp2fIMqlQKOjbx5tu3cy0FOhlMj39DFUNZLJnj+706tUT\nhUKBTqsj+9w5Dh8+Rl7+eQBunDqZoEB/s59hLpridGKikil19mdEkDcODTNRdcXsX/II30ljCcp4\nj//EL+H84dtwM9laPWq1mp9/XUV+/nmUSiXDw0Lw9x+KrY0NKPTTGGlpGdRq9GF2BdDbywsXV2ez\n+67T6khLz6Cist4DdXFxpreXZ5ve7dzcPPl7BrC1scGnX19UlubPxJWWlpGRkYmuzvQolUp8+/XF\nzt6ulTvrUdeoSUs/Q02Dd8XTw53u3fXfuFarIzs7m2PHoziXmwdAaEgQN0ye0GrbV4ZR1FZQUm2D\nk705NTsuLsXFJXy59HsA/IYO5qYbb0CpVKLVaolPSCIyOpacBsrHgKWlJQH+QxkWFkLPHqaVYlp6\nBpFRpkOy/fv7MCw0mP6+PiaFv6CwiIiIKOISEqmpaboze88ePQgLDSIwwA9Ly+ZLSlRVVxMdHUdU\ndCzFJSVNztvb2xEcFMCw0GAcHZtXijqdjsTkE0RFxZJ5NqvJeQsLC/yGDmZ4WAgeHvpSY+lnMli1\n5jd0Oh3jrr2aq0aFN9v2pcD63zdxIuUkNjY23H7rLNzdewGQl3+eiMho4hOSmp1f9HDvRVhoMAH+\nQ7GwaF7GKyoriYqOJTomjtLSsibnHR0dCA0OIjQkCHsTyker1ZKQmExkVIyRQTRgaanC328ow4eF\n0LNHD5OfM/1MBhGRMaScTG1eJn19CAsNZkB/0zJZWFhERGQ0sfGJ1NQ0zeTs2aM7oaHBBNXJpEaj\nYdWa38jIPIuDgz2PP/LARZ1vLt99Fz7PjSH62D2UvT6QGc77SHq6r1np/VHRsWzbsROAubNvZkB/\nXwCys3M4HhlNUnIKWm3T8kZ9vHsTGhLE0CGDTH7W0tIyIqJiiI2Np6KyaXTCxdmZ0JAgQoIDsLFp\nvihabW0tsfGJREXFGBlEA9bW1gQF+DEsLARX1+ZrCUqSROqpNCKjYjh1Oq3JeX2ocwDDwoLp17dP\ns20A5OXnczwimoTEpGbnXD09PQgLCcLfbwgWFhbodDq27dhJTGw8APcuuAP3Xj1Ntg8dYhR1VObn\nonbywLnDqxhpyNz0Pj+fcqV76X5WHx/Ke8tfxJySkG3l77//xtXVlZCQkDbd99euPRw7HomDgz2P\nPnQfKpWKqupq1v+2kYzMs/J1Dg722FhbI0kSRcUlcsKLUqnkxqmTCfAfatSuJEns3vsPh48cl49Z\nWVnRrc4zLC+voLqBMgkLDWLShOuavDwpJ1PZsHErGo1ewBQKBa4uzigUCmrUaqM5L08Pd+bMurmJ\nUi0sLGL12t+NvENnJydUKgu0Op3RRLytrQ1zZ92Ml5enURtqdS1/bNrCydTT8jE7O1vsbG2RgJKS\nUrmPAJMmXsfwsBAAdvz1NxGR0Tg7O/HIg/d2aJp4dXU1v/zyCwsWLDA5QGiOouJivlr6AwDTp00h\nMMAPgLiERLZs/dPo93ZxdkKhUFBdU0N5ef3Ivm8fb2bdMr2J8srLz2f12t+NfitnZydUFhZotbpG\nXrsDt865mV49jRVDdU0Nv/2+kfQzmfIxe3t7bG2skdAP7gyKWalUMu2GiQQGGHtjkiSxd98BDh4+\nKh9rSSZDggOZMun6JjJ5MvU0v/+x2UgmXVycUTYjk+7uvZg762YcHOwpKirmq2/03/GcWTMYOKA/\nFwc1CS96MtH+MJkL7fg2LJzUz3dw98CBBPZqXQn+9MtKsrNz8PcbwozpUwE4eiyCnX/vla9RqVQ4\nOXVDAVRWVVFZWSWfGzRwADOm39BEHjPPZrF2/R9GkR9XF2eUSiW1Go1RqNbVxZlb587ExdnZqI3y\n8grWrN/AuXP1kSxHRwesrazQSRJFRcXywEelUnHLjBsZOMDXqA2dTsf2P3cRHRMnH7OxtsbBQV+4\ntLSs3Cg6ctXocK69ZkyT9zg2LoEt2/6Un2dhYYGzsxMKoKq6xmgetl/fPsy8ZTo21tbodDq+/f5n\nCgqLjL5jU1yAUdRRmZtDbXcvnBoNWqsTPmPejf/hj7zrWJ66lds9lEANKas/JSL0SW4b1M5rc3QZ\nfH3tHM5/vY+X/Yv4OTyA3xefYsOkCrYt3YnX3XcR1E77wP7vf/9j4cKF3HzzzSxevJigoCCz7vvu\nh2Xk5Z/nqtHhjBt7NbW1tfy6cq3sHQ4ZPIhhYcH08e4tC0FVVRWx8YkcOx4pj/pvmn4DAX71htFg\nbAHce/UkfMQwhgweiEqlD2NotVpSTp7i6PEIsrJyAAgK9GfaDZPk55xMPcXa9X8AegUYPjyUoMAA\n2ehJkkRWdg6RkTHEJyYB0L27G3fdMU8fxkRvrH76ZQUVFZUolUpCgwMJDQ028m6LS0qIjo7jeGQ0\narUalUrFnfNvxaPOU9Jqtaxe+zvpZzIAvXc7IiwUH5++cl9ramqIT0zm6LEI2chOnDCeEcNCycrO\n4edfVgLw+CMPtCk811bKysro1q0bPj4+LFy4kLvvvtss4xgTG8+WbX9iY23NU088jIWFBfGJSWzc\ntA3QhzhHDA8jKMAP27oQsCRJZGSeJSIyhuQTKQB4eLhzx21z5GcWFBTy8y8rqa6pkUOyoSFBuLnV\nb6taWFhEVHQsEVExaDQarK2tueuOefSoCzXV1tayfNU6srP1cjJ40ECGDwsxlsnqauLiEzl6LEKW\nyenTJhsZxl2793LkaARgmCYYxtAhg4xk8mTqKY4ej+Ts2WwAAvz9mD5tsvyc1FOnWbv+DyRJwt7O\njhEjwggK9MfB3l7+TrJzzhERGUN8QiIAbq4u3HXnbdja2MgGZmT4MK4ff20bftl/R9k/T3HTd31Z\nMPI8a979jR6z5zH3v68ytVfLvqJOp+N/730MwOyZMxg0sD9HjkWwq84gurq6ED48DH+/oXKGrU6n\nIy3tDMciozh9Oh0AH5++zJ11sxxJyMrOYfnKtfLvPTwshJDgQJyc6ufd8/LziYzSRxd0Oh0ODvYs\nuPN2+f2pqq5m2S8rKSgsAvS/1bCwYDw93OXfq7yigti4BI4di6SishKFQsHcWTfTv78PoP+9Nm3Z\nTnyCXn/07u1J+PAwBg7oL/e1traW5BMnOXosgty8fABGhg/n+vFj5b7GxSewacsOQP+uhI8YRmCA\nnz7EjHEi44mUVEDvNc6fNxtLS0uORUTx187d2Nna8vSTj7T4m7TZKOrOrWTykBfw25bIJ6PtG55g\n0wvvcmrkDK4fPRx/T3uUQPHfTzBn8xzWfjgWZwB1Jjt/2kS6lSWVRbaMWXA7w1wuPMyh02hApUJZ\nfpDHgx7Hc/NhFvpZURP/PvPe78uXP8zBox2iKG+//ba8gFmhUDBz5kwWL17c4o4CAF989R0lpaXc\nMHkCoSFBbN66g9i4BKD1uY+qqirWrN9AVlYOSqWS+xbcQY8e3UlITOaPTVsBvaGbOmWiyfCJJEns\n3LWHYxFRAEyZdD1hocEUl5TwzXc/o9Fo6NmzB/PmzJRHbs2RfCKF3//YgiRJDB40gFm33IQkSXz/\n06/k5eVjZWXFvDkz6d3b02QbBQWFrFi9jrKycuzt7XjkwfuwsrJk1+59HDmq93gnXHct4SOGmWxD\nrVazfsMm0tLOAHDX/Hk4OTny2ZffAvDg/Qvo3sAgtDcGo2jAx8eHl156ibvvvltW/s1hGPl37+7G\ng/fdTf75Ar7/8Rd0Oh1eXh7MnXWzbAybIzYugc1b9UohKNCfG6dORqfTsfTbnygqLsbO1pZ5t85q\nMTSUm5fHytXrqayswsmpGw8/cA8WFhZs3f6XPIqfOmUiIcGBJtuoqq5m7foNnD2bjVKp5N4F8+nZ\noweJSSfYsHELoFee026YaDLUK0kSu3bv5egx/aBu0oTxDB8WSmlpGUu/+5HaWg09e3Rn3tyZODiY\nHtWmnEzltw2b0el0DBzgy5xZN7P+942cSEk1e/6oPdGWF1Jp5YqjRTnFNXY427WueKqqq/noky8B\nuOfu+fpB84o1gH5wOHPGjSYHXZIkceRoBH/v2QfA6FHhjL/2ampq1Hz1zff637lbN26bNwtXF9Nb\npGRknmXNut9Rq2txd+/FPXfdjkKhYO36DZxMPY1SqWTmzTcyaOAAk22Ul5ezas1v5OWfx9JSxUP3\n30O3bo4cj4jiz527ARg5YhjXjR9rMpKj1WrZsu0vebBzy4wbGTpkEHn5+Xz/469IkkTv3p7MnXWz\nyVAvQHRMHFu3/wXooxFTp0wk9dRp1qzbgFKp5IX/Pm3yXriAijZV6ZGc05xlw6pYGiYN6/KPcaTv\nPTw861oC6wwilcd47bE0Hn6pziBSybHFt/KZ9XTuWfAA949L4ckFP5D+L5bjKFUqlJpsNr76IZYf\n/MH/+elHU9YBD/O0xcs882fxhTfegIZjB0mSWL9+PUFBQcydO5eEhAST9ymU9QJQUVEpj5jGjR3T\najKAra0tc2fdjKOjAzqdjoioGACOHNOPxvv17dOiQQS9AZ9w/ThZoI8ei0CSJKKi49BoNNja2rRq\nEEHv0U6aMB6AEympFBeXcCYjk7y6kd0tM6a1aBAB3NxcmTdnJkqlkoqKShKTklGr1URF6z9X+Iiw\nFg0i6MNxs26+SfaEjkVEGr9kF3mKPC0tjfvvv59Bgwbx/fffG4V4G6Ko+42kujBpRGQ0Op0OR0cH\n5s6+pUWDCHpDeO3YMQDEJyRRUVHJydTTcmh09swZrc6V9OrZkzmzbgb0Hn7KyVNUVlbJg7SxV1/V\nokEEfWLF3Fk3062bo14mI/W/3dG6qEXfPt7cOHWSSYMIepm8fvy1DBk8SL5XkiSiYmKprdVgY2PD\nrXNaNoigDxtOnngdoA+5FhYVdWqFFQsHVxytAAsHswwigLaBvKgsLOToT48e3Vs0iKD/HkeNHM7w\nYaEAREbFUFtbS0JiEpWVVVhYWHDrnFtaNIign5ucMX0aAOfO5XI2K5vCwiJ5KmPyxOtaNIhgCMvP\nxMbGhtpaDdExcUiSJA98hg4Z1KJBBH049Mapk+RlNUfr9NzxiGgkScKpW7dWDSLoDeE1V48GIC4+\nkcrKqjbJRZuNYnmOjlueHUPxhlXENrCKRdHHcR3V32j/qpK9S1jh9xSTDSlYJQd47/sapk3wwgKw\nGzyF4OgP+CWlaXKH2Why2PnJUnLmfcs7I0+xOcHQKQdGPTKRPa/+QW6LDZhHcw61JEmsXbuWoKAg\nbr31VhITE5tcYzBYOp2O2Lh4dDod1tZWjBge1uTa5rC1tWXEMP218QmJZGSeleP7V48ZZVYygUKh\n4JqrRwFQWFTM6bQzxMTqPYOwkOBWDaKBkOBA7OvCWFEx+qQa0Icp+vv6mNVGjx7dGTpErwwjo2NJ\nTDqBWl2LUqnkqlEjzWrDysqS0SNHAHoDXdlMAsHFpjXjaFH3O2l1OtRqtTwaHjE8TA4BtcaIYWFY\nW1vJshRZN5jw8enb6oDEgFeD3yoqOobYuAR0Oh1WVpaEjzBPJm1sbAgfbpDJJDLPZsuh1zFXjTRb\nJq8eo5fJ4uISTp1OJzpGnwwRGhJkdtZ0cFCAfG1UdFwrV196aBok0NTUqEk5eQqA0SNHmD1nfdXo\ncH0OQE0NiUkniIzSv5d+QwfL2ZitMXCArzyoioyKISpG34ajowPBQS1Hwww4OjoQGqIfVEXFxHHq\ndJqcdHf1VaPMMkxKpZIxV+nlIis7h8yzWSTUTd2MGBHWqkE0ED58GFZWlmi1WmLjTTstzdHGijbl\nnEpyZNz8hzj46UJWxb7F6NH2QAmJx6wZ+VTDBIxK4lbtoPu0bzCIt+ZcBNFlzsyzrXtpVC54WWWw\nN6UC/FqYkK45w5ZP3+Xb3w9wqlSDBCi6jeGd39/GatEoJv5UTg/HL/g/tSf/PXicm+tusx4wiYD4\nTzhUfDc3OzdtdvHixaSmpqJUKrGwsJD/3/jfFhYWHD16tGkDdeh0OtasWcO6deuYO3cuixYtYsiQ\nIYCxUYyr8xID/E1ncTZHUKA/e/btR62u5eAhfT/cXF3w7m1+vahePXvi4eFOTs45Dh89Jk/Uh4S0\n7Bk0xMLCguAgfw4eOkpsXIJsjEJb8S4aExIcSEJiMufO5RJZZ1iHDB7YpgX4Q4cM4q9du+sUSX1y\nzo4dO8jI0IdWJUmS/2vp79b+3fBYc5mQDTEYxyVLlhiFVRvKQUrqKXkgEBRg/tIBKytLAvyGEhEV\nQ0xcAoV1cz2hwebNbxsIDQnk1Ok00s9kyok8/n5D27TcIjDAj91791NbW8vBQ4cBfSZj3z7mFw3v\n2aM7Xl4eZGXlcOTocTlRoi3ypFQqCQ4KYP+Bw8THJ+LtfXnVUNNq6o3iqdPpSJKEjbU1QwYPNLsN\nB3t7Bg8aQPKJk0RGxZCXr4/etF0ugti2YydJySnY2OhzBkKCAtuUxRsSHMihw8eoqKiQ55d7e3nS\no4UM+sb06+uNs7MTxcUlHDh0hNpaDRYWFnJymjlYW1vh7zeUqOhY4uOTGD/uarPvbZtRrDlDbGUo\nN/cfyb3hJbywPJolo8fgUJlGZE0ItxvlN5QSt6eW0AfqR3ya8gIqscTCMGBQWmKlqKSgtIUyVxUR\nvD5pLvuuXczzrw/j57n3kf3uMb6Y0BffXq5Yf30G6WsT99r5EKg9StR5HTc7N/1ht2/fzpEjR9r0\nFbSETqdj1apVrFmzhnnz5vHqq6+iVBiUYb1CNSSYmIudnS1OTt0oKiqW23B379XmUJGHey9ycs7J\nyy7sbG3bvODdo27X9ZqaGtlguLu3HLYz1QZATXXd5+nVtu/E0tKS7m5uZGXnGGWu/f7776xfv65N\nbXUEjY1j2DD9gnKtVid//05O3dpcicfDwx2iYoyMc1vlqeH11XV9aWsbtra2ODs7UVBQSLXhN3Tv\neQEy6U5WVo6clWpjbY2zs1Ob2wCMMlsBDhw6Ujev5iiH5ROTT8iJZ9eNuwYLCwvO5eYRF6/33EOC\nA+nR3Y0atZp9/xwE9CHhQQP717V5lMrKSro5OjIyXN9mUnIKZ7P0iUPjr70alUpFbm4esS222ZtB\nAweg0dZHFDR16xB79Oje4vx0899BL5JPnDSSi7a+l4YlQjpdvYy6t1EuXJydsbG2prqmxkhXtQWF\nQoGHey+Ki0tk/eDs7GR2RMWAh3svoqDVgWxj2vTNa3LiyAu7hh6qnkx+bAJPPfQl/7wxhomF0WQP\nGY9ReoOuivwCBT2d6x+hsLRGha7BtI8WrWSBpaWpF6mG+Hdv44uAZZx+awz2VOIw6ylm57kwqF+P\n1ncSt3LFy66c5MJaaGav9ObW/rQHOp2OTZs24evrSx+fgfKxf0N7zJQ0aeMCGm1e57WtoWbbuJAP\neBkU6E9LS2PdunX06KlX3Dpdvcy1W/fb3FD7PLljZLLtrZq6JTomjvMFhXh5eshGMS3tDMcjowH9\nvL6FhQXnCwrkZST9+vahR3c3atW1RktLDEYxJjaO/PMFeHl6yEYxLf2MnMg29pqrUAHnCwrl+/v2\n8aZHdzc0tfVtSpLEoIEDjDxFRRs8sma+haZH2vhdNvtaXtB7aXxTh8hJB9IGo6gl63ASA0fNxgIl\nbtc9yUzVDD7Zkkmw4wm8g+YZGymlCltLDWer6+fiLF1646T5k3LD4EhTSaHakT7dTYRtalL45bNs\nZmwNxR5AV0T84XJ8J7u0bhABdGoqNJbY2zT/lQ4bNgx7e3u0Wi06nQ6tViv/1/jv/Px8ioqKWn2k\no6MjTzzxBM8++yxubm78uGyFviuScaHttiBJUpNRsGF03hYat6FW16LValtMimjSRjPPbevnae76\ntrYBUF3V9B5HR0fc3NxkhaBQKOT/Wvq7tX8b/i9JEmlpTRcfN8eUKVNYvHgxI0eOJPlECvGJJ4wL\nrtd5221RXqa+u7Z4/M230TZ5kiSJqup/L5ON26itbbtMVpmQnccevr/JsWk3TGLaDZOMjgX4DTVa\n7gT6dcOLXvq/Jvc/+tB9TY5NnTKRqVMmGh3z9xuCv98Qo2P29k3bbDinqKyTg8bvqTk09x1UV9eY\nLNTQfBtNn2vquzWFRqNpUoCiuXZbo7EsVVVfyLvS9udCWxJt1GnsiR/EqN51dtRxJE8+6sO+V19n\n5UE7Qvs1dm0d8PW34kxO/Zeq7DmGmweVkpRT19mK08Rpw5kZ5AjVafz1+wHOGeUn6NBY9sLfS//D\n1iR9zyfq//DhVDNT7qtzSK/tg5+JBbRff/01e/bs4Z9//uHAgQMcPnyYY8eOERkZSXR0NHFxcSQm\nJnLixAmefvrpFh/VrVs3Xn75ZdLT01myZAlubvoJbmVd9qlOp5PnABMSk83rfx3pZzLkOUBf336A\nfnTalvqZDSfxfX30bWg0GvmYuRgmvb29vXB1cQYgMelEG9vQf34rKyv69vWW22jL6qBz53Ll9VOe\nnvXh2Pfee4/z58+Tn59Pfn4+eXl55Obmcu7cOc6dO0dOTg7Z2dlkZWWRlZXF2bNnyczMJDMzk4yM\nDM6cOcOZM2dIT08nLS2NtLQ0Tp8+zenTpzl16hQxMTGt9m3KlCkcPnyYbdu2MXKkPnlIqdQrea1W\nR++6wgWVlVXy2kxzMXx3fbx7ywlSbZUnw/X29nZypl9CYlKbvv+MzLPyHKBvXeJO+pkMo2IDraFW\n15JyUr+mrL+Pfq8lrVYrrzMzl/rv5PKaTwTQNPAUvet+i/z88/K8oDnodDoSk/TfgU+/vnK+gmFt\nsbkY3u0e3d0uWFedSEmVI3AGPZNyMhW12vydYMrLy+vXLNfJVkVFhVGxk9aQJMlIV7WF1o1i2V4e\nCwtl4uxnOD56In1l39Ia/0c/5O7a73jrZCBDmqyXdmL4TT05fjgH2c5Z+/PkJzcR8/H3HEiN5/cP\nf8HpxY+5pZcS9amVPD3zRh5vuITCeiiPLhnJjne+ZM2yt3ji9SJe3rqEEeYOfoqOscfhFq5qOSP5\nX+Hk5MQrr7xCeno6b7zxBq6uxgbbMKco6SRCQ/QT3znncpst7WaKiLpwj5eXByNHDJczEA0ZpOYQ\nn5BIbW0tFhYWjBwRJgtbZFSM2cqwoKCQtHS9sA4LDSE0NLiu7SSzPT2dTidnxwUG+DF8WAigz0A8\nnZZu9ucxLE9xdXXBy9PD7Ps6ksmTJ3Po0CEjY2jAokGiTa+ePfDy0vfZsKTBHLJzzpFTl3kcFhos\nL5+IjU0we/spjUYjl7wKDgpgWJj+N8zNy5czSM3BIJOeHu6MCh8mV2NqWLWkNRISk1Cr1SiVSkaM\nGCaXN4uIjDZbJguLiuQF7KEhl+6OLqbQ1s0pqlQqfH36yvOpbZGLU6fT5IIKw4aFytWvIqNizJ62\nqayqkge3oSFBhIXqddXp0+lmbw0lSRIRUdGAPpt1xIgwlEolarVaNlDmEFW3nMPGxoaR4cPkko5t\n+U6ysnPkQgBhbZSL1o2ilTfjbxxGv1FP8tLknsY3OI/l3V1H2Pfd9GYK3yrxvvW/eK1aSf2KCyVu\n17/Lhg+uh1P5+D62ihWPDcUasPJfSGzGT4yxa/gyWNH//hVsevNGhk9+lq9Xf8S8/uZWxdGR8ccv\nWD12D4Pbodxc45fUycmJRYsWkZ6ezuuvv46LibVADbMO+3j3xq2uNuDGzduorKpq9p6GRMfEyd5c\nWEiwPgPRX5+F9c+Bw2TVTfC3RG5ePrv37gdg6JDB2Nraygb6TEamvO6xJWpq1GyoKxjg6OjAgP4+\nBAX4YWFhQW1tLRs3b2t1jlaSJP7auVtO0w4NCaJnjx6y57R1+1+UlTWt29mY5BMnZcUeFhLU6bt/\nG4zh9u3bGTVqVLPXKC3q5UCSJPlFTTmZapYhqayqYuNmffUbN1cX+nj3JiQoUD63ZdufrSpASZLY\nuu0vuQZmaHAQvb085bT9jVu2G5UPM0VsXALJJ04CeuNsaWlJYN2a2wOHjjRbt7Yx+fnn5UXnQ4cM\nwt7OTlbEmWezjMoXmkKtVrNho14m7e3tm5QXuxwweIoWFhYoFArC6t7LqOhYs6I4paVlbNuxC9BH\nD7q7ucrvdlFRMbv+3tvqAEOr1bJx0zY0Gg0qlYqAAD8GDugvh143bNxilMxmisNHjsmVisJCgrG3\ns5OzaP/es4/8ZuqmNibzbJacYW+oa2v4TpJPpMhraluisrKKTVu2A3qv1zAANZfWjaK1L7Nf/45v\nF07EvckMpBLHQeH4uzYf/1f2vZfPJ2/hw73FRsdVroMZM3k8wZ42DTpQTfKeDEL8m2aeWTj3wbeX\nTdsWVVZG8tUvQ/noaT/aswSrs7MzixcvJj09ncWLF+PcqFZgYwzV49VqNQqFgimTJ6BUKikoLOKX\nX1c1W3hZf30t+w8clisz+PTrg9/QwYB+LZiTUze0Wi0rVq8nPjGpWYUoSRInUk6yfMUa1Go19vb2\nXDv2KkA/kjMsyP179z7+3vOPySyt3Lx8fl2xmty6avNTJk2o28TTluuv05fSSj2Vxpp1GygubloM\nHPTloDZt2SF7eOHDw+RScJMmjMfSUkVZmX77nDMZmc2+yBqNhqPHI/n9j82APhMzNCTIqDBwW+ai\n/i3mGEMDlqr6JTi1tbX4DR2MTz994eOt2/9i/4HDJkNM2Tnn+OXXVRQWFqFUKpkyeQIKhYJu3Rzl\nBf2JSSf4bcOmZouBg74Sz28bNskhtWuuHq2vpalQyPVHi4qKWfbrSrJMeIxqdS0HDh2RK+v07eMt\nz5tdNTocZ2cntFotK1evJy4+0aRMppxM5ZcVa6ipUWNvZyd/hv6+PvKC/t17/2HX7r0m59fy8vP5\ndcUaec3uDZOv1w/Q6taHWvyrpJWLh2FAp9Vq9YOl0BB5veD63zdyLCKq2YIQkiSRfiaDn39dSXl5\nOZaWlkycMA7Ql3001AU+FhHFlm1/NlsMHPQ1eVev/U2O0ky8fhw21tZYWFgwZZK+IlDOuVx+XbGm\n2WLgoJ8D3bV7rzzwHjJ4kDzNM+7aq7G3s6OmRs0vK9aYLBKv0+mIi09k5er1aLVanJ2dGD1aX9w/\nwH+ovNRn89YdHDh0xOS7kpWVzbJfV1JUVIxSqWTypOtRKBTy9eboh47fJaP8KG8sWE7Ylx8xrWcL\ngqrO5HiKPaEBruYl0bRIGQdfe4St137Gm+PaJ3b6ySefUFJSwlNPPYWTk/kp4zv+3EVEVAyuLs48\n9MA9KBQKkpJT+GPTVllpuLv3wt9vCLY2NkiSRM65c8QnJMujMy9PD+bNnYm1db2X3LBkGui9t8AA\nP7l6RUlJKbHxCXLRXzs7W26bO4teDaqe1NbWsnb9BrkItKWlCr+hQ+TtZGpqaozSzUGfqNB4Me++\n/QfZf+Cw/Levbz8GDeiPSqVCq9WSfiaDEymp8udtXO8S4HRaOut+2ygrgB49uhPoPxQ7OzskSSIv\n/zxx8YlymLZHj+7cdussHOzt5V0GlEolzz716AXttm0u5eXlzJo1i8WLFzN69Giz7ystLePzr/Sl\n6Azlq6prali95jfZCFlZWRHgPwQPd31tyarqank9J+ijDjOmT5WLH0BdGb+/98qVUAAGDuhPf99+\n8vd/6nQaJ1NPy8poWFgIkyaMN/r+k0+cZMPGLfUy2asn/n5DsbU1yGQu8QlJskx6eLhz29yZRoup\nCwuLWLF6nWyYHRwcCArwk3dOKCktJS4uUY4U2NracNvcWUYp+xqNhrXr/yAtXb/WVKVS4Td0ML29\nPFEqldSo1SQln5A9EkAuoajRaPj086VU19Rw3fixjApv530VO4Bz53L54eflANw5/1a8e3tRXl7B\nitXrOH++ANB/T4EBfvTo3h2FQkFFRQVxCUnyeUtLFbNnzsCnX1+5XZ1Ox6Yt2+U5QaVSyZDBA+nX\nt49cEDzlZKpcLhH0mbNXX2U8uGu4gwfoa5cOHTJYXxBcp+NsVjaJSSfk99bHpy9zZs4wWlKScy6X\nVWvWU1WXGOfs5ERQoL9cY7WwsIjY+AR5PrpbN0duv3W20Y4b1dXVrFzzmzztZG1tRYDfUHlpWlVV\nNQmJSfI2UUqlkptvmiZ7qn9s2kpCYjI9e3Tn/nvvavE3uThbR9XkkVbuio9bG2sFXCiaYjIKrOjT\ny/zMq44iNzeP73/6FTCuK3kmI5Ptf+6ioKDQ5L1KpZLAAL86T6rpYv+ysjI2b/vTSLCbo493b6ZN\nndSkAj7oR6i7du8jKjq2xfCns7MTkyaMl+d9GhMTG8+efftbTP6xtrZiVPgIuQJHY3JyzrFl218t\nJhkoFAqGDhnElMkT5PVQP/28nMKiYrMq4Hcmq9f+zqnTafTo0Z275s/D2tqK2tpa/ty526RnZcDN\n1YUpkyc0uzhekiSOR0az/8AhWfE0h62NDWPGjGLEsNBmv/+MzLNs37GT863IZID/UCZPvK5ZmSwv\nL2fz1j9bnR/27u3FtKmTmi1BptVq+XvPP0RFx5osmwd65Tpxwjh5N4zDR47z9559KBQKnnj0gVbL\nxF0KSJLEDz8vJzc3jz7evZk3dyYqlYrq6mq27dhJ8omTLYY/e/XswbQbJjW7FlCSJPYfPMyRoxEt\nhj/t7e25btzVTXY9MXAy9RR/7tzd4gbIhmL04+vWfjamsKiILVv/bDW07uvTjxunTmr2t6utrWXH\nX38Tn9B8ZMxAdzdXpkyeICeRpZ/JYMUq/drl1mr7wpWyn+IlzopV6+RsqmuvGcOwsGBs6rzCzLNZ\nREbFGAmLjbU1gQF+BAb6Y2/XumEvLCoiKjqOEydO1m8yrFAyaGB/QkKC5N0QWsKwC0JsXAJVDeY6\nPT3cCQ0Jxqdfn1bn7gy7IERFxxop1m6OjoQEB+I3dHCrlXwMuyBERsUYZWZaWVri7zfUqKxXTs45\ntv+5S04+ueuOefL85KVIWnoGK1frX04P915MmjAeT08P/ei/spK4uAS9N9wgZOjd24uw0GC8e3u1\n+v1rNBpOpKQSFR3bZBuv0NAghgwa2OqicINMRkXHGmX7WVtbE+jvR1Cgv1lp/oVFRUTHxJGcfNJo\nk+FBA/oTGhJkVoWT6upq4hKSiImNN5JJD3d3wkKD8OnXV45oHI+IZu8/BwD9HOUtM25stf1LhYYF\n/vv19eb6666Vt/YqKysjOjaehMRko2Qqn359CQsNxsOMIh5qdS2JSclEx8Ybzdl3d3MjLDSIAf19\nWw0r6pcindHv/3qufsrH1taW4KAAAv2HmlWCLT//vH6+NLV+/1cLpQVDhgwkNDgIl7qM9paoqKgk\nNi6hbv/X+nelj3dvwkKD5UhXdU2NfrC+dz9arZZu3Rx5+IF7Wn0HhFG8CFRVVbFyzW9yGMzCwgI3\nVxeUF3H+64pCkqiqrjYauTYX1r0UaVjBH/ShIjtb2wtcJS3QaXUUFhXJHqV+qmGWvM3S5cKBQ0fY\nu++A/Lezs5N+ezYhFxeETquloLBIjn65uDhz262zcDZj6ksYxYtEdU0Nf+/eR3xCUoshIUHb6O7m\nyjVXX2U0z3apcyLlJP/sP2QycUHQdlQqFf5+Q7j+umvlvT4vN2Ji4zl4+KjZSyAEraNSqfQ7dIwb\na3YhA2EULzLVNTWcTD1FRXmlUZWbS4mKuglvezN3z+gMLC0t6dmju9FGuJcTkiSRlZXDudxc1Gau\nMbzYHDl8mKqqKiQJ+vbtg2//i7WTvfkoFQrs7OwYOMC31e23LgcMm+Xm55+XM2kvRfJy8+jZylZl\nnYlSocTewY6B/X3N3lnDgDCKgib85z//wcrKirfffruzuyLoRHx9feWSdi+88IKQBwEAmzZt4q23\n3uLQoUOd3ZUO4fJYzCO4aOTk5PDVV1/xxRdfmFXrVXDl0nDLIDF2FoBeDl599VUOHz7Mrl27Ors7\nHYIwigIjlixZQlVVFWVlZXz66aed3R1BJ9IwLP1vd3kRXBmsX7+e6OhoAN58883O7UwHIYyiQCYj\nI4Nvv/1W/vvTTz81q+ya4MqkoVEUnqJAp9OxaNEi+e89e/Zw4MCBFu64PBFGUSDz+uuvGy3yLSws\n5Msvv+zEHgk6ExE+FTRkxYoVJCYmGh27Er1FYRQFAKSmpvLzzz83Of7hhx8aLZwWdB1E+FRgQKPR\n8NprrzU5vn37diIiWt9Q4HJCGEUBAIsXL252/WReXh7ffPNNJ/RI0NmI8KnAwE8//URqavP7XF5p\n3qIwigISExNZuXKlyfPvvfeeWVvHCK4shFEUgH6HnzfeeMPk+T/++IP4+PiL2KOORRjFTkNDwals\nTJfP1lGSkU35RYhavfrqqy2Gx7Kysvjpp586viOCSwoxp9jJaAo4lW1aQ1wsHfHNN9+QkZFh8rwk\nSSxZsqRjO3EREUaxU1CT+vPTLNpbJm+Tpc5PYO+WzeyOzkG/z4ESspfzf6/tIK8DhT46Oprffvut\n1ev+97//ifJ0XQwxp9iJqFP5+elF7C2TNQT5CXvZsnk30TmGnVA6XkdUVVXx1ltvtXrdmjVrSElJ\n6ZhOXGSEUewE1PH/Y/7akSxcMBhrdBQf+Yp3ViRj4e5E5g+zGXX7MtLU4DTqKR61WMx9K3PoKJX0\nyiuvmOUFpKWlsWLFig7qheBSRIRPOws18f+bz9qRC1kw2Bp0xRz56h1WJFvg7pTJD7NHcfuyNNR0\nvI748ssvyclpftPphuh0uiun4pEk6HA01dWSRv6rSFo3rps0d1e5/Pe2+f7S1KWpUo0kSVLRFml2\nrwHSc0cr9Kdzl0kjPR6WDlU017JWKjr+g/TZzpwG7Teg+Lj03Rc7pZxmT0rS4cOHJcDs/4YMGSJp\ntdoL/RoElxkhISHyb//oo492dne6DkXrpHHd5kr1KmKbNN9/qrQ0tUb/55bZUq8Bz0kGFdGyjrhw\nysrKpB49epitH1QqlZSent6+negEhKfYkajTWP7YfF74YCGzZywhqhIo2c/SI+HMH2Yott2Nq1/7\nniUzvLECdJUFFOqc8XKp2/Or5wTutF/D+tSmiS7ViZ9yx8J8rh3jTrObUDmFMMl1GXe9cpCSZk6/\n/PLLbfo4ycnJrFu3rk33CC5fRPj0YqGlpqZ+g++S/Us5Ej6fehVxNa99v4QZ3laAjsqCQnTOXhhU\nREs6AnQUR/zI57vO0dwW4iUR3/OliXOffPIJ+S1s+N0YjUbD//73P7Ovv1QRRrEDKdz5Is8fCuFm\n72SicipRS0BxDAeqg+gvb0ChxKH/SEJ6WYE2nz/f/grrZ7/gHl/DfnCODPArZ09Co8oy6hN88dBq\nrv30aQJNFoG3wHvuOyyIfpjFh5tO2O/YsQO1Wk1VVRXl5eWUlJQwaFD9FkwPPPAAZ86c4fTp05w8\neZLk5GSuvvrqf/elCC4bRPi041GnLeex+S/wwcLZzFgSRSVQHHOA6qD+1KsIB/qPDEGvIv7k7a+s\nefaLe5BVhCkdQesDZ6eQSbguu4tXDjYdNi9cuJDa2lqqq6upqKigtLSUe++9Vz4fEBBARkYGaWlp\npKamkpyczLPPPvsvv5HOp+UtiAX/gnIS/jiAxagnCLnzv6TfqT+qySylSmmNqvFwRFfIgQ9fZGPg\n16y5PwiHBuctlWpKKo3HcsV/v8QH1k8RNbCVzVSV7sx4MZz/Pr2C5w7ej1eDdpVKJUqlEktLS/lY\nQ0Xo5OREnz592vKhBVcQwih2NIXsfPF5Dg1bzTN75rMhRI2EhqrSKpTWqiYei67wAB++uJHAr9dw\nf5CD0fnmdIQ8cP5ur+mBs4U3c99ZwLJJizl8+CNGNdhyUKFQoFKpjHaqt7KyMvq3t7f3BX3ySxnh\nKXYUmlyOHSqm75gBNNzlTeXSj97aTM5XNzioK+TwN59ybNibfPpgEDW73mRpvOGCWvKzVPh6NNw4\ntZC/P/iLQQ9PopcZv6B9yG2MiX+PdZmth8Aahskux30KBe2HWJLRwZQn8McBC0ZdFcKdW9I5umQk\n9qhw6dcbbeZ5jFXEYb759BjD3vyUB4Nq2PXmUmQV0ayOqB8439XKwFnpPoMXw9fw9IqsVpN1GsrB\nlaofhFHsAMoivuHlp5/j0zgtlTve5sV3t5BpWM3gNo47nHdxMN8gftXEvD2N6x55jWeu98BSoaD7\nTf/g5lE3tKtJZW96MLeGOdU/oCKO3w45c0OoE2bhGMhNQ06y6nBxq5c2FPqGSlHQ9RBzih2LJvcY\nh4r7MmaA8ebIbuPuwHnXQepVRAxvT7uOR157hus9LFEounPTP24YVESzOqJNA2d7Qm4bQ/x762ht\n3CyMouCCcBz2IK88FIy18zhe/vhj3vm/aXgbIhCqgdy3yJNlv6agt5M2BL90iEpJQjL8V7WD2W76\nyyujfmJn+Evc7N7gASVJHKkYwGDXZtNrmsGBAWG2JEdkUdPKlQ2VnzCKXRsRPu0oyoj45mWefu5T\n4rSV7Hj7Rd7dkolh3KwaeB+LPJfxa0rdEZtgXjpUWa8fJImqHbOpUxHN64g2DpwdA29iyMlVtDZu\nFkZRcMGUJ+8n1/MaBjs0PqOi/4NLmbVnEcvPtLIYviaRpUvO8fxn03FpcFhdlEGRVU+cLRtdXx3H\nezcEMuHNqEaVclQ4edhRejqP1oq1ifCpwIAIn3YUjgx78BUeCrbGedzLfPzxO/zfNO/6BA9Vfx5c\nOos9i5bTmoowpSPaPHB2GECYbTIRWS0Pm4VRFFwglZw6dBrrwOF4NhfOtxvG4rXPULnxMMUm29CQ\ns28Xzm/8wH0+xvlQkk6HpFChbCyT2lLOJMWTlFbSJMVakrTodK0rNhE+FRgQ4dOOpJzk/bl4XjOY\nJuNmwG7YYtY+U8nGFl030zrC1MC5Ou49bgicwJtRjbLRVU542JVyOq/lYXNXMIoi+7Qj0BWTcCiP\n3ncOalbgAXAexSNPtNSICo+JT3BPM2csnTxxrImitLbRCfsxfJ4u8XmTOzSU5lTj4OVCY+eySddF\n+FRQhwifdiCVpzh02prA5z0xlQbjPOoRWlQRLegIUwNnbekZkuKT8CxpMmxG0upobdzcFYyi0Hod\nQeVJDmS6EB7UvflF9f8SpdNgQqxOklpi7ui9moyEKnyHe2NySWMdInwqMCCMYsehK07gUF5vxg4y\nOWz+V+gHznlNBs72Yz4nXcri+3GOxic0peRUO+Dl0vKwWRhFwQWhzY8homIw4wfbt37xheAUzC3B\n5/krrtS86ytT2JXkzZzRbq1eKsKnAgNiTrHjqDx5gEyXcIK6d8Sw+QIGztUZJFT5Mty75WGzMIqC\nC6IsaR/5g28h3LWjnuDOlGfCiV8fQ7kZV1cnrmKL+xPM69/6CyjCpwIDYk6xo9CSHxNBxeDxdNS4\nua0D58qUXSR5z6G1cbMwigLz0eWz47nJzHjnCPF70wl4cDreHTMIBKDXjLe4Ne5jNp9rTVkV8/d7\n2xj7wf0MMKM/InwqMCDCpx1FGUn78hl8SzgdNm5u08C5msRVW3B/Yh6tjZuFURSYj66MtLh87KTd\nLMtbwAe39+mQ+UQZu3AWfujLV8+uJqu5ar76TlG44wWW2LzNh5NdTF1khAifCgx0BQV4MdHl7+C5\nyTN450g8e9MDeHC6d4fqCLMHzsV/8962sXxw/4BW+9MVZEJovfZC5cvDW3byxpSbWbL0cfxby2hp\nB5zHvs2y2aks353bbJV7So7x875RfPPVTNzNfPtE+FRgoCsowIuJriyNuHw7pN3LyFvwAbf36dBh\ns3kDZ10hO15Ygs3bH2LOuLkryIRYktGeqFwZENJxAZGmWNF35iv8n6nTTiN5ZsnINrUowqcCA11B\nAV5MVL4Ps2XnBOLznHh2SA+TSzHaE+exb7Ps/Dss353Lfyb0auIJlhz7mX2jvuGrmSa2n2tEV5AJ\nYRQFRojwqcBAV1CAFxuV6wAu6ri5lYGz08hnaMu4uSvIhNB6AiNE+FRgoCsoQEHb6AoyIbSewAgR\nPhUY6AoKUNA2uoJMCKMoMEKETwUGuoICFLSNriATQusJjBDhU4GBrqAABW2jK8iE0HoCI0T4VGCg\nKyhAQdvoCjIhjKLACBE+FRjoCgpQ0Da6gkwIrScwQniKAgNdQQEKLpwrVSaEURQYITxFgQFhFAWN\n6Qo1cIXWExghPEWBAWEUBY3pCjIhjKLAJFeq0AvMoysoQMGFc6V6jcIoCkxypQp9R6IpOEV2penz\nupIMsssvj70JhVEUNKYrbCcmjKLACKH8Lhx16s88vWgvZQ0qK1dnHONIllr+W0k2y//vNXbkXfqG\nURhFQWOEURR0aa5Uoe8Q1PH8b/5aRi5cwGBr0ObtY+lbL/PAdaO5f0sBsgl0GsVTj1qw+L6V5Fzi\ndlEYRUFjGibfXan6QRhFgRFdYSTYERRveoIP+j3LTE/9K2XRcywPLXyee0a6NdmKxirgYR7N/y+v\nH20hznoJIIyioDEN5aBhUt6VhDCKAiOEUbwQSti/9Ajh84dhb9b1PZlwpz1r1qeibuasrjiCHz/f\nxTlTG8O2pWcR3/PlrnPNb0LdCsIoChojPEVBl6PTlZ8mk9/eeJ8DxaYuqCb268X8klx9ETvVGsXE\nHKgmqL95JhHAcYAf5XsSKGt8ojqRT+9YSP61Y3Bvh43ZnUIm4brsLl45WNLmey8Zo3hZysSVySUz\naO5AmRBGUWCSiy/0NUQtvpX1YfcyxllH8fHvWPTcg0zx6cHkdYV119gQdOckjt3xBHvKL3L3TKGp\norRKibWqDa+TpRJ1SWUjD07NiS8eYvW1n/J0oE379M3Cm7nvLCD64cUcbmO09tIwipepTFyhXBrh\n046VCWEUBUZ06kiw8Dce+WY0z092BZQ4Dp3JswvvwLuqlCptg77YX8V/bv2L+7/LvLj9M4XKhX69\ntWSeN39UWpufhcrXA+uGB4v/5qUPrHnqroFYtWP3lO4zeDF8DU+vyOJC1VinGcXLVSauUC6J8GkH\ny4QwigIjLory05RTXGHwkbRo6v5ZcuB7YsbPYFBdZoqFvStOVs33p/eNc8haupP8ju+tGbgx7g5n\ndh3Ml42OriSaZW+9zpcH8ji18g1e/2IHmRrD9TWk7k0n+NYwnBq0Uvj3B/w16GEm9Wrv19KekNvG\nEP/eOjLbYBUvvqdYLwsGLl+ZuDK5FMKnHS0TwigKTNIRQl9z+lfuCfXGzcEa1wHDGXvNdTy8PhMN\nUHU6imofLyzNaMfC2R1VyhkuZv5mTdrvLH7yed765DM+/nwZny18mz0FOkDFwPsW4bnsV1LqDJ/S\nKYS7Fr7HunSJ0t1fsvixyXgb0lAro/hpZzgv3ezeoPUK4n47hPMNoUaGsr1wDLyJISdXcbjY/Hsu\nplHUFR/l3QkuWPo9xPqM+vSjS10muhqXQvi0o2VCGEWBER09ElTYh/B/e/JRl6Xx99KX+M+7K/li\nrjcqwLKbGxQUo2m1FUBdSrmVbbuGGVt83KlvmTv5E+zuX8TCp57gzt7refHjfRTWvUKq/g+ydNYe\nFi0/00r/a0hcuoRzz3/GdJeGx0tIOlLBgMGutEN+TVMcBhBmm0xEVk2zp2fPno2bmxs9e/bEw8OD\n3r17c/LkSfn8Tz/9REBAACEhIQwbNozw8HC+++679umbOomPZt7PoQlrOPiqNR/f/DBbCvQK91KW\nia7IpRA+7XCZkASCBtja2kqABEiffvrpRX22JvklyXXQUim37u+KmC+k/z48R/K3RvK66RHpvx/u\nlwq0+nPFv42TnO49LFVflI5lSMumdpNcb9suFekPSBnfjJQch30ppWkaXlgkHfryU+mfItNN1Wb/\nKX36Q5RU3PhETbz0opuVNG93mfHxqljp3SkB0vVvREoVrXa0Sop9d4oUcP0bUmTji2sSpVe7K6VZ\nO0ubvXP37t3y727Of/b29tK5c+da7ZG5aDXaBn9oJcNfl6xMdFHuvfdeWQZGjRrVYc8p2/8fadTg\nYdK0e56Qnnt2gTRuiI804JqXpSPlHS8TwigKjGhoFD/55JN2b78lYZc0adJno0dLX6RrWmmlQFo7\nzV9aGHNx1J/m9OdSGE7SvO1F8vM3THeWBr4S3X4KuDpWet7VWrpjb7nx8fL90mN9kTzv3S01b86M\nLpb2P9ZXwvNeaXfji2sSpFdckG75y3QrV199tdlG8fnnn7+AD9k8l6NMdFXuu+8+WQbCw8M77Dnn\n106Thr9wVCrXFksHXwyQFDhLM35Kk2okqcNlQhhFgRF2dnay0H/88cft3n6Lwi5JUnXyl9LdD/4s\nnao11YJGyt30hHT7R/EXzSMoP/iY5KkaIX2fVTf8LPtHetDLU7qnsVf3b9CekT7pizRpU1H7tdmQ\nikPS/bYO0uPHq0xesm3bNrMMYrdu3aSCgoJ269rlKBPNUpshrX/9PWl/keGAVqour5a93qqYr6RF\ny5Ik07/Apc8DDzwgy8GIESM67Dmlkb9IqxLLpJw/7pG8QOr30DYpr0EwoSNlQhhFgRH29vYdahRb\nE3ZJkqTq7Hgpvbz5+yWpWspJPHtRFYvmzDfS1T3GS6vyJEmSaqWMX2+WnO0mSWvz2vMpxdL2GVaS\nz0fpkraVK2tOr5b+c/vj0rLUNrzuBaul0QRJP+W2fNnw4cNbNYqLFi0y/7lmcDnKRHN9iHxptHT7\nZv1goerkWumzxbdI3fu9ISXLDk25dOCJYdL97TmYusg8+OCDshwMHz68Q59Vc3KpNMURSTVssXSs\nrEw68uH/pL1F9ec7SiaEURQY0dAofvTRRx3yjNaE/dKjXIr5/F7p9v9+KH373WfSq3M9Jcuwz6XT\nrUVv2kjOTyMkx6mb6uYtTVO27xHJF0spcFG02S99xaEHJWfvN6QTrfT5999/b9Egurq6SiUlJWY+\n1XwuP5loRMEKaWSPZ6WYhp5LzleSn89rUnJNg2Pp70p9+38kZVzs/rUTDz/8sCwLYWFhHfegsmPS\na8NUEo5TpG9SaySp/JD03MhZ0h+FHfdIAyL7VGBEh69DKj/O/+Y9xnam8OXqFxmuiGft6mMUX9Jl\nFO0Jeux7lr/7DPffdxvBZRX0nXo1Hu2cJuo+5RnC49cT00oFDodrvuRU6R7u7W3u61tN4qotuD8x\nj/6t9HnGjBkEBgaaPP/f//6Xbt26mflcM7mMZEJTnE7MwX0cTSnQ161VF5Nbrm2yds4kvW9kTtZS\ndl6miykvzjrFGuI/uY9FERocBjtx5M07mTj8et7P7IGrOesw/iXCKAqM6Nj1aJ0r7BdEdRI/PPsE\nX8bpq9Vos7fzY4Q3t88eTDsVYqun1wzeujWOjzefa7XyTEn8cRQB/czrQ/HfvLdtLB/cP6DV5R4K\nhYKFCxc2e65nz5488cQT5jyxDVweMqErOsqXD81k/vPf8nd6KXnHlrHktS/54ME7+P60xvy1cxbO\nuKtSOHOZLqa8OEbRmoCXYpAkibJjq/jux9X8lVSBlPUVVzt00CMb0Nq4RtCFaX+hrxP2l9q52Q5E\nVxTJqp+30WfYrfxzKoo1P2yl9/vrWRjc7iYRsCN84Yf4zniW1df8wm1eJkxYZTwbjvky81HH1pvU\nFbLjhSXYvL2eyS6tXw4wd+5cFi1aREpKitHxF154AXt784uem8elLxNlER9y94Pb8X9rGcsnu8tK\n85o9D+C/tCe/fm6NZYQbxOvXzrU88FBTWm6F3WW6mLLhOkWxdZSgS3AplHG6lFB6zGd71hEWBlhg\n6TudN9Zt46s7BxnXLG1PnMfy9rLZpC7fTa6p/Z7sArj7yRvrK+S0QMmxn9k36hu+muludlEApVLJ\niy++aHTMy8uLRx55xMwWrhy0eRt59Kb/wX9/YnEDgwjg5D+Jq8ZPw98BnK+ai+u+CAx7kdSkb+bz\nj9ZzMm0rX3y2jlRDWdySSPY53c4414v8QdqJLqEfOn7aUnA54eTkJE+kv//++53dHUEnUVtbK/Xr\n10+WhS+//LKzu9QJVEhH/tNPshj8mhTbXKJvVar097E8SSNJZq+dK1g7TfJfGHPZFhh48sknZZkI\nCAjo7O50CMJTFJhEulJHgoJWUalUPP/88wD069eP+++/v5N71AlUxrJ8ZTr975jF4OZCAzb9GT+8\nh94Dt+jHAz/eydG3lnPaRP0xbd5mFv8+gdWvBnVcpKGDEeFTQZejS4RHBGZxzz334OXlxauvvoql\n5SWU9XKxqMokJr8bwcN7N1M7U0v5uXyqG9gF68GPsHTxMCyaLy+LRtef579/Gv/L1SLSNfSDSLS5\nyNTW1lJYWESNWt36xZ2AQqFAqVSiUCgoLikhI/NsZ3epCUqFEls7W1xdnDt3R/h/SXFxCeUVFZf0\niPu1115n3PjrLkk5ALCytMTZxRkb6w6wNCoHujvYYG/djO9QmcCatWXMfKJHgwxgLSXpMWyKOcP0\ne6fSt5EltXYfilcrj5QkiaKiYiorq9BJl55cWFvb4OPbH4VCgYeH56UrF1ZWuLq4YGXV9sGcQrpS\nzf0lRllZGcciooiOjqO6xsRQUtAmPDzcuWrUCAYNHHDZGEdJkkg+cZJjxyM5m5Xd2d25IlCpVAQF\n+jFi+DDcXM1MsTUHXTF/PzaCR2x/4viHY5BzfXVFHFm2kqKJDzLFy9ivqElYjO/YUrZkf0hIG+y0\nJEmknEzl0OFjZOeca7eP0JWxtrYmJDiQEcNC6dbNjEztOoRRvAjk5Z9nxcq1VFZVdXZXrkjGXDWS\na68Z09ndaBWdTsfW7X8RG5fQ2V25IlGpVNw65xb69vFutzZ1Jcf45LGFHOw7n/tvHIpVTjxRp2oY\ndMsCbhxgZ3ytVkvxphvwX/U8Kauux3w1DHv3HeDAoSPt1m9BPXZ2tsyfN4cePbqbdb0wih1MQUEh\nvyxfTWVVFVZWlowcMZygQH8cHPTrvbRaLWlpZ6iqrpbv8fRwx83N/JxtSZLIzjlHYWGRfKxbN0f6\nePdukwdVUlJqFA6xsrLE16dfm+aTampqOJ12Bo1Gn22gUCjo17eP/HnNQafTkX4mg4qK+hXOvXr2\noGfPHvL5rOwc/tl/iMyzWQBMnTKRkGDTlVguBTZv3SEbxH59+zBq5Ah6e3liYaEPzxUWFpGVnSNf\nb2NjjU+/vqhU5s9yVFVVk5Z+Bq1Wv55DqVTi068PdnZ2rdxZj1arJS39DFVV9TLp4d6L7t3dzG5D\nkiRycnIpKCyUj3VzdKRPn7bLZObZLHn+qrFMVlVVkZB4gsNHjlFRWYlKpWLe3Jn08e5t9jNaR0Nx\negxRyaU4+48gyNvBaHmLrng/Sx75DmlsEBnv/Yf4Jec5fJv531VkVAzb/9wFgHdvL64eM5reXh5y\nUkte/nlyc/Pk6+3t7OjXr49R0ktrlJdXkJ6RgaTTf48WFhb4+vZrU9i5traW02lnUDeY+und2wsX\nZ/O3xZYkicyzWZSUlMrHXF1c8PR0b5NcFBQWkW30rtjg69MXCwv9L1NeXkFsfAJHjh5Hra7Fzs6W\nu+bPw9WMSMKVYRS1FZRU2+Bk3yHbs/4rNmzcQmLSCWysrbn9tjm49+oJ6F/2yOhYYmLjqKxs6kH2\n8e5NaEgQQwYPlH/oxqjVauLiE4mMjiU//3yT8y7OzoSGBBEc5I+trW2zbUiSxMnU00RGx3D6dHqT\n89bW1gQF+BEWGtyioc7NzSMiKoaExCRqa43T7xQKBYMGDiAsNIh+ffuYFP7y8nKiYuKIjomjrKxp\nrTNPTw/CQoLwGzoYlUqFVqtlzbrfSUvPwN7ejicefbBNiuJikp2dw0+/rARg+LBQJl4/DoVCgU6n\n40RKKlHRMaSfyWxyn62tDcFBAYSFBOPcgvLJys4hMiqGxKQTskE0oFQqGTpkEGGhwXj3Nj2rVVpa\nRmR0DDGx8UYDEgPevb0ICw1uVSbjE5KIjIolL79pLTNnZyfCQoIICgrArgWZTD2VRmR0DKdOpTU5\nb21tRWCAP2GhwXSvk8mSklJ+XbGGktJSPNx7seCu2y9SSL2c3Xf58NyYaI7dU8brA2fgvC+Jp/ua\nJ4darZbPvviGyqoqfH36MWfWDCwsLNBoNCQlnyAyKtZooGTAwcGBkOAAQoODcHQ0XeYl/UwGkVGx\nnEg52SQxRqVS4e83hGGhwbi79zLZRmFhEZFRMcTGJTQ79ePj05ewkGAGDexv8juvqq4mNjaBqOgY\nCouKm5zv3t2NsJAgAgP8sDZhqHU6HcknThIVHcuZjKbvip2tLUFBAYSFBuHspH9Xzp3LZfmqddTU\n1BDg78dNN04x+TkNXOZGUUPmpvf5+ZQr3Uv3s/r4UN5b/iLD2xK3MIOamhreeustnnnmGZydnc2+\nr7S0jC++/g5Jkph2wySCgwIAOH06nfUbNlFbWytfa29vj0KhQK1WG43EfPr1YebN05sISnFJCavX\n/k5BQf1I3NbGBguVCp1Oa2RoHR0dmDdnZpPwgUajYdOW7SQl11cusbS0rHuWRHl5hXxcqVQyfdoU\n/P2GNPmcx45H8teuPfLfCoUCe3s7QEF1dbXsNQIMCwth4vXjmhivs2ezWbt+g5HHbG9vh0KhRFNb\na/Qyenl6MHvWDOzt7CgqLuarpT8AMHf2zQzo79ukf+3Jjz/+iL+/P+Hh4W267/c/NpOUnEKP7m7c\nf+9d8m/9+8YtRorfysoKKysrJEmioqL++1epVNwyYxoDB/Q3aleSJA4cOsK+fw7Kx5RKpewZVlVV\nGRnJ0aPCGTd2TBPldTrtDL9t2Iha3VQma2vV1NTUy2TfPt7MmnlTEy+jpLSU1Wt/5/z5AvlYSzJ5\n65xb6Nmjh1EbGo2GLdv+JCExWT5maanC2tqG5mRy2tRJBPr7AXAmI5PlK9cCcMftc9vZWzSBOoEX\nPSdifziThXbfEhaeyuc77mbgwEB6mVG15kTKSdb/vgmAxx65H6du3aisqmLd+j+M5pytra2xtLRE\nknRGAxYbGxtmz7ypyWfV6XTs+nsvxyKi5GMqlQobG/33WFlZZZTgdf111xI+PKyJXCQlp7Bx8zYj\nGdJHfRTU1NQY6bAhgwcyfdqUJpGl8+cLWLX2N0pLy+Rjdna2KJUWaLUao4iEq6sLt865BZdGeram\nRs3vf2zmdFq6fMzUu2JpqeKWGTfKusDgiSuVSh5/5IFWo1YdYBR1VObnonbywLmjSxnpMvj62jmc\n/3ofL/sX8XN4AL8vPsWGqea78+ZQWVmJvb09Tk5OPP300zzzzDM4ObX+jNi4BDZv3YGVlRVPP/Ew\nKpWK06fTWb3udyRJwtbGhrDQYEKCA3Fy0hdZ1ul0pKWd4XhUtKwsPT09mD9vtixsZWVl/PTLSsrK\nylEoFAT4DyUsNBhPj/oQRG5ePlHRscTExqPVarGxtuauO2+TR9Y6nY61v/0hP6OPd2+GDwth4ID+\nxiGIuHiOR0bLyuimG28gwH+o/BkPHz3O37v3AXovYPiwUAID/LC10efk1dbWknwihWMR0Zw7lwtA\nYIAfN06dLPc1KyubX1euRavVYmVlRWhIIKHBQXKoQ5IkzmRkEhkVQ/KJk4B+ZHnX/FuxsbHhp2Ur\nyM45x8jw4Vw/fmzbf+A2cO+99/Ljjz9yww03sHjxYrON4/sffY5arWbKpOsJCw1Go9GwcvV6Ofzr\n69OPYWHB9Pf1kQcMpaVlRMfGERkVIxuU2TNnMGhgvWFsOBfl5ubK8GGhBPgNkQdRarWaxKQTHI+I\nIq8umjBieBgTrx8nt5GWfoZVa35DkiRsbGwICwkiJCRQHm1LksTptHQio2I4mXoa0IdT5982V87u\nKy8v56dfVlJaWoZCocDfbyhhoUF4eXrIv3Ne/nlZJjUaDdbW1tx1xzx61IVldTod63/fKD/Du7cX\nw4eFMmhgA5msqCA2LoHjEdGUl+ujCTdOnUxQoD+SJPHdD8vIP1/A6FEjGH/tNeb9qP+KMv556ia+\n67uAkefX8O5vPZg9by7/fXUqvcxwFv/862+OR0bj3duLO+ffSnVNDb8uXy3/VoMHDSAsNIR+fb3l\n77GwqIjomDiiomOpqVFjYWHB7fNmy1EASZLYtmMn0TFxAPTq1ZMRw0IZOmSQrEOqq6uJS0jieEQU\nRXWe2/hrr2b0qHp5Tkw6wYaNWwD9AGl4WAjBQf44OOg9U61WS+qp0xyPiJY9Nx+fvsyddbP8exUW\nFvHzLyupqq7GwsKCoEB/wkKC6FUXMdOH2c8RGR1LXHwikiTh4GDPgjtvl5NjNBoNK1atkwcJvr79\nGB4Wgq9PP+N3JSaOiKhoqqqqUSgUzJk1gwH9famtreWjT79Co9EwY/rUZgf2DbkAo6ijMjeH2u5e\nODWKoFQnfMa8G//DH3nXsTx1K7d7KIEaUlZ/SkTok9w2qP3TpnUaDahUKMsP8njQ43huPszCQefZ\ntnQnXnffRVA7FJA1GEUDzs7OPPPMMzz99NMt7hhw9HgkO3ftobubKw/ev4CS0lK+/f5n1Opaund3\nY96cmS1mRUXHxLF1+1+A3pBMnzYFSZL4cdkKzp3LxdJSxZxZN9Ovbx+TbeScy2X1mt+orKrC2cmJ\nB++/G5VKxe69+zl0+CgA144dw1Wjwk2HPqqqWLN+A1lZOSiVSu6563Z69epJWnoGK1evA6B/fx9u\nuelGkynQkiTx1649HK8buU68fhwjhodRVVXF0m9/orKqCqdu3bjt1lktxv2TklPYsHELkiQxoL8P\nc2ffwvrfN3Ei5SShIUHcMHmCyXvbA4NRNGCOcZQkibff/QiA226djU+/Pmz/cyeRUbEATJp4HcPD\nQkzeX15ewaq1v5GXl49KpeL+e+/E1cWF5BMn+W2D3ssYOmQQ06dNMTn/qNVq2bZjpzynOX3aZAID\n/CktLeOb739GrVbj5urCbbfOblEmDQM9gAC/odw0/QYkSeLnX1eRnZ2DSqVizqwZ+PTra7KN3Nw8\nVq5ZT2VlFd26OfLwA/egUqnY+88BDhzUG/ixV1/FmKtGthiOW7t+A2fPZqNQKFhw1+14uPdi3W8b\nSTmZelFkoR4t5YWVWLk6YlFeTI2dM3ZmRvE3bt5OfEKi/F0a+q9QKJgxfSp+QwebvLeoqJiVq9dT\nXFKCrY0NDz6wAHs7OyKiYthRN0cZFhrMpAnjTU4r1NbWsmHjFnkgMm/OTHx9+5GXf54ffvoVnU6H\np4c7c2ffgp2d6XD3kaPH+XvPPwCMDB/G9eOvRavV8s33P1NUVIytjQ23zp2Jp4e7yc+TkXmWNet+\nR62upVfPHty74A4UCgVbt/8lG3jDoNIUZWXlrFr7G/n557G0VPHAvXfj7OzEl0u/p7i4pNX74QIW\n7+vOrWbG4Kt49WhF4xP89XMa497/i9iT65nnoW+6+O/neOzISG4wGER1Jju/+ZLvfvqWTz/6lYii\nf7cWR6lSodRks/HVD7H84A/+z88KVJ5cd20eix5fS047LPVpPG4oLi5m0aJF9OvXjzfeeIPS0tJm\n7zMIoq7u/qioWNTqWmxtbFo1iAAhwYFcO1afVRmfkER5eTkZmWdlj+vmm6a1aBBBP6KfM2uGvt8l\nJZxISUWtriUiMhrQz2+NGW1a+QDY2toyd/YtOHXrhk6nk0MyR44dB/RJMDNnmDaIoA+pTrx+HEMG\nD6q7NwKdTkdsfCKVVVVygkRrE+FDhwxiyqTrAUg9lcb5gkK5750xE7Bt2zZGjhzJ1KlTOXbsWLPX\nKBSK+j7qdFRWVhETqzdOY64a2aJBBH24at6cmdjb2aHRaIiMigHgyFH99+/d24ubbryhxYQcCwsL\npk6ZiI+P3lgdPhqBJElEx8ShVquxsbZm3q2zWpXJoEB/2QNLSEqmtLSMs1nZctLDjOlTWzSIoPdc\n5s6+BYVCQWlpGUknUqitreV4RDSgV+RXjxnVskza2DB31s04OzshSRLHjutlUqnsjKU5Fji4OmIF\nWDiYbxABdDp9WNJCZUFhUREpJ1MBmDRhfIsGEcDFxZl5t87E0tKSqupq4uIS9AbqiF4uBg0cwOSJ\n17U4z25packtM27Eo25O0fBOH4+IQqfT4ejowNw5pg0i6OV71MgRhI8IAyAySu/Bppw8JXuhs2fN\naNEggj5adctNNwL6SJch2c4wkLvm6tGtGjTDVJGdrS21tRoio/XvitLw/rV4t542G8Wq9EjOac6y\nYVUsDc2iLv8YR/rew8OzriXQ017fcOUxXnssjYdfGoszAJUcW3wrn1lP554FD3D/uBSeXPAD6SbK\nIpmFJoednywlZ963vDPyFJsT9L2yDniYpy1e5pk/i/9F4y1TVFTEq6++Sr9+/XjzzTcpKyszOm9R\nJ4xarRatVktMbDygf+nNXTcTPjwMGxsbJEkiJjZBVoieHu5N5pdM4eXlKV8bFR1LYlIyarUapVLJ\n1VeNMqsNWxsbRo4cDujDKrm5eXJizpirRpmVoapQKBh7zVWAPtxx+nQ6UXWfJzDAz+yM2+CgAPn7\ni46ONeuejmbbtm2Eh4czbdq0Zo2jSqUPq2i0GmLjE+pCxZaMHjnCrPYdHOwZPiwUgNi4RLKysuUE\njGuuHm0y8aUhSqWSa6/Wf//5+efJzMwiqm4EHhoahJOZ+ySOGB6Kra1BJuNlmXTv1ZPBgwaY1Yan\nhzuDBuqvjYqKJSk5hZqaGhQKBVePMU8mbWxsGBWul8mk5BNUXYZLnrRa/ahdqVQSFa3/LRwdHQgN\nCTLrflcXF4IC/QGIjI4lLf0MxSX6suTXXnOVWclGKpWKMXV6IC09g3O5uSQkJgEwMny4yYSoxowZ\nPRILCwtqa2tJSEwiqs4g9e/v02KCV0P69/eht5en/vNExRIbF49Op8Pa2kr+rVvD0dGBYXUDTUOo\nvi20uaJNeY6OW54dwyc/ryL2rdGMrosqFkUfx3XU9Ub7u5XsXcIKv6d4w5CdXHKA976vYVqkFxaA\n3eApBEffxy8pd/GK3wVMQOoK2fn4KCb+VE4Pxy/4P7Un/z14nJsBcGDUIxO59dE/yJ1yN83lVn32\n2WdkZmai1WrR6XTN/l+r1VLTymL7oqIiXnnlFT766CP+85//8MQTT+Do6CgrKq1Wx+m0dCoq9RPk\nISHmLx2wtLQkKMCPo8cjiYmNp7TO8Jr70hgICwniZOopMjLPoq6bHB8yeGCLI8DGBPgP5e/d+9Bo\nNOytS+ywt7dj4ADzk1u6u7nSx7s3GZlnOXI8Qs5ECws1//MolUpCg4PY+88BYuMS6FvnLRcXF/HF\nF18Y/YaG/xr+be655v69f//+Fvu2detWtm7dytSpU1m8eDEjRuiNnspCRW2tBo1GS1zdyDfAfyhW\nVubLfXCQP/v2H6S6upr9dfOILi7ObVqX5+HhTs+ePcjLy+fg4aNygkJblrPoF8sHcOTocWJi4ymv\na6OtMhkaEsSJlJOczcqWkz4GDxqIQxu2p/L3G8qu3XuprdUYJYxdLsieolIpy0VIUGCbsqhDQ4KI\niIymuLiEw3VeYm8vT7PX5QEM6O+Dg4MD5eXl7PvnELW1GiwsLAgM8DO7DVtbW4YOGUR8QhJRMXHy\nMpKwC5CLs1nZpJxMlTOYA/392rQ0LCQ4gH8OHKKqqppTp5tmMLdEG41iOaeSHBk3/yEOfrqQVbFv\nMXq0PVBC4jFrRj7VcC1UJXGrdtB92jcYpvU05yKILnNmnm3dD65ywcsqg70pFdCSUaw5w5ZP3+Xb\n3w9wqlSDBCi6jeGdP5Yy9eszSF83f5v1gEkExH/CoeK7udm56flly5Zx/Pjxtn0FLVBYWMhLL73E\nhx9+yH/+8x8mTJgE6Bf1GjKsHBzszR6RG/D09AD0c3sG5WE4Zn4b9aELw4i6rW3YWFvj5uZKbm6e\nnCXq7t7LLC/FuC8eZGSepaoueUSpVNKrZ882twEYZaXm5+fz9JOPt6mdjqKxcbRQGQZIWvm78/Ro\n2/fv4OCAk1M3iotL6n9Dj7at71IoFHh5epCXly+3YWdn2yTbrzW86uSpqvrCZdLLSCbrvhPPlkNs\njbG2tqJ79+7k5JwzymIE2Lr9L0rLynFzdZETi45HRJFaF+GYfct0VCoVGZlnOXhY791fe/VoPDzc\nqaqu5o9N2wD94DGkLnN8246dlJSWGbcZGU1qXcLarFumY6lSkZGZxcG6OfuxV4/G08Od6uoaNmza\nqm9z0ABCggPrPUULZb1ctPE76NmjO5aW+kHXhX6PSqUST49epJwsl+XC1dVFTpgzFy9PD+ITkowy\njdsq5151ciRJEtXy52lbG46OjnTr5khpaVkTuWiNtoVPa84QWxmKX//ruTe8hN+XR1MOUJlGZE0I\ng4wigqXE7akldFB9poumvIBKLLEwvMNKS6wUlRSU1mKSighev+46PioaxWOvP0lwdgI9n/6JNSve\n5Poepm8DwM6HQO1Ros43P7HYVmVuLsXFxSQnJ1NcUgyAVlefznwha6eau6et7bRHG83do6Ad2rig\nfjQ9dimuUczIyODMmTOyrDUM5VzIMrr2+e7av40LaaejZNJAYWER+fn5FBcXy8fKyivIz88nPz9f\nnl+qrq6Rj9XULUmRdJJ8rGG6f0Fdm4VF9YUyyhu0Sd28dk1NgzbrBm46nU4+VlaXzS17ig100aXy\nm7bHu30hfelouWiNNnmKmpw48sKuoYeqJ5Mfm8BTD33JP2+MYWJhNNlDxmM0I6SrIr9AQU/n+kco\nLK1RoaM+H0KLVrLA0tJU52uIf/c2vghYxum3xmBPJQ6znmJ2nguD+vVofdNUK1e87MpJLqyFZjZr\n8fb2pqCgAAsLCywsLFAqlSiVSqO/LSws0Ol0HDnSegkmCwsL7rjjDl5++WUGDBjAydRTHIuIkUeD\noE+RlySpTT9Yc+FbdRvrpzZXgFxd0/ai5I2fq76AwuaN29BqtWg0mjZVbmnuuRZKC6ysrIx+R8O/\nTf2tUCia/f1N/fvkyZPk5eU10yNjAgICePXVV5k9ezYKhYKl3/4of1YDNW38/iVJavKbtbUN/T2N\nf8NadDpdmwYVzT23rfLUHm3o72n+Xbjj9rlNjo2/9mrGX3u10bFBA/sbLXMBvff8xKMPNm3ztjlN\njo0bO4ZxY43LDA4c4MvAAcb3N9dmwzlFA239TbVabZOCGRcmF41kS932Gs3NPbdGXdOmaZrm9F1r\nU1iNae5dMZc2GEUtWYeTGDhqNhYocbvuSWaqZvDJlkyCHU/gHTTP2EgpVdhaajhbXZ/vY+nSGyfN\nn5Qbfj9NJYVqR/p0NxE6rUnhl8+ymbE1FHsAXRHxh8vxnexi3i7iOjUVGkvsbZo3QGvXrjWnFcrK\nylpceqFSqbjjjjt46aWXGDCgPtHAQlkfMjNkXtXUqDmdlk5/Xx+zng36JALQhxAKCgspLS0jMflE\nm0IKSUn6NqysrOjt5UVJSSmJySe4arTppRiNOZebJ88B9vXuTVZWNplnsygrK8PR0bzEIa1WS3KK\nfq1hH+/e5J8vkItkN1z/2BqJdZ/Hw72X3P9+Pj5tfnnaSuMlGY1pbAwNWNQZfE2dLKScPEVScoqc\nPGMOGZln5Xnpvn28yc4+x+m0dKqqq80Oc6nVtXL6fR/v3uScy60r35XepsIHiQaZ9PCguKSEkpJS\nkpJP0Lu3Z5vbsLS0pLeXJ0XFxSQln2g187Qhefn5nK8rYOHp6U6uGQOWSwltXejZQmmBp4c7Wdk5\nJCWnMHTIILPbOJGSKmde9/H2Ijcvj5SUVCZPvM7saFh5eYW81rBvX2+ysnMoKSklJ+ccHq1kjRqQ\nJInEJH3RBW8vT06lpVNTU0NS0gmuGj3S7M9jkAsHBwd69ezBqdNpJCWntJp52pD0MxkXHI42f2io\nTmNP/CBG9a6zo44jefJRH/a9+jorD9oR2q/xS+mAr78VZ3Lq47nKnmO4eVApSTl1iqviNHHacGYG\nOUJ1Gn/9foBzRgMeHRrLXvh76ecqa5K+5xP1f/hwqpl1QatzSK/tg585pSVawFSqv0qlYsGCBSQn\nJ/Pjjz8aGUTQzxMY7nd1dZEzsCIiY8x+dkFhEWnpGQAMCwuWEyJi4xKNqkm0hE6nI7IuSzPQfyjD\nh4UA+gzEtuzUYMgydHVxZsxVI+UMREMGozmknDwlV+QIDx8mj9AjoqLNXlJRXlEhL+Jvy4vSkQQE\nBLBmzRpiY2OZM2dOE6WusqgfIIWF6PuceTar2VJopjDIjX53kHAsLfWl7tpSYDwhMUnOPB41aoSc\npNMWmSwqLpYzj4eFBcsJNrHxiUYVcVpCkiQ58zjAfyjDh+sHB+cLCtu0HZFBJl2cnVtdnnQpotMa\nwqdKOdnsRMrJZsscmsKwvGrgAF9G1mVoVlZVye+IOUTHxumLN1hbc9WocLnIR0SU+XKRnXOO3Dy9\nPA8bFkJQXZJOZHSs2duj1dbWyln6IcEBDAvTvytnMjKNKiW1hkGevbw8mlRNao3WjWLZXh4LC2Xi\n7Gc4PnoifWXf0hr/Rz/k7trveOtkIEOaOApODL+pJ8cP5yDbOWt/nvzkJmI+/p4DqfH8/uEvOL34\nMbf0UqI+tZKnZ97I4w2XUFgP5dElI9nxzpesWfYWT7xexMtblzDC3NrGRcfY43ALV7XjbjKgN4b3\n3HMPJ06c4Mcff6R//+aXRjQcpel0OlmBp546bZYiq62tZdNm/WS/g4MDA/r7EhIUgEKhL5+2bfvO\nVg2JJEns3vMPxcX6NO3Q0CA86zIQAbZu+9OsVPbUU6flBbShocH6rNhAffLBocPHyDLDuJaUlvLn\nzt2AviqFs5MToXUGIisrhyPHIlptQ6vVsmnzdnQ6HVZWVgwd0vJaro6mNWNooOGcoo9PX7mC0abN\n280KdSUkJpN8Qp9dGRYShLW1Nf5+es/6n/0HZWXUEgWFRezeq8+eHTJ4IPZ2drJMnjqdJv++LVFb\nW8vGOpnUZx73JyjQH6VSSU1NDVu3/2nW4Gb33n/kqENoSBAe7r3k+ptbt//VbD3gxpw6nSYvYwgN\nCbxstg9riMFYKJUWDBk8CBtrayRJYtMW49Jqpjh6PFKuihQWol/qZfD4d+7aIy/PaInsnHMcPKRP\nCgoK9NdXlaqTi9i4BHntZEtUVVezpa6oQ4/ubvT28pQHS6WlZez8e69Zumr7n7vkxJiQoEB8ffrJ\ny682btlu1qArLiFR7rNhANoWWjeKVt6Mv3EY/UY9yUuTexrf4DyWd3cdYd9302laE16J963/xWvV\nSlLU9cfcrn+XDR9cD6fy8X1sFSseG4o1YOW/kNiMnxhj1/CLs6L//SvY9OaNDJ/8LF+v/oh5/c2t\niqMj449fsHrsHgb/y3Jzhh9TpVJx7733cuLECX744Qd8fVsONzVMIa5RqxkyeCD9+upH5pu37uDQ\n4WMm19AUFhWxfNU6eW81Q1UKBwcHrhkzGoD4xCQ2bNwih9QaU11Tw/Y/d8nGJiw0mJ49eqBQKOT2\nCgqL+GX5apNKVafTERUdy7rfNgL6hfqhdd7qqJHDcXLqhlarZcXq9SQlp5gU/MyzWSz7dRUVFRVY\nWVly3Tj9AnCffn0YPGggAH/v3seefftNesClpWWsXvs7aelnAJhw3bVYWVnK36HFRUy0MdcYGrCs\nK2ygVteiUCiYPPE6QL9IefnKNUY1bBui1Wo5ejxSNkS9e3vKZaquvmoU9vb2qNW1LF+5ltRTp5v9\n/iVJIi39DL8sX0V1XajVsF508KAB+PTTe1hbt//FwcNHTcpkUXExK1avIytLvz5y0gR9eM7B3p6x\nV+tlMjHpBL9t2NxsQXHQzw3t+HOXvHQgJDhQLpJvqIlbVFTML8tXGe0M0RCdTkdMbDzrftuIJEn0\n7NFdHlwZlhtdTFn4NxjkRqvTYmlpyYS6jNb0M5msWvOb0W4SDamtrWXfPwfZWVdzeOCA/vj69gNg\n/LhrsLa2oqKykmW/riYj86xJuUg+cZIVq9ah0Wjo1s2RUXXrZkOCAuRBym8bNhMZFWPSSOfln+fX\n5avlQhoTJ4xHoVDQvbsbI+qmB45HRLF1+19UVzefCVpZWcUfm7YSF58I6GW7WzdHlEolkybo35Vz\n53L170qD3YAaotFoOHz0OJs2bwf00wOGAgi1bdARHVwQvJx9j45j2S07+W6icyvXVpPwy7fkTXmM\n8T3aQaArj/Pi9Z9wzdZfmPovPcWysjKeeeYZXnrpJXx8zJ8LrKio5JPP9etFbpp+AwF+Q6muqWHl\nqnXk1FWlsbW1ISjQX64RWVOjJiEpmbS0M3I7kydeJy9GBb0w//nX33JoQ6lUMmTwQAYO6I9KZYFW\nqyMt/QwJicmyghs6ZBAzpk81mtBPTDrBH5u2yi+Md28vAvyHYmdniyRBbl4eMbEJco1JV1cX5s+b\nY1SVv6CgkOWr1snXODs5ERwcgJurKwqFvuxSbFwC5+oUnKWlitkzb5YVMehf8DXrNshzGtbWVgT4\n+9HHuzdKpQK1upbkEyc5mXpKvmfsNVdx9VWjjHYZGHft1Vw1yrxapBfK+++/T9++fZvMGbaGoVRV\n9+5uPFBXEDwmNp4t2/6Ur+nXtw/+fkOwsbGu2w4sl9i4eNlr6tmjB7ffNttoMXVubh4rVq+TR9du\nri4EBwXi4qKvW1pSUkpMXIK8i4q1tTXz5tyCl1f93F9NTQ0rV6+XB2C2NnUy6VUvk4nJJ4x2UZk4\nYbys8KBpGT9TMpmYlCwnhQweNIBbZtxoJJPJJ1LYsHGr7EH17u1JoL+fLJN5efnExMXL4UUXF2fm\nz5tDt26OVFVX8/mX31JbW8v1469lZPgws3+fzmLt+g2cTD1Nv77e3D5Pn8Rz8PBR9uytXw87oL8v\nQ4cMwsrKEp1Ov/VSXHyiPH/ex7s3c2ffYlRR6kxGJmvWbZAHmL169iAoKIBujg5Ikn7QHRMTT1Fd\nZq69vT23z5st16EF/Tzj8lVr5QGbg4M9QYH+uPfqhUKhX0YTn5Akh7sVCgXTb5xCgF99boBOp2PT\nlu1ygXeVSoXf0MH4+vTDwkKJRqOvn5qUnCL/5iHBgdwweYLR+xUVHcu2HTvlv3369cFvaP27kpV9\njri4BHnP2l69enL7vNnY2tiQm5fP9z/+AsBtt85qteJSx++SUX6UNxYsJ+zLj5jWswVjp87keIo9\noQGu5iXRtEgZB197hK3Xfsab49o5dtpG1v32ByknT+Hq6sLdd8zD1tYWtbqW7Tt2El9XNcIU9vb2\nTJ44Xi6N1hB9vcEI9h881GJIQaVSMWJ4WLM7I4B+QnrL1j8pMVGqzsCA/r5Mnza52S2oSkvL2Lh5\nW6tzQW5urtx04w1ySamGaLVa/ty5m5jY+BbnH2xtbbhu3Fh5x5EjxyLY9fdeAB5/5IE27bB9McnJ\nOcePy1YAxns/nkw9xbYdu+RBhSmGDhnE1CmTsLZuGvYoLCpi46Ztre7Y3rNnD2ZMn2qk+AzU1tay\nbccuEhKTWgxz2dvbMfH65kuQSZLE0eOR/LP/UItZyRYWFowYHsq4sVc3m/F6JiOTLVv/bDX019/X\nh+nTpmBnZ4tOp2PLtj+Ji0/E0tKSxx+53+R2aZcSJ1NPs3b9BgDmzJohV56Ki09g19/7WtyYXKFQ\nEBwUwKQJ45vN3M7NzeOPzdtanYvz7u3FTdNvaHb9dFV1NZu37DAakDZHt26OTJ0yCV+fpgZHkiT2\n/XOQI8ciWqwuY2VlyVWjRzJ65IhmddWJlFR2/LXLaKeU5vAbOpipUyZiZWVFbW0tq9f+TkbmWVxd\nXXjo/gWtDmYvztZRNXmklbvi49bmAjoXhqaYjAIr+vQyf2PVjiLzbBa/LF8N6Efx48ddQ39fHyws\nLORdEGLjEozCCl6e+nj8wAG+rWaPGXZBiIyOpbDhhq7duhEaHEhAgx0rTFG/C4JhnzLDhq5W+PsN\nJTQkEFeX1gcXefn5cskujbZ+k+H+vj7yXn6tCaRhF4SYmHgqKuuFv1fPnoSFBjN40ABUKhXl5RUc\nj4iSF0gPHjSAWbfc1GofO5PlK9fK3vCokcMZFhYi15M17B94tm5+CAx7Wfob7aLSEjnncomKjuXE\niZPy2lilQsnAgf2b7KJiirKyMqJj44mNTaCq2ngBdlhokNEuKqZQq2vr9gKMabLJcEjdnnnmyGRa\n+hkio2JJP5OBQSYtLa0I8B8i76IiSZJ+8f2ho3JYffSo8CbLLi5VJEni2x+Wcf58gb7s4phRhAQH\n4mBvj0ajqdtrM5ZzubnyPXZ2doQEBRAUGNDqNkiSJHE2K5vIqBhST6UhSXXZrhYWDB0ymNCQIHr1\nbD0RpaiomKiYOOITklA3WKrRt483YSHB+Pj0bXVJT3V1tVztpqTBgMfFxYWwkCD8/Ya0WuFJp9PJ\n+782zGOwsbGR35Vu3RzR6fRVxPbtPyTXip4+bYpZFXou8/0ULw8SEpPZuHmbPAK3sLDAxsbmghZv\nC/QvRsNEDC8vD+bNmWlyc9JLhaqqKlav22C0Y7itrU2HFZG40pEkUKtrjNbojRgWyvXXXXtJFnIw\nRVlZGStXr5eXloB+w1xD9rqgbUiSfkqgoVc64bprCR9hXjhdGMWLRPqZDA4eOlo36hW0B3pPyo+x\n11x1yRtEA2q1mn8OHCIuLrHF0Jigbbi792LEsBACA/w7uysXRGVVFfv+OUhCYtIFLbwXNE8f796M\nDB/epvrMwiheZIpLSsjNzTN7PVdncOJEMt7e3tjZmV+Y+WKiUCqwtbGhj3fvNhUJvpTQarXyYnxJ\nd+m9gvn5+fz99y59QXStjunTp9PNjI21LzaWliq6u7nRvZl50suR2tpafV3g6upLUi5A74Xl5p6j\nT5+WE1Y6E0tLS3r16tHmmr4gjKKgGW644QbGjRvH888/39ldEXQS+/fv55pr6neuT0pKYsiQlncs\nF3QN3nnnHfbs2cO2bds6uysdgghaC4w4fPgw27dv5/333zcqhCzoWjSekzNnIbngyqe8vJz333+f\n7du3c/To0c7uTocgjKLAiMWLFwNw/vx5vvjii87tjKDTaJz8I4yiAODzzz/n/Hn9mleDrrjSEEZR\nIHP48GF27Ngh//3BBx9QaaJajuDKprFRNLd2peDKxeAlGti2bRvHjh3rxB51DMIoCmQWLVpk9Hde\nXh5fffVVJ/VG0JmI8KmgMZ999hkFBcaFAF577bVO6k3HIYyiAIBDhw7x559/Njn+3nvvCW+xCyLC\np4KGlJWV8cEHHzQ5vmXLFo4fP94JPeo4hFEUAE29RAO5ubl8/fXXF7k3gs5GGEVBQ5rzEg1cad6i\nMIoCDh48yF9//WXy/LvvvmvW9lKCKwdhFAUGysrK+PDDD02e37x5MxERrW/7drkgjKKg1Swy4S12\nPYRRFBhoyUs0cCV5i8IodnEOHDjQopdoQHiLXQthFAVgei6xMZs2bSIyMvIi9KjjEUax09BQcCob\n0yksOkoysinv4Ex4c9canTt3jqVLl3ZsZwSXDMIoXgJoCjiV3VKSW8friE8//dRo952WuFK8RWEU\nOwU1qT8/zaK9ZUZ7R1ZnHONIlqEYsBKyl/N/r+0gr4OE/sCBA+zcubP1C+t45513hLfYRRBGsZNR\np/Lz04vYW2akIcg4dgRZRXSwjigtLW1xLrExGzduJCoqqv07cpERRrETUMf/j/lrR7JwwWCsAW3e\nPpa+9TIPXDea+7cUYJBvp1FP8ajFYu5bmUNH2MW3334bKysrrKyssLS0bFJcW6FQYGFhgVKpRKlU\nkpuby7ffftsBPRFcagij2Jmoif/ffNaOXMiCwdaAlrx9S3nr5Qe4bvT9bCmo1wYdqSO+/vprysvL\njfRD4/04G+oHhULBG2+80c696AQkwUWmSFo3rps0d1d5o+Ol0q7be0ohS7MlbcPDucukkR4PS4cq\nLk7v/Pz8JPQ7ukrPPPPMxXmo4JLj3LlzshwA0h9//NHZXeo6FK2TxnWbKzVVEbuk23uGSEuztcbH\nL6KOePTRR2WZCA0N7fgHdgLCU7wIaGtqkMfZJftZeiSc+cPM3Jap5wTutF/D+tTm9ljTURzxI5/v\nOkez4/iSCL7/chfnxCBf0EaEp3gx0VJTU//9luxfypHw+ZirIv6NjiiJ+J4vTemPVpCu0A2WhFHs\nSNRpLH9sPi98sJDZM5YQVQkUx3CgOoj+Zm9V6MgAv3L2JJQ1OVOd+Cl3LMzn2jHuNLt3u1MIk1yX\ncdcrBym58E8h6IIIo3hxUKct57H5L/DBwtnMWBKFXkUcoDqoP+bvZnrhOsIpZBKuy+7ilYPmaYjG\n4dMrEWEUO5DCnS/y/KEQbvZOJiqnErUEmqpSqpTWqNrwzVsq1ZRUNlJK6hN88dBqrv30aQJtTN1p\ngffcd1gQ/TCLD4tSbQLzEUbxYlDIzhef51DIzXgnR5FTqUZCQ1VpFUprVZuU8wXrCAtv5r6zgOiH\nF9NWFSE8RUEbKSfhjwNYjLqKkDu3kH50CSPtQeXSj97aTM5Xm9tOLflZKnw9rI2OFv/9Eh9YP8Vd\nA61avl3pzowXw1nz9AqyxEYHAjMRRvEiUJ7AHwcsGHVVCHduSefokpHYo8KlX2+0mecxW0X8Sx2h\ndJ/Bi+FreHpFVqvJOg09RWEUBW1Dk8uxQ8X0HTMA24bH3cZxh/MuDubXi5+uJJplb73OlwfyOLXy\nDV7/YgeZmrqTNansTQ/m1jCnBo0U8vcHfzHo4Un0MuMXtA+5jTHx77EuU1hFgXkIo9jxaHKPcai4\nL2MGGGkI3MbdgfOug9SrCB0l0ct46/UvOZB3ipVvvM4XOzIxqIh/ryPsCbltDPHvrUOoCGEUO4Sy\niG94+enn+DROS+WOt3nx3S31Rk41kPsWebLs1xRZqJVOIdy18D3WpUuU7v6SxY9NxlulP1cZ9RM7\nw1/iZvcGD6iI47dDztwQ2vAlaAHHQG4acpJVh4vb5wMKrniEUexIyoj45mWefu5T4rSV7Hj7Rd7d\nUm/kVAPvY5HnMn5NkTUETiF3sfC9daRLpez+cjGPTfamTkW0i45wDLyJISdX0ZqKEJ6i4IJwHPYg\nrzwUjLXzOF7++GPe+b9pspEDFf0fXMqsPYtYfkbTUjNQk8jSJed4/rPpuDQ8XpLEkYoBDHZtNr2m\nGRwYEGZLckQWNW3+NIKuiNhkuCNxZNiDr/BQsDXO417m44/f4f+m1Rs5VP15cOks9ixaTmsqot10\nhMMAwmyTichqWUOIRBvBBVOevJ9cz2sY7NDMSbthLF77DJUbD1NssgUNOft24fzGD9znozI6oy7K\noMiqJ86WjW6pjuO9GwKZ8GZUo/JxKpw87Cg9nUdzSdsCQWPEJsMdTTnJ+3PxvGYwzauIxax9ppKN\nLbpubdcR1XHvcUPgBN6MapRVo3LCw66U03nma4gr1VNUtX6JoO1UcurQaawDn8fT1By38ygeeaKl\nNlR4THyCe5o5I+l0SAoVysaDNm0pZ5LiSfIsabLuSJK06HRXphALOgYLCwvZGAqj2M5UnuLQaWsC\nn/fEtIp4hBZVxAXoCG3pGZLik/AsaaIhkLQ6WlMRXSF8KoxiR6ArJuFQHr3vHNTsKPDfYunkiWNN\nFKW1jU7Yj+HzdInPm9yhoTSnGgcvFxo7lwKBKYRR7Dh0xQkcyuvNnYM6QkOY1hH2Yz4nXWqqIdCU\nklPtgJeL0BAifNoRVJ7kQKYL4UHdm19U/y9ROg0mxOokqSXmzvNUk5FQhe9wb0wuaRQIGtFwXlEY\nxfal8uQBMl3CCereERriAnREdQYJVb4M925ZQ3QFT1EYxQ5Amx9DRMVgxg82vyZFm3AK5pbg8/wV\nV2re9ZUp7EryZs5ot47pj+CKRBjFjkJLfkwEFYPH01Eqoq06ojJlF0nec2hNRQijKLggypL2kT/4\nFsJdO+oJ7kx5Jpz49TGUm3F1deIqtrg/wbz+HTMqFVyZCKPYUZSRtC+fwbeE02Eqok06oprEVVtw\nf2IeQkUIo9h+6PLZ8dxkZrxzhPi96QQ8OB3vDhSwXjPe4ta4j9l8rrXwSDF/v7eNsR/czwAh8II2\nIIxi+6LL38Fzk2fwzpF49qYH8OB07w6ZXjFgto4o/pv3to3lg/sHtNof4SkKzEdXRlpcPnbSbpbl\nLeCD2/t0qMBjF87CD3356tnVZJnUVzoKd7zAEpu3+XCyi6mLBIJmabgs40pVgBcTXVkacfl2SLuX\nkbfgA27v08GjVHN0hK6QHS8swebtD2mrirhSZUJkn7YXKl8e3rKTCfF5OD07hB6tlCRtD5zHvs2y\n8++wfHcu/5nQq6kRLjnGz/tG8c1XM3EXXqLgX3ClKsCLicr3YbbsnEB8nhPPDulhcilGe9Kajig5\n9jP7Rn3DVzNN7LTTiK6weF8YxfZE5cqAkI6bJWiKFX1nvsL/mTrtNJJnloy8iP0RXEl0hVDZxUbl\nOoCLqiJa0RFOI5+hLSqiK8iECJ8KBIJm6QpegUDQGGEUBQJBq1ypXoGgbQhPUSAQdFm6ggIUXDhX\nqkwIoygQCJpFGEVBY7pCSF0YRYFA0CxdQQEK2kZXGCgJoygQCFrlSlWAggvnSpUJYRQFAkGzdAWv\nQNA2ukL0QBhFgUDQLF1BAQounCt1oCSMokAgaJUrVQEK2kZXiB4IoygQCJqlKyhAQdvoCtEDYRQF\nJhGKsO1oCk6RXWn6vK4kg+xyczeH7lyEURQ0pqFM6HSXhxy3FWEUBUaInREuHHXqzzy9aC9ldZWV\ndWWn+GfTH+xOyENdd42SbJb/32vsyLv0FUpX8AoEbaOhfhBGUdAl6ApC3yGo4/nf/LWMXLiAwdag\nSf+F++78nsIBwTgeXsjM/2wnTwc4jeKpRy1YfN9Kci6jr1cMkARgvMfmlaofhFEUGCGM4oVRvOkJ\nPuj3LDM9lUAFxz9YRPzUR5g+tB/D73iS0K3/5dsTen/RKuBhHs3/L68fbSHOegkgwqeCxnSFjaeF\nURQYIYzihVDC/qVHCJ8/DHsAXQFRe/Jx7eWgf8Gse9LPKpWdieV11/dkwp32rFmfKodVG6Irjvj/\n9s48PqrqbPzfmclkDwlhzb6SkJ2w7/uigiKbgIpotdWqrdW+v7cVrbu2xbq29a2tWkVBFtlBUUB2\nCEkge0JCNggkISFkXyaZmfv7Y2ZuZkhCJpgQkPP9fPgwmbn33DN3nnue8zznOc/Df/+xn9JuGHOq\nT33KR/tLuZ6mhPtUcDXm44NQioLbgttB6LufKlKONREd5GT4U9/AlSawUZqUihIbmqis18pnuASH\nU3cwg9qrm2rK5MMHV1E+ZUK3FIZ2HTYb9zUP8afj1T+pHWEpCkBYioLbEGEpXgfaRmoaldjZGO+d\n0pF+9hLNWqMi0WvRSA64O5vV9FYraa5uuMqCayb7n4+zYcqH/C7Kvnv6pvLhvr8+TPITrxDXRW+t\ncJ8KrkasKQpuO4RSvA5s+uLvraPocpPhb+VAhs8cxOWiaoPSay4mRxPGXZF95FNayi9iE+iBnXk7\nVT/ywjt2PPPQEGy7sXvKwfN5fvRGfrfuIl35RYVSFFyNsBQFtx23w0yw++nH1Afd2H+83Kh07Bnx\n3F8Z88Of+ff3x9j5jw84c+/b/CLYZClqyD1USMzS4biatXLlx3fYG/IEswd192PpxLDlE0h/+xuK\nuvCTijVFwdXcDkrRpvNDBLcTwlLsGE3BVv78Xhy2Qd44qlxRFV8k6vd/YGo/G4Y8+jKe935FzrMv\nMNQGVD6L+deWKWSdPkPtvR+yLbhv68PWkMTn+0bzwpuDzVqvJ23LCdzu/JeFouwuXKLuYejZfxJX\n9Qx+7l0/X1iKArg9lKKwFAUWiECb9mnO+w/3zfkAx8deZtUzv2GF92aef/8wV4yPkE3Qr/h40UFe\nXnsOOZzGdgBhYycx2lwhoiHz4zcp/cPfubuv+RWqyTpZT3CoO90QX9MW52CGO5zh1EWN1acI96ng\nam6HSbNQigILbgeh7zK6Ijb89n84OvJ5fhXtCOhoKL+EMvwehruZDnJkxCubeLZhB3FVHTelLTnM\nfrfX+ezRAEs3TXMl5yttGeimtjyhKY2374xi5htJdB4n00Ta23cSNfMNkq4+2MYVD8ca8sva2wTS\nPsJ9Kria28FSFO5TgQVCKbZFd34H73+rYPaeMbgBUM3pndkMvms8HhZmnRtjf/2ba7Zl4zGL3zzS\nzgeSHr2kMNvGYbp4Deey0snyrLZir6GOmnNZpGd5Ut3mYAlJp0d/nQafsBQFYKkUJUlCkqSf3eRJ\nKEWBBUIptqWpNItSmxCeijJGj9Zl8u1pRyY+F2QZPfpTULvi6aIhqabF8n2nCfyjUOIfVjXixIR/\nFCK1d7C2hpImZ7z6qtv5sH2E+1RwNeZKEQzWoo3Nz0uN/Ly+jeAnI9YU22LvFUNg30yc1EpAS9H2\nd9hYGcl/Ipy77yJKV0KH2fJ+bjV63K65rtFcsJFVLx4h5rW/sSLISrXcdJ6MxkAe92l//2NaWhpa\nrdZCEWo0reuPZWVlpKamWnzet29fvL29rbu+4GeBUIqC247e25Khpa5Kg4ObEypAp9WhsumRkJMu\no/K9n3++HMdf336P2lA1RT/EUz90FSOuI4qzY1yJWRDD5Y1p1PzOz+imbZ/mCwfZuu4TfhjyGEte\nicGabf4NOfvJ8lnCuH7tf/7OO+/wxRdfdHj+mjVrWLNmjcV769atY/ny5VZc/XrQodWpuElEQGCk\nPaX4c0ME2ggs6BWlqMnnq0di8ennjJ17MCMnT2L6E5sp0nZ+6o3BieinPmXt6md57NHlxNTW43fX\nxKvWE386g+94ltHpm0mpu/ZxzpM+Iq/mIL/wtvbxbSJz/W4G/2YZQR30+cUXX2wz4F2LsLAwli5d\navXxXUFfFc/qmX1Rhz/O5vPWBwYJeh5zTxIIpSi4DegV96nCiWH/e5Dy5loKfvyYF36/mq//eR8+\nN4MfoymLz577DR+lGbLV6Ir38N9TPty/ONQqC61LDJrPW0vTeH9XaaeZZ6rTE1FE+lvXh6ofefu7\nybzzWHCH2z2Cg4N54IEHrO7qSy+91GaA7Baas3hv4WOcmLmR4y/Z8f69T7C74hZZ29YWseX1v3Gs\nyvi3roHqhtZnqCn1X7zy5RmaeqVz3cPtYCkiCQRmLFiwQAIkQLrzzjt7uzu9jq74K2mWe5D06FdH\npMNbP5SevvsO6Yk12VJTT12w8pD03OTl0roL2o6PqU+TPv9gp3S+xYr2dBXSnsfHSw9tLpGu0aIk\nSZKUk5MjqVQq+ffv6F94eLik0+m68q26hE5r1rZOJ/XclbqTJun0C+Ok+3dVSJIkSZUHn5cWP/E3\n6YtPX5VW/uJf0pkmSZKkOunYb0ZIjx2o7dWe/hQ2b95sIQsVFRU37uJ18dJfH7lbGh8dJkVMflba\nV9Ek5X56vzQyarQ0c8nz0g9l3SMpQikKLFi0aJEs8HPmzOnx69XF/1V65O7xUnRYhDT52X1SRVOu\n9On9I6Wo0TOlJc//IHWTnP8kdI2Xpbzk49KJlAKpUtPz19MUbpZe++teqbQzLWYFVXHvSqv+my7V\nW3n8gw8+2KlSXL9+/U/vWDvcCrLQIRXrpDEDnpNSWiRJ0l2Q/hnpK/0pUyNJUpX07UI/6f79NYbj\nCldLfkHvSed7s68/gW3btlnIQllZ2Y3tgK5SOvJ8tKRWx0p/OlYg7V81V1r+fxlWy7c1CKUosGDJ\nkiWywM+aNeuGXFNXeUR6PlotqWP/JB0r2C+tmrtc+r+M7hRzgbVkZ2df01qMiIjoWSvxFpWFqh0z\nJPv7DkmNkiRJtfulRTZ+0v+VSpIkNUpxjzlJPh9cMByozZT+x36o9NkN1iXdxc6dOy3koaSk5MZ3\nQlMgfbl4oIRygDTij4elym4WR7GmKLCgNzJWKN0m8sr2z5h/8XUmjf5f9H/4iF+FO96QawssCQkJ\nYdmyZR1+3mNriUZudlnQVhWScvww8TkVhgLRzVVcqtPRmJ9EU4AXagCphRZT2TAA9DQ3GAOGVG4M\ntsnhXBfLeN0s3BRrirbeTJo/jUDHclJ3f0t6TfeuOQulKLCgtzbv23pPYv60QBzLU9n9bTrdLOeC\nLvCnP/2pXcUXERHBkiVLevz6N6Ms6Cvj+ejxhTzwh//wY2ENZQlrePPVj3jnVw/yab4WdZ9+UFFl\nyHvrFML4AC2lVTpAw5XLfRgZadq/00xNnS0O3Vkb7AbS+9GneioOvMRTe2awI2kdi8r/wvxHviC/\nG4OUhVIUWNArSlFfwYGXnmLPjB0krVtE+V/m88gX+Yhg/N4hNDS0XWvx5Zdf7vmUXjehLNSeepfF\nM1+kZOFHrP34TZ69fx7zHniW56Yk8d4PAxkfaIfb+PtwP3yKagClLyv+PIPDH3zFge8+4b91j/Gn\nqcbaJ9WnOex6P1O7dY/rjeNqS/FGTpz15ft4cck0Rs/9OzUjJhIcfAcrZ3twZdvjzLjjQd482cle\nJmvpXm+s4FbnoYcektcLJk6c2MNX00lle1+QFk8OlBycJ0nvZjZJ0pXvpIc8kFD7S9MeeEOKu3UD\n9W5psrKyJKVSKctCZGSkpNfre/CKN6csaC9tlx70HCAt+Ppi2+jdso3Skvu/kcokSZK0BdLfx42T\n/lnYelTjxVTp+MlsqdLsxIpNc6WIVSk9F73cw/z4448Wa4q5ubm93aVuRyhFgQUPP/ywLPDjxo3r\n7e4IepHly5fLsrBx48be7k4vUC+d/L2/pAp9VUptT4s15ko/JpTJyrLpzEfSyl99IeV1sFVGe2mn\n9Jv735PSb1WNKEnSoUOHLJRidnZ2b3ep2xHuU4EFvZfmTXCzYVpbjIqKYvHixb3dnRtPQyprvy4k\n6MFFhLaXYtY+iGkjB8gJEexCf83Hr4xA1UHJSq0+iD98+jsiui2L/I3npgi06WFuhpwhgpsIUSVD\nYCIsLIwlS5awePHin115IKtoLCKlvA8xI71pGxejo670CjYDB2BvZlrYeUTg10FzdoPD8OqZnt4w\nhFIUdDs1NbVkZmVTX19/Uyodta0do0aPQalQ4O3tzd59B3q7S+2itlXj6TGYIcFBt+SArdfrOZub\nR0npJVqaWzo/oZe4Z/5C+ri637RyoFQqcXJyIiJ8KC4u3Vi1BMDGmf7O9jjZteNQa8hg46ZaFv5m\ngFmqPR1lJzawMcWNu39xF37XEWGq0+nIys6hvLwCbcvNJxeVlZU8+fRvUSiUKBUKsnPyKCkt7+1u\ntUGpUuLi4kJURBgODg5dOlchSaJQ2o2gvqGB/fsPkZF1RtSm60bc3fsybcokQkOCe7srVpOWnsGR\no3FUVVf3dld+NiiVSqKjIpg5fQq2tt2030FfxY9PjeLXDp+T+O4EXOT3Kzm55msqZ/2KO7ws7QpN\nxisETq5hd/G7DOuCm1SSJOITThMXn0B9/S26ifEmRK1WM2J4DFMmTbA64b1QijeAxsZG1n79DWXl\nhhmVg709gwcPQqUSS7rXgwTU1zdQWnoJMBTDXbzwHoYEB/Vux6wgIfE0e/cflP/2GDwIRydHbj1b\n9+ZAp9NTUlJKk7H2Y0CAH/cturdLFT+uhb46gQ+eWsVxvwd4bF4YtiXpJOVpCFnwMPOCLZMK6HU6\nqnbeScT6P5CzfkarEu0ESZLY9+MhEhJPy+95e3liZ28n5OI60Wp1XCwuocVobcdER3LXHbOs8ioJ\npdjDSJLEl+s2cOFCMQqFghnTpxAbE4VabaiA3tjURFp6JsXFJfI59vb2REeG4+Ex2KofUafTcTY3\nj5yzebJLVqlUEjIkmCHBgVYNEJIkUVxSSlp6Bk1NrZECXp4eREWGY29vXU2I6poaklPSqKyskt/r\n06cPw2Iice/b16o2NJpmMjKzOF90QX7P1taWyIgwfLy95HtyqayMHbv2UF5+GZVKxaOPrKB/v5t3\nA9iZ7LNs2bYTAF8fb+bMnsGA/oYCh5IkUXThIukZWTQ3t+7K8/XxJiI8DDs766yfK5WVJKekU1NT\nI7/Xt68bw2KicO3Tx6o2mowyedFCJu2IiozAs0symU/O2dxWmVQoCQkJYkhwkNUyWVJSSmp6Jk1N\nrbUlPI0y6WCUyebmFuLiEzh6LA6AkSNimT1zmlXf1Tq0VBWmkHSmBreIUUT7OFtUG9FXHeXNX3+C\nNDma82//nvQ3LxO3vIPCle1wPC6eg4eOAjA0dAgzpk3B1dXwW0mSREHBOTLPZKPVGmqpKRQKAgP8\nGBoaIo8jnVFWfpnU1HTq6uvl9wYM6E9MVCTOzk5WtVFf30BqWjqXylrdpU6OjkRHRzJo4ACr2tBq\ntZzJPktefoHsMVOpVIQNDSEwwN+qbEmSJHG+6AIZmWcsnhU/Xx8iwofKnoLGpiYOHDxCckoaAHNm\nTWfE8GGdtv/zUIq6eqqb7HF1uvkqkp4vusBX6zYCcPe8O4iKCAegurqGw0ePk5mV3eFi9aCBAxg7\nZhQR4UPb/Vyr1XLsxEmSU9I6dLk4OTkROyyK8WNHd1ghOz0ji7j4RMrK2l8bsLGxITwslEkTx3U4\nsBaXlHL02Aly8wra/RwgwN+XCePH4uvTfrX2uvp6jhw9QXpGljzDu5p+/dwZPXI4w2KiUCgU1NXX\n89nnX1FXV98Dg2FbTI9LV9cxJUniv1+spfRSGV6eHty/bDFqtRpJkkhJTSc+4RSXK660e65arSYy\nIoxJE8fh7NT+AHa+6ALHjsdRUHi+wz4EBQUwafxYPD092v28pqZWlknTAHw1AwcMYMyYEbIcX41W\nq+V4XDxJyWnUmw3A5jg5OTIsJooJ48Z0KJMZmWeIO5lgMQCbo1KpCA8LZfLE8bIC2X/gMCfjE7Gx\nseGZp5+weiLx06jjwEMB/M+EZBIeqeW1IfNxO5zF7/ys8wK1tLTw94/+Q1NTE2FDQ5h/910olUr0\nej0JiUkknk6iurqm3XPt7eyIjopg4sRx2Nu176vNzcvneFw8Fy4Ut/u5QqEgNGQIkyaMZcCA/u0e\nU1FxhSPHTnAm+2yHcRBenh6MGzuakCHte2s0mmaOHj9BamoGjU3tF8/q08eFEbHDGDN6RLvKUZIk\nkpJTSUg8TcWVynbbsLU1PisTxuPk5IgkSWzeuoOcs3m4u/fl8cce7vTZ7QGlqKeh/BLNrh649bhM\naina+Te+yHOnf81RNiSG8fba5xlprd+iCyQmJhIUFERfK60dE1u37yLrTA6DBw/ikYfuR6FQUHqp\njA2btsiKTK22ISgwEDs7W9liu3y5Qm5jzOgRTJ862eLHbGxqYtPmbRbC7u/nKw8Q1dXVFJ4rkj/z\n8fZiyaL5FhafJEns//EQ8WZumwH9+8kWqkajITevQB4gnZycWLZkAYMGDbT4jmeyc9i+8ztZuTs5\nOhLg74fKRoVOp6fw3Hnq6gzZJhQKBXPvnE10VIRFGxUVV1i/cQvVRgtHpVIRFOiPg4MDkiRRVlZO\n6aUy+fiY6EjunDMTpVIpz7Tt7ez47dOPdzjQdge1tbVMnjyZl156iXvvvddq5XjxYjFffLUegBUP\nLMXH2wu9Xs933+8jJTVdPm7QoIEMGjgAhUJBY2MjefmF8n117dOHZfctpN9V1nBqWga7v/tBVtjO\nzs74+/miUinRaXUUFJ6jvsEgayqVivl338nQ0BCLNi6VlbF+41ZZkdnY2BAcFICdnZ1ssZWbyeSo\nkcOZOX2Kxfdvamrimy07LCx8fz8fXF0N2Vyqq2soPNeqtL29PVmycL5FIIQkSfx48Agn4xPl9/r3\n7ydbqBpNM3n5+bS0GGXS0ZGlSxYwePAgGpua+PAfH6PT6bjrjlkMi4my6rf5STRn8LznLJziiljl\n+B+Gj87lH9+vZMiQKAZZMf4lJafy3ff7UCqVPP3rx3B2dkar1bJtx7fknM2Vj/Py8qCfu7s8EczP\nL5R/7wH9+7F0yUL69LEc+OLiE/nxwGH5bzdXV3x8vFAqlbS0tJCfXyi7nW1tbVm88B78/Xwt2jhf\ndIFNm7ejMR5nZ2dLUGAAarUavV7PhYvFFl6hqZMnMH7cGIs26urqWL9pqzzpVigUBAT44eLsjCRJ\nVFZWUXThonx8cFAAC+bPs7CC9Xo9u7/7gbT0TPm9wYMGMrCDZ8XN1ZVl9y3E3b0vJaWX+O8XawF4\neMXyDieFJq5j9NDTcKmElv5euF5lmDVl/J1l837P9rLprM39lvs9lICGnA0fcir2tywP6eYNOvpi\ndq/eCv86zBMR9+AwOpI3jjzJttn1fPfxPrxWPkR0NwWkbd68mY8++oinn36a5557jn79rHOPlJdf\nBiB8aAgKhYIrlZV8veEbGhubsLOzZcqkCURFhmNnNtOTJImLF0s4fPQYheeKOBl/CrWNmsmTxgOG\n2bi5Qhw1IpZRo4bjZhx8TFRVVROfcIrE08kUXbjIN1t2sOy+hbLSOHTkmKwQA/x9mTRxPF6eHpYD\nnUZDWnomhw4fo76+nq83bGblQ8vp6+YGQH7BObZu340kSbi5ujJ1ykRCQ4Lb7HfMzSvgwKEjVFRc\nYde332Nrq5YH5traOtZt+Iba2jpsbGyYPHEcMdGRbaLGSkovcfRYHGdz80hJTUelUjFn1nSGBAdx\n8NBRmjQa6usb5IlBT5GcnMzChQsZNmyY1crRZPE4Ozvj7eWJJEn8sO+ArBCHBAcycfxYPDwGW5zX\n2NhISmo6h4+eoLqmhnUbvuHhFffLkZZnss+y69vvAYMVPW3KRIKDAtsUi87OyeXgoaNUVVezdftu\nli6xIzDAsHmgsqqKrzdspqGhEVtbNVMmTzS4zK+WyeISjhw9TkHheRIST6O2sWHqlImAQSbNFeLI\n4cMYNWq4LCcmqqqrSUhMIiHxNBcuFLNpy3buX7pYlskjR0/ICtHfz4dJE8fj7eVpcX81Jpk8coz6\nhga+3riZlSuW4963Lz7enhSeK7KYVPYotr7c9UAon+z4in9d3kmV8wC2bSri/71knUI2yYW/ny/O\nRiWxY9ceWSFGRoQxbuxo2c1uoq6untNJKRw7cZLyyxWs37SFhx5YKk96TyelyArRY/AgpkyeSIC/\nr8V9bGlpIetMDgcOHaG+voGN32xjxf33yTJ46VIZGzZtpaWlBUcHB6ZOmUhE+FALZSVJEoXnijh0\n5BjFxSUcPHwMtVrNqJHDAcNvtX7jVjmeYvzY0YwYPqxNpPDliivEnUwgNS2D3LwCtu34lkUL7kap\nVCJJEnt+2C8rxJAhQUwcP5bBgwdZtNHQ2EhKShqHj56gqrpaflYGDxqIo6MDDQ2NXK640qlS7HKk\nh750A/NDx/NSfP3VH7D3iwKm/m0vqWc3s8zD0HTVj//DUyfHcKdJITYXse/fH/HJ5//hw/e+4lTl\nT9iWoPTlVweOsSrCDurySbjsy2h/B7DxZPqUMl5+ehMl3bTrQalUUlNTw1tvvYW/vz9/+MMfKC/v\nPBTZNKM1CevOXXtobGzC3t6eFQ8sY+SIWAuFCIaZlLe3J8vuW0RkRBgAR4/HyUrw+IlWd8i8u+Yw\na+a0NgoRwM3NldmzpjP3ztmAYdZ34mSC/Pr4iXgAoiLDWbpkYZvBBwwumlEjYlnxwFLs7exoaGxk\n5+49xu/Wwrbtu5AkiQH9+7HyoeWEh4W2WS8yrG8GsXLFcjyND9yOXXtoaGwE4Lvv91FbW4dabcMD\nyxYzdsyodsOoPQYPYvHCexg5IhYwPPi5efkWg3dHbteewKQchw8fztatW68ZVdxitLYd7O1QKBTk\n5hVwOikFMCiQxQvnt1GIAA4ODowdM4oHli1GrbahtraO777fCxgU5o5d3wHg4TGYlQ8uI2RIcBvX\nk8nVuPKh5QwY0B9Jkti2fZd8r3bt/p6Ghkbs7exY8cAyRo2IbeOOUygUeHt5snTJQqIiDa7T43Hx\nshKMO5kov55752xmz5reRiGCYQY/a8ZU7p47B4ALF4o5duKk4fXFYo4eN6wLRoaHsey+RRZryCbs\n7OwYOSKWFQ8sw8HensbGJnbs2mP8zN7ifvc8Lkz6YB//fHghj721m7zE93ntpbsYZOXIavLC2Nsb\n7ndKajpnsnMAmDJpAvfMu7ONQgRwdnZi8qTx3LfIMCG7fLmC/UYlWFlVxfd7fwQgMMCfB++/j8AA\nvzb3Ua1WEx0VwcMr7qdPHxe0Wi1btu1Cr9cjSRJbd+ympaUFZ2dnVj60nGFmsRAmFAoFAf6+PLh8\nCcFBgQDs3X9QnpQcOHSUsvJyYzDcfKZOmdju1pn+/dyZd9ccpk+dDMDZ3DySklMByDmbJ68Ljh45\nnEUL7mmjEAEcHRwYN3Y09y8zTLJqamrZ88M+FAqFPMZaIxddVoqNhacp1V5g2/pUzNWivjyBk36P\n8MSiKUR5Ohkabkjg1acKeOKFybgB0EDCK0v5u93dPPLwL3lsag6/ffgzCn+C/CptbFBqi9nx0ruo\n39nO/4YbfBZ2kU/wO9WLPPtD1fU3bn4ds4Gmrq6O1atX4+/vz+9//3tKS0s7PM8kiJIkUXqpTA5e\nuGfeHQzswIdvfs25d86WH4rTySnodDqSUgzCMmpEbBs3ZHvEREfKC8xJyWno9Xp5QB44oD933TGr\n0wXuQQMHMG/uHYBhICsrLyczK5smjQalUsmSRffi5HjtEj/2dnYsXjQftVqNVqslLT2T6uoacvPy\nAZgzawZeXp7XbEOhUDBrxlS8vQ3HnUpKobe3KZorx23btrWrHE33V2/87FRSMmBwIc6aOa1TS9PL\ny5M7Zs8AIDevgOrqGlLTM9FqtajV6jau8fZwcnRkyaL5KJVKmjQaMrOyKSsvl11X8+bO6TRgQqlU\nctcdsxg4wHDc6aQUgzwZB7ARsTHEREdesw2AqMgIRhknN8kpaeh0Olkm+/fvx9y7ZncqkwMH9Ofu\neQaZLC4uofRSWS/JggpndxdsAZWzG45dGFVNE0iTIjp1OhmA0JBgxo8b3en5QUEBTJ1ssNYzMs/Q\n1NREUnIakiTh5OTUxg3ZHq6ufVi84B7AECiXX1BI4bkirhjX7RYtuLvdCY45NjY2LJg/V1Z4SSlp\naDTNsnU3aeK4DtcbzRk7ZiThYaGAQbYkSeK08Vnx9fFmxlUu+/bw8fZizqzpgEGh1tTUdimKt8tK\nsa5Ez4LnJlC1bT2pZlqxMjkR97FBmD+W1YfeZF34M8wxTXSqj/H2pxrmzvRCBTiG3kFM8jt8mfMT\ncuBrS9j3wceULPsPfx2Tx64MU6ecGfvrWRx8aTuXrr91mfYe0IaGBt59910CAwN55plnKC5uu5it\nUBp+Dr1xkRgMe+uCAgOsuq5KpZIto6wzOaRnZslrkaNGDbe6/6ONx9bV1ZGekcWZ7LOGNkYOtzp8\nfUhwIH37ugGQlJQqD4RDQ4fg5tbWUm0PZycn2fpNSkqR74mTk2OHAUVXo1AoGD1yBAD5+YXyOmRv\nk5yczIIFC2TlaI6sFPV6qqqryc8vBGD0yBFWr0uGhw3Fyckw8Tid3HrvIiPCOgzAuRo3V1eGhg4x\ntpFKUpKhjb5ublZvaVGpVIwaaZDJM9lnSc/IkteMR48aYVUb0Cq/9fUNpGdmkXXGYCGNGhFrtUwG\nBQbg7m5Y5zfdj1sJ0/fUanWUlF6S3aljRo20Wi5iY6OxsbFBq9WSkpZBqtElP3xYtNXBRoMHD8LP\n1weA00mpJCUbJijeXp54deJuNKFWqxkRGwMY1rlT09JpaWlBpVLJ71vDGKMMlV+uIPNMthw8NnqU\n9c9KZEQYjkZvU3JqmtXXhi6vKdaRl+XC1Ace5/iHq1if+hbjxjkB1WQm2DHmGXNLoYG09d/Tf+6/\nMRnL2tJTJNe6sczBqGBs+uJle55DOfUQfo0fT3OO3R+u5j9bj5FXo0UCFH0m8Netf8b25bHM+ryO\nAS7/5H+bPfl/xxO513iaXfBsItM/4ETVSu51a9tsRkYGtbW1tLS0oNVq5f/be52UlNRh9xobG/nw\nww/5+OOPefTRR/njH/+Ij49BwJQmS1Gvl9cJYqIiuxS9GBE+lB/2HUCn05GSYhB4fz/fdl2mHdHX\nzQ0/Xx/OnS8iOcVgLdrY2BA2NNTqNhQKBTFRERw8fIys7BwaGgzuz+iozi0Dc4ZFR5KUnMqVyip5\nIIyMCO/S3rIhwYE4OBhcZwUF5+T3X3/9dXJzz6LT6eR/er2+3dfX+uxa53SGSTnGxsbKa44qM6V4\nNtdgGTvY2zMkONDq76xSqYiKDCfuZCJZZ3KoqjJs/rfGMjMnJjqSzKxsSkpK5a0b0dERXZLJ8LBQ\nvt/7I1qtVnZt+fp4y5Mma3BzdSXA35eCwvOytWhw9Vo3OQKDTA6LjuTHg0fIzjnbYWTzzYppr7JO\nryMnxzA+uLv3xcvLOkUEBg/M0NAQ0jMySU/PlJcluioX0dERnDtfRG5eviwLXW8jkoOHj6HRaEjP\nyAIMVm9Xssp4eAxmwID+lJdflidtjo4OBAdZZ0iA4VmJjAwjPuG0fF+tpWtKUXOO1IZY7g0awy9G\nV/PHtcm8OW4Czg0FnNYM436L4Kca0g62EPvLVv+xtq6CBtSoTM+eUo2tooGKmmusA9Wf4rXZ93F4\nyiv84bURfHHfoxSvTuCfM/0IHOSO3b/OIf2rg3MdA4jSxZN0Wc+9bm0tvRUrVlxT2XUVjUbDRx99\nxCeffMLKlStZtWoVCjO3mV5nWODsaiCIra0tjo4O1NbWodNfXxvm55jCqh0dHbC1tW6fU2sbBkVs\n+i4Abm5d64urmVWp0xujxay0NE2oVCr6uLjQ2Ngk3xOAxMQEjh8/3qW2eoqkpCRZOb7wwp8Aw703\n3TsXF5cubzI3TYTMQ+O7Kgvmk6nrlUm1Wo2TkyPV1TVyX65PJi2/j6OjQ5e3UrQnkwB/Xv0eZeWX\n8fP14Xe/eQKATVu2y2vpf33zZWxtbUlKTmXN2g0A/PLRhwgfGkptbR0vvfZnAKZPncTdxqWDv/zt\nfS5dKsfXx5tnf/trADZv3cHR44Z10b+88RJ2dnYkp6TJ0ca//MVDhIeFUltXx0uvGtqcOmUi8+fd\niUpp+P11Wp3Fs93VLT+mZ9D0PAFtolE7bcNMLkxLAF39TZ2dnGSr9aeMVW6ufSgvvyy30cfFxao9\njJZtGL6ProNtJB3RJaWoLUmjbPgkBtgMZM5TM3nm8Y848voEZl1JpnjoNCwCxfWNlFcoGOjWegmF\n2g4b9LQuuejQSSrU6o4EQEP66uX8M3IN+W9NwIkGnBc9w+KyvoT4D6DT4cTWHS/HOs5caQHaRr5a\nu/G1qzQ3N/Pll1/i6urKkFDDmp+kv/W3g94KKJU3117VUaNG8fLLL+PrH0B2bsFNme/258q8u+bQ\n1KTBybnVgzVqRCz+voZtBypjxKuvrw/3LzVUAfEcbAh2sre3k98z34I0947ZhjadWtscMTwWX6Nn\nyBRF6+vj3dqmMYDK3q5tmyobo1LU//wSa9+qdEEp6rgYl8WQsYtRoaTf9N+y0GY+H+wuIsYlG5/o\nZZZKSmmDg1rLhaZWZaDu642r9gfqTIE12gauNLvg27+DmaEmhy//Xsz8b2NxAtBXkh5XR+Ccvp0r\nRAB9M/VaNU727SvdntjPZmtryy9/+UtWrVqFp6cnn/73SwAkSY/S6CrpaDNuRzQ3N8uuSpMbruY6\n1tFM1zXNuBoaGmlpaenS5KDamK9TaZairrq6xupsNQDVVa05P00z5eou5gHV6XTU1NYa22jty5gx\no+nXzx2VSiX/UyqVHf5t7Wem1y0tLaxatarT/pmU4dy5cwFDEAQYLCLTvautrZVdhtZiypdqPmuu\nrq7pNMipvTag9Xes6aJMtrS0yGvbSlkma7vUBpjJk5lMNjc3dyl/aXsyCchRsub4+/m22YvXz70v\n/dwt5dewrSC2zfntt+mDv5+PxXvu7n3ltc5rtWn67XU6vSzH1dU1SJLUJWvR9GyrzCaFNTW1XbIW\nzeVCoVAgSVKXx6q6+no5otb8+3SVKvn7GGWrttbw7HTBWjR9H5VSibYLk1HrtUJzAQfTQ5i82HiK\nyxh++2QAo156ja+X+DJ2+tWRb84ERtiyr6QJk5WmHDiBe0PeJ6tEAwPtoD6fNN1oVka7QFMBe78r\nJuruCQyWe6VHqx5EhJfhgddkfcoHzb/nv3dZmcqrqYTCFl/GdrCL9osvvkCj0WBjY2PxT61WW/xv\nY2PD22+/zQsvvNDhpdRqNY888ggvvviivJ4IWLhPhwQHkZySRkpaOmPHWL+QnpF5Rh48Y6IjuXCx\nmILC81RX11jtmqisquLcecNm/piYKIpLStFqtWRmZVu9biBJEilpGQAMDQ2hpKSUktJLpKSmE+Df\nUcGctqSkGdZF3fu6ERoawom4eNLSM7uUtDc3L5/GRkNmjAB/Pw4fPQHAH//4PP3bCWHvLmpra6+p\nFK9WhiZaA20khgQFsnffARqbmjibmy8HvnSGTqeTo/nCQkPIzjnLlcoqUlLTZWvEGkx7Iz0GD8LD\nYzCnk1JISctg3NjRVsukKeuNUqlkWEwUFy4Wc+58EZVVVZ1GKpqorq6RgyhioqIoLS1Dp9ORkXmG\n2GHRVrVhyggEEDIk2CId3K1Aq1LUETIkiONx8Vy5UsnFiyVyhHVnNGk08jaOyIgwampraWxsIiU1\nnUkTx1ndl9RUw7MdHBSASmVDds5ZUlLTu7SuaArysbOzIzIijJLSS2Tn5NLY2Gj1umJJSam8vzs2\nNprzRRdoaGgkN6/AqghWMNxP05pmyJBgMrPOWP0dOle7tYd4angssxY/S+K4WfjJCsuOiCffZWXL\nJ7x1NoqhbSYkroy8ZyCJcSXIOy7sIvjtB/eQ8v6nHMtNZ+u7X+L6/PssGKSkOe9rfrdwHk+bb6Gw\nC+PJN8fw/V8/YuOat/jNa5W8+O2bjLJ2UlyZwEHnBYzvwIgJDg4mIiKC0NBQgoKC8PPzw8vLi4ED\nB+Lu7o6LiwsODg6o1eoOB2sbGxseeeQRsrOz+fjjjy0UIpgH2kgMN0ZgXblSSV5+x+nQzNHpdCQa\nw7TDQkOIjAiTXTfmmWg6IyHBcKyzsxNREWHyQJyQmGR1TbTcvHw5e0XssGj5+5zJPmt1xYf6+gbS\n0rOMbcQQOyxKft9awTVVFADDPqye3qxvDaNGjWLXrl3Ex8e3UYjQGlCh1+txc3MlMMAfMCQItzap\nlKHkmME6i42NJnaY4f6nZ2RZXVmhurpGjjweHhsjK5/Kyio5AKgz9Ho9CacMa/FDQ4cYol+N+TNN\nv4s1mBJgOzk5EhUZRpgxmUPiKetlMj+/UE75NbwLEY43C+ZK0cNjsLwl5mTCKavlIjk5jZYWLSqV\niujoSDnwLSkl1SI36LUovVQmT5qHx8YwPNYgFxcuFlvkZb4WLS0tckR6dGQ40VGRqNVq43Yb6yOD\nTyacAgxbc8KHhsqWfXwX7kl6RpbsXetqZqPOlaKtD9PmjcB/7G95Yc5AyxPcJrN6/0kOf3I3befm\nSnyW/j+81n9N644LJf1mrGbbOzMgr5zAp9az7qkw7ADbiFWknv+cCY7mX9qWoMfWsfONeYyc8xz/\n2vAey4KszYqj5/z2L7F96hFCuyHd3NVmu1KpZMWKFWRlZfHZZ58RENB+ZJS8JUOvZ/CggfKMfqcx\nkfU1v4Fez+7v9srHDY+NQaVSMSza8CMnJJ4mLT2j076npmXIijU2JhqlUikPIGXl5Xz3/b5O17rK\nysvZuduQOcXby5NBAwcQNjQUezs79Ho9mzZvl4WwI5o0GjZt2U5LSws2NjZERYXj5upKkDGqbM8P\n+y2SULeHqaKAaW/d8NgYrHxOeoTOlKEJ8y0Z0DqAF124yL79Bzt92C8Wl7Dnh32AYW+am6srUVHh\n2NjY0NLSwqbN2+SUXR3R0NDIps3b0Ov12NnZETY0lEEDB+Bt3Bu6a/ceOfNIR+j1er7ds1dO2TU8\nNgalUklsjGEQPXU6mdS0zmUyLSNTntQNi45CpVLJ96T8cgW7v9vbqUyWX66Qkxd4egxm8KCBvSoL\n14ONqjXQRqFQyPcgO+csJ+LiOz0/P7+Qg4cNycQjwofiYG8vTzTr6urZsn1Xh3lsTVTX1PDNlu2A\nIZVgYIA//n6+uBsjiTdv2ylHOneEITXdbtmFPsy4HcS0/erw0eOczc3r9PvExSeSmZUNGGTL/J6c\nL7rA/gOHOn1WLlwslpMXDAkOpE8fF7oiFp0rRbtAFr/2Cf9ZNcvMrdl6ukvIaCLc27eilH6/4B9z\ndvPuoSqL923cQ5kwZxoxnvZmHWjizMHzDItoG4WocvMlcJB91zZVNpzm/74M473fhbdTNbvrmAY1\npVLJsmXLyMjIYM2aNQQHX7uOn61ZNQyAe+bdacjC0dTEmrXrSTydjEZjOZuTJIkLF4vZsGkL6RkG\nd9mEcWNkd8qE8WPkgWzn7u/Zu/9gu377qupq9u47IKcB8/H2YtzYUYAhEGDcWMPm4NS0DDZs2srF\ni8VtBE6j0ZB4Kok1X22gqakJRwcHORLP1lbNvffMRaFQUF5+mc/XrCPrTE6bWb6poO6aL7+WZ513\nz71D3kd015xZuLg409KiZe3Xm4g7mUhjY1sFW3qpjM1bd8gWxvDYaIYEB1oog54KnrqakSNHsnPn\nzk6VoQlTv3Q6Hc3NLQwJDpRn4wmnkvhmyw65FJY5jY2NxJ1MZO3Xm2hp0eLi4sxdc2YBhgwe95g2\nr5eUsubLry0qpZjQ6XRkncnh8y/XUVZ+GYVCwYJ75sqRx6bfokmjYc1XG0g8lSTnujRhSD1YzIZv\ntspKb9zYUfIWCPPXu779nh/2HWjXe1BdXcO+/QfZacxA4+3lyYTxhlyZ3t6eTDDmzUzPyGT9xi1c\nuNCeTDaTeDqZNV99TWNTEw729twz707jZ4Z+q3sw/2130lrRoRFJkhgWE0VoiMGLc/DwMXbs2tNu\nyrq6+noOHz3Ohm+2otfr6d/PnRnTpwDg3rcvc4yJHvLzC/ly3UYKCs+1uY8tLS2kpmXw+Zp11NTU\nGjbg3zsPpVKJQqHgXuPG/9raOj7/ch0pqeltMkYZ0rydZ+3Xm2RPw8wZU+WEI9OnTmKgMYvSN1t2\ncPDwUXlPqzkVFVfY/d0Pcmq6IcGBDDd6MUJDgmVrLz7hNFu27eSSWR5kE42NjZyIS2Dt15vQarX0\n6ePCnXNmIkkSzcYx1hq56PkqGXXxvP7wWoZ/9B5zB15DrTUXkZjjRGyku3VBNNekluOv/ppvp/yd\nN6Z2LYF3R7z//vscPXqUV155hchI633s+/YfJD7xNH36uPDk44+iVCopLb3Ehk1b5STNarXaIvly\nsZlPHQybWadPuyoheGMjmzZv58LF1oQBAf6+cnh6VVW1RfJlH28vFi+aL5fbgfbruA0c0P+qhOBm\nyZedHFm6ZCGDr0oInnUmhx27zBKCOzkSGOCPSqVCp9NReO48tbWtCcHvumNWm3WKyxVX2HBVQvDg\noAB5HaL0UpmF0oiOipAz8dzIhOAajYa9e/cyb968Lp1XV1fP3z/6N5IkydVSTFaXuWU1eNBAOYVV\nY2MTuXn58n3t08eFZfctalMe6+qE4C4upoTghvufX1Aou1dVKhX3zLuTsKFXJQS/VMb6q5LUBwcF\nWiQELzOTyVEjYpk5Y2qbhOCbNm+3SO7s7+crb7eprq62qOLh7eXJkkXtJAQ/cFh2oYEh4bWnMSev\nKUm9aXB2cnTkviUL8OithOA/kfLyy/znszUALF+6iAB/P1paWti+81tyzrZaVt7envQ35luuq6u3\nKL3Uv38/lrWXEPxkIj8ebE0I3tfNTU4Irm3RkptfIK/B2tqqWbRgPgH+lkFI584X8c2WHWYJwe0I\nDgqQK7wUFV3gillC8CmTJ8gTGxN1dXUW+U8VCgWBgf64OBu261VUXLGQmaCgABbcM89iu1h7CcE9\nBg+So3gbGw1rjnLyfNc+LLtvEf3c+1JaeonPjAnBV65Y3mkyghtTOkpTRkGdOwH9btDsTVvF+Qpb\nfAdZH5HXGV2N0jRxueIK//7kcwBmz5rOSGO6tarqao4cPXHN0lEDBwxg7JiRsgviarRaLceOG0tH\nNXRUOqrzMj1p6ZmcjE+0GPTMkcv0TBrfYemoi8UlHD0eR941Skf5+/kyYfwYOXPG1dTV1XPk2AnS\nMzJlRXw1/dz7MmrkcGKHRaNQKKhvaODT/35FXV0dI4YPk9M73Yxs3rqD7Jxc+vVzZ+WK5dgbFU5y\nShrxiaep6LB0lA0R4WFMnji+w9p3584Xcez4SYuJ0NUEBQYwccLYDgeF6poaDh+5djmzgQP6M2b0\nCKIi208vqNVqOX4inqSU1I7LmTk6EhMTycTxY69dzuxkYofuXEMNvlAmTxon70c7cOgoJ+LijaWj\nHm+TU/hmZc1X67lwsRgvTw8eWL4EGxsb9Ho98QmnOXU6ucOMTXZ2dkRHhjNp4rgOU/zl5uUbciVf\nbJttC0ylo4KZOGFch2knL1dc4cjR42Tn5Hbo0vb09GD82FGEDGnfc6bRNHP02AlS0jI6DIYylI6K\nYczokR2WjjqdnEpCwikLRWyOqcza5ImtpaO2bt/FmeyzuPd14/FfPtIbpaMEV7Pxm61yncFZM6bK\na4NgmOG0LehqT3RU1wq65pzNsyzoqlQSGhLcpYKuxcUlbQq6enl5WhR07Yyq6uo2RYZdXfswLDqq\nTYh6R2g0GjIyz8gL/9BaZNjXx1u+JxUVV9i281suXSpDpVLxi5UPdFgT7mag6MJFvjRuEvfy9ODu\nuXfI98RUZDgtPbPdwqnWDvBXrlSSnJpm4U43FRm2NgOSqfD1RbOB1FT42vOqKiodYSp8bT6Qmgpf\nhwzpgkyWlJJ61UBqKnxtsjC1Wi3xiaflQr3DY2PkPLG3Atk5uWzeugOAwEB/7pozS7b6JEkiv6DQ\nosalociwP2FDu1JkuJyUlLZFhodFR+LsbF0pobr6elJT2xYZjomObFNOriO0Wi1Z2Tnk5ZkXGbYh\nPKzrRYavLshteFZaC3I3Nzdz8PAxEo1BYbNmTpPz7V4LoRRvABqNhrVfb5LrATo5OeHt5dHlbCYC\nA5IkUV/fYFG3b+G989rUCLwZSUlNZ/d3P8h/+/p44+Tk2OUMJgIDWq2Oi8XFslXq5+vD0iULetSF\n3hOciIvngFGpKxQK/Hx9cHCwF3JxnbS0aCm6cFGeTEVFhjPvrjlW3U+hFG8QDQ2NfL93v5zrU9A9\n9OnjwvSpk+XM+rcCaekZHDx8TF5nFfx0FAoFEWFDmT17eodV6G92Ek4lcex4XKdR3ALrUalUxMZE\nMWP6FKuNEKEUbzCVVVWkp2dRV19/06b8qqurw87OFrW6O+J2ewZbW1s8PAYRFhpyS1rcpojQktJL\nVu8lE7RFqVTi7OREVGR4l/Pn3oy0tLSQkXmGsvLLN7Q2aFdpbGjAoQtZlG40KpWSPi4uREdFdrgO\n3xFCKQra8MILL+Dl5cWTTz7Z210RCAQ3GXFxcbz66qt89913vd2VHkEoRYEFly9fJiAggL59+5Kb\nm9ulHJQCgeDnz8yZM9m/fz+JiYmMGGF9/cxbhS4XGRb8vFm9ejV1dXUUFRXxxRdf9HZ3BALBTcTB\ngwfZv38/AG+88UYv96ZnEJaiQObSpUsEBgbSYNzzGBAQQE5Ozi0XyScQCHqGSZMmcfRoa5Rsampq\nl5KZ3AoIS1Eg85e//EVWiAAFBQWsXbu2F3skEAhuFvbs2SMrRDBsjXrrrbd6sUc9g7AUBQAUFxcT\nFBTUJttESEgIWVlZXa56LRAIfl6MHDmSU6dOWbynUqnIyspiyBDryp/dCoiRTgDAW2+91W76pZyc\nHNavX98LPRIIBDcLW7dubaMQwbC16M9//nMv9KjnEJaigKKiIoKDgzvcLxceHk56errIriEQ3Ibo\n9XpiYmJIT09v93O1Ws3Zs2fx87O+0PjNjLAUBbzxxhvX3ECemZnJ5s2bb2CPBALBzcKGDRs6VIhg\nSDiwevXqG9ijnkVYirc5BQUFhIaGdpo9IyYmhqSkJGEtCgS3ETqdjvDwcHJyrp2e0t7envz8fDw8\nrl2W6VZAWIq3Oa+//rpV6aRSUlLYuXPnDeiRQCC4WVizZk2nChEMtTT/9re/3YAe9TzCUryNyc3N\nZejQoR3WzruakSNHkpCQ0MO9EggENwMtLS2EhIRQWFho1fFOTk4UFhbSv//NW77NGoSleBvz6quv\nWq0QARITE3+2+Q4FAoEln3zyidUKEaC+vp7333+/x/pzoxCW4m1MZmYmLS0t6PV6+d/SpUspKDAU\nRF6yZAlPPvmkxedeXl5ERLRfdV0gEPx8KC4upra2Fq1Wi06nQ6fT8dZbb/HNN98AEBwczMcffyx/\nptPpcHR0ZNq0ab3c85+GyN91GxMeHt7mPfMq3B4eHkydOvUG9kggENwseHp6tnlv8ODB8msnJyem\nT59+I7t0QxDuU4EF5plruuJaFQgEP3/Mo89v1nqwPxWhFAUWmBfs/bkKvUAguD7MJ80/15U3oRQF\nFpgLvVCKAoHAHGEpCm47hPtUIBB0xO0waRZKUWCBcJ8KBIKOMLcUhftUcFtwO8wEBQLB9XE7jA9C\nKQosEO5TgUDQEcJSFNx23A4zQYFAcH3cDuODUIoCC8SaokAg6IjbwVIUad4EFly4cAGNRoNKpcLZ\n2fmWT+4rEAi6j5ycHAoKClAqlTg6OjJhwoTe7lK3I5SiQCAQCARGhPtUIBAIBAIjQikKBAKBQGBE\nKEWBQCAQCIwIpSgQCAQCgRGhFAUCgUAgMCKUokAgEAgERoRSFEB9Aqt/cQ8TYsKJnPIc+69oyPvs\nAUZFj2HWfavYWy428QsEtzW30Rgh9ikKDOirOPriFKb/TcUfD25h6u6n+cRnNZ88EY5jb/dNIBD0\nPrfJGCGUoqCV5kK+emAMK7ZIjPjfzex7cxJuwpcgEAhM3AZjxM/s6wh+ErbeTJo/jUDHclJ3f0t6\nzc/HJSIQCLqB22CMEEpRYERPxYGXeGrPDHYkrWNR+V+Y/8gX5Df3dr8EAsHNwe0xRgilKEBfvo8X\nl0xj9Ny/UzNiIsHBd7BytgdXtj3OjDse5M2Tdb3dRYFA0IvcTmOEWFMUCAQCgcCIsBQFAoFAIDAi\nlKJAIBAIBEaEUhQIBAKBwIhQigKBQCAQGBFKUSAQCAQCI0IpCgQCgUBg5P8Dxxn+0UMlxz8AAAAA\nSUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Image.open('./ladder_net_arch.png') # the ladder net architecture diagram from [1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ladder net building blocks:\n", + " * dirty encoder - the most left, built first\n", + " * clean encoder - the most right, built second, sharing weights and BN params of the dirty one \n", + " * decoder - the middle one, built on top of the dirty encoder, sharing its transposed weights\n", + " * batch normalization divided between normalization part (mean, inv_std) and learning part (beta, gamma)\n", + " * Gaussian noise injection in between the batchnorm parts in the dirty encode \n", + " * ReLUs (no gamma needed to learn in BN) used in encoder's latent layers, softmax as a classifier\n", + " * denoising (combinator) function g used as a nonlinearity in the decoder (instance of Merge layer):\n", + " 1. used in Curious AI paper:\n", + "$$ \\hat{z} = g(\\tilde{z}, u) = (\\tilde{z} - \\mu(u)) \\odot \\sigma(u) + \\mu(u) $$\n", + "$$ \\mu(u) = w_0 \\odot \\text{sigmoid}(w_2 \\odot u + b_0) + w_3 \\odot u + b_1 $$\n", + "$$ \\sigma(u) = w_4 \\odot \\text{sigmoid}(w_5 \\odot u + b_2) + w_6 \\odot u + b_3 $$\n", + " 2. used in MILA UDEM paper:\n", + "$$ g(\\tilde{z}(l), u(l+1)) = b_0 + w_0^z \\odot \\tilde{z}(l) + w_0^u \\odot u(l+1) + w_0^{zu} \\odot \\tilde{z}(l) \\odot u(l+1) + \\\\\n", + "w^\\sigma \\odot \\text{sigmoid}(b_1 + w_1^z \\odot \\tilde{z}(l) + w_1^u \\odot u(l+1) + w_1^{zu} \\odot \\tilde{z}(l) \\cdot u(l+1)) $$ \n", + "$$ \\mu(u) = w_0 \\odot \\text{sigmoid}(w_2 \\odot u + b_0) + w_3 \\odot u + b_1 $$\n", + "$$ \\sigma(u) = w_4 \\odot \\text{sigmoid}(w_5 \\odot u + b_2) + w_6 \\odot u + b_3 $$\n", + "Cost terms:\n", + " * classification cost\n", + " * reconstruction cost(s)\n", + "$$ \\text{Cost} = - \\sum\\limits^N_{n=1} \\log P\\left(\\tilde{y}(n) = t(n)|x(n) \\right) - \\sum\\limits^L_{l=0} \\lambda_l \\sum\\limits^N_{n=1} \\| z^{(l)}(n) - \\tilde{z}^{(l)}_{BN}(n) \\|^2 $$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Implementation" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using gpu device 0: GeForce GTX 970M (CNMeM is disabled, cuDNN 5005)\n" + ] + } + ], + "source": [ + "# %load ladder_net_layers.py\n", + "from lasagne.layers import MergeLayer\n", + "\n", + "import theano\n", + "import theano.tensor as T\n", + "\n", + "import numpy as np\n", + "\n", + "def _create_milaUDEM_params(shape, name):\n", + " values = np.zeros((6,) + shape, dtype=theano.config.floatX)\n", + "\n", + " b_lin = theano.shared(values[0], name='bias_lin_{}'.format(name))\n", + " b_sigm = theano.shared(values[1], name='bias_sigm_{}'.format(name))\n", + "\n", + " w_u_lin = theano.shared(values[2], name='weight_u_lin_{}'.format(name))\n", + " w_u_sigm = theano.shared(values[3], name='weight_u_sigm_{}'.format(name))\n", + " w_zu_lin = theano.shared(values[4], name='weight_zu_lin_{}'.format(name))\n", + " w_zu_sigm = theano.shared(values[5], name='weight_zu_sigm_{}'.format(name))\n", + "\n", + " values = np.ones((3,) + shape, dtype=theano.config.floatX)\n", + " w_z_lin = theano.shared(values[0], name='weight_z_lin_{}'.format(name))\n", + " w_z_sigm = theano.shared(values[1], name='weight_z_sigm_{}'.format(name))\n", + " w_sigm = theano.shared(values[2], name='weight_sigm_{}'.format(name))\n", + "\n", + " # combinator params used in combinator calculations\n", + " return [w_u_lin, w_z_lin, w_zu_lin, w_u_sigm, w_z_sigm,\n", + " w_zu_sigm, w_sigm, b_lin, b_sigm]\n", + "\n", + "\n", + "def _create_curiousAI_params(shape, name):\n", + " values = np.zeros((8,) + shape, dtype=theano.config.floatX)\n", + "\n", + " b_mu_sig = theano.shared(values[0], name='b_mu_sig_{}'.format(name))\n", + " b_mu_lin = theano.shared(values[1], name='b_mu_lin_{}'.format(name))\n", + " b_v_sig = theano.shared(values[2], name='b_v_sig_{}'.format(name))\n", + " b_v_lin = theano.shared(values[3], name='b_v_lin_{}'.format(name))\n", + "\n", + " w_mu_lin = theano.shared(values[4], name='w_mu_lin_{}'.format(name))\n", + " w_v_lin = theano.shared(values[5], name='w_v_lin_{}'.format(name))\n", + " w_mu = theano.shared(values[6], name='w_mu_{}'.format(name))\n", + " w_v = theano.shared(values[7], name='w_v_{}'.format(name))\n", + "\n", + " values = np.ones((2,) + shape, dtype=theano.config.floatX)\n", + " w_mu_sig = theano.shared(values[0], name='w_mu_sig_{}'.format(name))\n", + " w_v_sig = theano.shared(values[1], name='w_v_sig_{}'.format(name))\n", + "\n", + " # combinator params used in combinator calculations\n", + " return [w_mu_lin, w_v_lin, w_mu_sig, w_v_sig, w_mu, w_v,\n", + " b_mu_lin, b_v_lin, b_mu_sig, b_v_sig]\n", + "\n", + "\n", + "def _create_combinator_params(combinator_type, shape, name):\n", + " if combinator_type == 'milaUDEM':\n", + " return _create_milaUDEM_params(shape, name)\n", + " elif combinator_type == 'curiousAI':\n", + " return _create_curiousAI_params(shape, name)\n", + "\n", + "\n", + "def _combinator_milaUDEM(z, u, combinator_params, bc_pttrn):\n", + " w_u_lin, w_z_lin, w_zu_lin, w_u_sigm, w_z_sigm, w_zu_sigm, w_sigm, \\\n", + " b_lin, b_sigm = combinator_params\n", + "\n", + " lin_out = w_z_lin.dimshuffle(*bc_pttrn) * z + \\\n", + " w_u_lin.dimshuffle(*bc_pttrn) * u + \\\n", + " w_zu_lin.dimshuffle(*bc_pttrn) * z * u + \\\n", + " b_lin.dimshuffle(*bc_pttrn)\n", + "\n", + " sigm_pre = w_z_sigm.dimshuffle(*bc_pttrn) * z + \\\n", + " w_u_sigm.dimshuffle(*bc_pttrn) * u + \\\n", + " w_zu_sigm.dimshuffle(*bc_pttrn) * z * u + \\\n", + " b_sigm.dimshuffle(*bc_pttrn)\n", + "\n", + " sigm_out = T.nnet.sigmoid(sigm_pre)\n", + "\n", + " output = w_sigm.dimshuffle(*bc_pttrn) * sigm_out + lin_out\n", + "\n", + " return output\n", + "\n", + "\n", + "def _combinator_curiousAI(z, u, combinator_params, bc_pttrn):\n", + " w_mu_lin, w_v_lin, w_mu_sig, w_v_sig, w_mu, w_v, \\\n", + " b_mu_lin, b_v_lin, b_mu_sig, b_v_sig = combinator_params\n", + "\n", + " mu_sig_pre = w_mu_sig.dimshuffle(*bc_pttrn) * u + \\\n", + " b_mu_sig.dimshuffle(*bc_pttrn)\n", + "\n", + " mu_lin_out = w_mu_lin.dimshuffle(*bc_pttrn) * u + \\\n", + " b_mu_lin.dimshuffle(*bc_pttrn)\n", + "\n", + " mu_u = w_mu.dimshuffle(*bc_pttrn) * T.nnet.sigmoid(mu_sig_pre) + \\\n", + " mu_lin_out\n", + "\n", + " v_sig_pre = w_v_sig.dimshuffle(*bc_pttrn) * u + \\\n", + " b_v_sig.dimshuffle(*bc_pttrn)\n", + "\n", + " v_lin_out = w_v_lin.dimshuffle(*bc_pttrn) * u + \\\n", + " b_v_lin.dimshuffle(*bc_pttrn)\n", + "\n", + " v_u = w_v * T.nnet.sigmoid(v_sig_pre) + v_lin_out\n", + "\n", + " output = (z - mu_u) * v_u + mu_u\n", + "\n", + " return output\n", + "\n", + "\n", + "def _combinator(z, u, combinator_type, combinator_params):\n", + " if u.ndim == 2:\n", + " bc_pttrn = ('x', 0)\n", + " elif u.ndim == 4:\n", + " bc_pttrn = ('x', 0, 1, 2)\n", + "\n", + " if combinator_type == 'milaUDEM':\n", + " return _combinator_milaUDEM(z, u, combinator_params, bc_pttrn)\n", + " elif combinator_type == 'curiousAI':\n", + " return _combinator_curiousAI(z, u, combinator_params, bc_pttrn)\n", + "\n", + "\n", + "class CombinatorLayer(MergeLayer):\n", + " \"\"\"\n", + " A layer that combines the terms from dirty and clean encoders,\n", + " and outputs denoised variable:\n", + " $$ \\hat{z} = g(\\tilde{z}, u)$$\n", + " \"\"\"\n", + " def __init__(self, incoming_z, incoming_u, combinator_type, **kwargs):\n", + " super(CombinatorLayer, self).__init__(\n", + " [incoming_z, incoming_u], **kwargs)\n", + " self.combinator_type = combinator_type\n", + " z_shp, u_shp = self.input_shapes\n", + "\n", + " if z_shp != u_shp:\n", + " raise ValueError(\"Mismatch: input shapes must be the same. \"\n", + " \"Got dirty z ({0}) of shape {1} and clean u ({\"\n", + " \"2}) of shape {3}\".format(incoming_z.name, z_shp,\n", + " incoming_u.name, u_shp))\n", + "\n", + " self.combinator_params = _create_combinator_params(combinator_type,\n", + " u_shp[1:],\n", + " self.name)\n", + "\n", + " def get_output_shape_for(self, input_shapes):\n", + " return input_shapes[0]\n", + "\n", + " def get_output_for(self, inputs, **kwargs):\n", + " z, u = inputs\n", + " assert z.ndim == u.ndim\n", + " return _combinator(z, u, self.combinator_type, self.combinator_params)\n", + "\n", + "\n", + "class SharedNormLayer(MergeLayer):\n", + " \"\"\"\n", + " A layer that combines the terms from dirty and clean encoders,\n", + " and outputs denoised variable:\n", + " $$ \\hat{z} = g(\\tilde{z}, u)$$\n", + " \"\"\"\n", + " def __init__(self, incoming2stats, incoming2norm, axes='auto', epsilon=1e-4,\n", + " **kwargs):\n", + " super(SharedNormLayer, self).__init__(\n", + " [incoming2stats, incoming2norm], **kwargs)\n", + " stats_shp, norm_shp = self.input_shapes\n", + "\n", + " if stats_shp != norm_shp:\n", + " raise ValueError(\"Mismatch: input shapes must be the same. \"\n", + " \"Got dirty z ({0}) of shape {1} and clean u ({\"\n", + " \"2}) of shape {3}\"\n", + " .format(incoming2stats.name, stats_shp,\n", + " incoming2norm.name, norm_shp))\n", + "\n", + " if axes == 'auto':\n", + " # default: normalize over all but the second axis\n", + " axes = (0,) + tuple(range(2, len(stats_shp)))\n", + " elif isinstance(axes, int):\n", + " axes = (axes,)\n", + " self.axes = axes\n", + " self.epsilon = epsilon\n", + "\n", + " def get_output_shape_for(self, input_shapes):\n", + " return input_shapes[0]\n", + "\n", + " def get_output_for(self, inputs, **kwargs):\n", + " to_stats, to_norm = inputs\n", + " assert to_stats.ndim == to_norm.ndim\n", + "\n", + " mean = to_stats.mean(self.axes, keepdims=True)\n", + " inv_std = T.inv(T.sqrt(to_stats.var(self.axes,\n", + " keepdims=True) + self.epsilon))\n", + "\n", + " return (to_norm - mean) * inv_std" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# %load ladder_nets.py\n", + "from lasagne.layers import InputLayer, DenseLayer\n", + "from lasagne.layers import Conv2DLayer as conv\n", + "from lasagne.layers import Deconv2DLayer as deconv\n", + "from lasagne.layers import MaxPool2DLayer as pool\n", + "from lasagne.layers.special import InverseLayer as unpool\n", + "from lasagne.layers.special import BiasLayer, ScaleLayer, NonlinearityLayer\n", + "from lasagne.layers.noise import GaussianNoiseLayer\n", + "from lasagne.layers.normalization import BatchNormLayer, batch_norm\n", + "from lasagne.nonlinearities import rectify, linear\n", + "import lasagne\n", + "\n", + "from ladder_net_layers import CombinatorLayer, SharedNormLayer\n", + "from utils import softmax, unzip\n", + "\n", + "import theano.tensor as T\n", + "from collections import OrderedDict\n", + "\n", + "get_items = lambda zipped: unzip(zipped)\n", + "xe = T.nnet.categorical_crossentropy\n", + "\n", + "def build_encoder(net, encoder_specs, activation, name, p_drop_hidden,\n", + " shared_net):\n", + " # encoder specs is a tuple of string and tuple of integers\n", + " for i, (transform, specs) in enumerate(encoder_specs):\n", + " if transform == 'unpool':\n", + " specs = net.get(specs)\n", + " # if specs have already the name of the corresponding pool layer\n", + " update = build_enc_layer(\n", + " net.values()[-1], name, transform, specs, activation, i,\n", + " p_drop_hidden, shared_net\n", + " )\n", + " net.update(update)\n", + " # apply activation\n", + " if i < len(encoder_specs) - 1:\n", + " act_name = 'enc_activation_{}'.format(i)\n", + " net[act_name] = NonlinearityLayer(\n", + " net.values()[-1], nonlinearity=activation,\n", + " name='{}_{}'.format(name, act_name)\n", + " )\n", + "\n", + " # classfication layer activation -> softmax\n", + " net['enc_softmax'] = NonlinearityLayer(\n", + " net.values()[-1], nonlinearity=softmax, name=name+'_enc_softmax'\n", + " )\n", + "\n", + " return net['enc_softmax'], net\n", + "\n", + "\n", + "def build_enc_layer(incoming, name, transform, specs, activation, i,\n", + " p_drop_hidden, shared_net):\n", + " net = OrderedDict()\n", + " lname = 'enc_{}_{}'.format(i, transform if 'pool' in transform else 'affine')\n", + " nbatchn_lname = 'enc_batchn_{}_norm'.format(i)\n", + " noise_lname = 'enc_noise_{}'.format(i)\n", + " lbatchn_lname = 'enc_batchn_{}_learn'.format(i)\n", + "\n", + " if shared_net is None:\n", + " # affine pars\n", + " W = lasagne.init.GlorotUniform()\n", + " # batchnorm pars\n", + " beta = lasagne.init.Constant(0)\n", + " gamma = None if activation == rectify else lasagne.init.Constant(1)\n", + " else:\n", + " # batchnorm pars\n", + " beta = shared_net[lbatchn_lname + '_beta'].get_params()[0]\n", + " gamma = None if activation == rectify else \\\n", + " shared_net[lbatchn_lname + '_gamma'].get_params()[0]\n", + " if not isinstance(shared_net[lname], (pool, unpool)):\n", + " # affine weights\n", + " W = shared_net[lname].get_params()[0]\n", + " else:\n", + " W = None\n", + "\n", + " # affine (conv/dense/deconv) or (un)pooling transformation: $W \\hat{h}$\n", + " net[lname] = get_transform_layer(\n", + " incoming, name+'_'+lname, transform, specs, W\n", + " )\n", + "\n", + " # 1. batchnormalize without learning -> goes to combinator layer\n", + " layer2bn = net.values()[-1]\n", + " l_name = '{}_{}'.format(name, nbatchn_lname)\n", + " bn_broadcast_cond = layer2bn.output_shape[1] == 1\n", + " if len(layer2bn.output_shape) == 4 and bn_broadcast_cond:\n", + " ax = (0, 1, 2, 3)\n", + " elif len(layer2bn.output_shape) == 2 and bn_broadcast_cond:\n", + " ax = (0, 1)\n", + " else:\n", + " ax = 'auto'\n", + " net[nbatchn_lname] = BatchNormLayer(\n", + " layer2bn, axes=ax, alpha=0.1, beta=None, gamma=None, name=l_name\n", + " )\n", + " if shared_net is None:\n", + " # for dirty encoder -> add noise\n", + " net[noise_lname] = GaussianNoiseLayer(\n", + " net.values()[-1], sigma=p_drop_hidden,\n", + " name='{}_{}'.format(name, noise_lname)\n", + " )\n", + "\n", + " # 2. scaling & offsetting batchnormalization + noise\n", + " l_name = '{}_{}'.format(name, lbatchn_lname)\n", + " # offset by beta\n", + " net[lbatchn_lname + '_beta'] = BiasLayer(\n", + " net.values()[-1], b=beta, name=l_name+'_beta'\n", + " )\n", + " if gamma is not None:\n", + " # if not rectify, scale by gamma\n", + " net[lbatchn_lname + '_gamma'] = ScaleLayer(\n", + " net.values()[-1], scales=gamma, name=l_name+'_gamma'\n", + " )\n", + "\n", + " return net\n", + "\n", + "\n", + "def get_transform_layer(incoming, name, transform, specs, W):\n", + " if transform == 'conv':\n", + " layer = conv(\n", + " incoming, num_filters=specs[0], filter_size=specs[1],\n", + " stride=specs[2], pad=specs[3], nonlinearity=linear, W=W, b=None,\n", + " name=name+'_conv'\n", + " )\n", + " elif transform == 'dense':\n", + " layer = DenseLayer(\n", + " incoming, num_units=specs, nonlinearity=linear, W=W, b=None,\n", + " name=name+'_dense'\n", + " )\n", + " elif transform == 'pool':\n", + " if len(specs) == 4:\n", + " psize, pstride = specs[1:3]\n", + " else:\n", + " psize, pstride = specs\n", + " layer = pool(\n", + " incoming, pool_size=psize, stride=pstride, name=name\n", + " )\n", + " elif transform == 'deconv':\n", + " layer = deconv(\n", + " incoming, num_filters=specs[0], filter_size=specs[1],\n", + " stride=specs[2], crop=specs[3], nonlinearity=linear, W=W, b=None,\n", + " name=name+'_deconv'\n", + " )\n", + " elif transform == 'unpool':\n", + " pl = specs\n", + " # print(pl.name, pl.output_shape)\n", + " layer = unpool(incoming, pl, name=name)\n", + "\n", + " return layer\n", + "\n", + "\n", + "def build_dec_layer(incoming, z_l, name, transform, specs, l,\n", + " combinator_type, layer2stats=None, last=False):\n", + " dirty_net = OrderedDict()\n", + "\n", + " if l > 0:\n", + " # transformation layer: dense, deconv, unpool\n", + " lname = 'dec_{}_{}'.format(l, transform if 'pool' in transform\n", + " else 'affine')\n", + " if transform in ['pool', 'unpool']:\n", + " W = None\n", + " else:\n", + " W = lasagne.init.GlorotUniform()\n", + " dirty_net[lname] = get_transform_layer(incoming, name+'_'+lname,\n", + " transform, specs, W)\n", + " layer2bn = dirty_net.values()[-1]\n", + " else:\n", + " layer2bn = incoming\n", + "\n", + " # batchnormalization ... u_l\n", + " ul_name = 'dec_batchn_u_{}'.format(l)\n", + " bn_broadcast_cond = layer2bn.output_shape[1] == 1\n", + " if len(layer2bn.output_shape) == 4 and bn_broadcast_cond:\n", + " ax = (0, 1, 2, 3)\n", + " elif len(layer2bn.output_shape) == 2 and bn_broadcast_cond:\n", + " ax = (0, 1)\n", + " else:\n", + " ax = 'auto'\n", + " dirty_net[ul_name] = BatchNormLayer(\n", + " layer2bn, axes=ax, alpha=1., beta=None, gamma=None,\n", + " name=name+'_'+ul_name\n", + " )\n", + "\n", + " # denoised latent \\hat{z}_L-i\n", + " comb_name = 'dec_combinator_{}'.format(l)\n", + " dirty_net[comb_name] = CombinatorLayer(\n", + " z_l, dirty_net.values()[-1], combinator_type=combinator_type,\n", + " name=name+'_'+comb_name\n", + " )\n", + "\n", + " if not last:\n", + " # batchnormalized latent \\hat{z}_L-i^{BN}\n", + " layer2norm = dirty_net[comb_name]\n", + " bname = 'dec_batchn_z_{}'.format(l)\n", + " dirty_net[bname] = SharedNormLayer(\n", + " layer2stats, layer2norm, name=name+'_'+bname\n", + " )\n", + "\n", + " return dirty_net\n", + "\n", + "\n", + "def build_decoder(dirty_net, clean_net, name, decoder_specs, combinator_type):\n", + " L = len(decoder_specs) - 1\n", + " net = OrderedDict()\n", + "\n", + " # dirty_enc_affine_1 ... z_L\n", + " z_L = dirty_net['enc_noise_{}'.format(L)]\n", + "\n", + " # batchnormalize denoised latent using clean encoder's bn mean/inv_std\n", + " # without learning\n", + " enc_bname = 'enc_batchn_{}_norm'.format(L)\n", + " layer2stats = clean_net[enc_bname]\n", + "\n", + " # batchnorm and combinator\n", + " update = build_dec_layer(\n", + " dirty_net.values()[-1], z_L, name, 'N/A', None, 0, combinator_type,\n", + " layer2stats\n", + " )\n", + " net.update(update)\n", + "\n", + " for i, (transform, specs) in enumerate(decoder_specs[:-1]):\n", + " # dirty_enc_affine_L-i ... z_l\n", + " z_l = dirty_net['enc_noise_{}'.format(L-i-1)]\n", + " enc_bname = 'enc_batchn_{}_norm'.format(L-i-1)\n", + " layer2stats = clean_net[enc_bname]\n", + "\n", + " if transform == 'unpool':\n", + " # print(dirty_net.keys(), specs)\n", + " specs = dirty_net.get(specs)\n", + " update = build_dec_layer(\n", + " net.values()[-1], z_l, name, transform, specs, i+1,\n", + " combinator_type, layer2stats\n", + " )\n", + " net.update(update)\n", + "\n", + " # corrupted input ... z_0\n", + " z_0 = dirty_net['input_corr']\n", + " transform, specs = decoder_specs[-1]\n", + "\n", + " if transform == 'unpool':\n", + " specs = dirty_net.get(specs)\n", + " update = build_dec_layer(\n", + " net.values()[-1], z_0, name, transform, specs, i+2,\n", + " combinator_type, None, True\n", + " )\n", + " net.update(update)\n", + "\n", + " return net\n", + "\n", + "\n", + "def build_model(encoder_specs, decoder_specs, p_drop_input, p_drop_hidden,\n", + " input_shape, batch_size=None, activation=rectify,\n", + " combinator_type='MILAudem'):\n", + " net = OrderedDict()\n", + " net['input'] = InputLayer(\n", + " (batch_size, ) + tuple(input_shape), name='input'\n", + " )\n", + " # corrupted input\n", + " net['input_corr'] = GaussianNoiseLayer(\n", + " net['input'], sigma=p_drop_input, name='input_corr'\n", + " )\n", + "\n", + " # dirty encoder\n", + " train_output_l, dirty_encoder = build_encoder(\n", + " net, encoder_specs, activation, 'dirty', p_drop_hidden, None\n", + " )\n", + "\n", + " # clean encoder\n", + " clean_encoder = OrderedDict(net.items()[:1])\n", + " eval_output_l, clean_net = build_encoder(\n", + " clean_encoder, encoder_specs, activation, 'clean', 0., dirty_encoder\n", + " )\n", + "\n", + " # dirty decoder\n", + " dirty_decoder = build_decoder(\n", + " dirty_encoder, clean_net, 'dirty', decoder_specs, combinator_type\n", + " )\n", + "\n", + " return (train_output_l, eval_output_l, dirty_encoder, dirty_decoder,\n", + " clean_encoder)\n", + "\n", + "\n", + "def get_mu_sigma_costs(hid):\n", + " shp = hid.shape\n", + " mu = hid.mean(0)\n", + " sigma = T.dot(hid.T, hid) / shp[0]\n", + "\n", + " C_mu = T.sum(mu ** 2)\n", + " C_sigma = T.diagonal(sigma - T.log(T.clip(sigma, 1e-15, 1)))\n", + " C_sigma -= - T.ones_like(C_sigma)\n", + " return C_mu, C_sigma.sum() # trace(C_sigma)\n", + "\n", + "\n", + "def build_costNstats(y_onehot, output_train, output_eval, num_labeled=None,\n", + " pseudo_labels=None):\n", + " pred = T.clip(output_train, 1e-15, 1)\n", + " N = num_labeled if num_labeled else pred.shape[0]\n", + " class_cost = xe(pred[:N], y_onehot[:N]).mean()\n", + "\n", + " if pseudo_labels == 'soft':\n", + " n = 0 if num_labeled else N\n", + " class_cost += xe(pred[n:], pred[n:]).mean()\n", + " elif pseudo_labels == 'hard':\n", + " M = y.shape[1]\n", + " n = 0 if num_labeled else N\n", + " pseudo_target = T.eye(M)[pred[n:].argmax(axis=1)]\n", + " class_cost += xe(pred[n:], pseudo_target).mean()\n", + "\n", + " pred = T.argmax(output_eval[:N], axis=1)\n", + " y = T.argmax(y_onehot[:N], axis=1)\n", + " accuracy = T.mean(T.eq(pred, y), dtype='float32')\n", + "\n", + " return class_cost, [accuracy]\n", + "\n", + "\n", + "def build_rec_costs(X, clean_net, dirty_net, decoder_specs, lambdas,\n", + " alphas=None, betas=None, use_extra_costs=False):\n", + " L = len(decoder_specs)\n", + "\n", + " # get clean and corresponding dirty latent layer output\n", + " z_clean_l = clean_net['input']\n", + " z_dirty_l = dirty_net['dec_combinator_{}'.format(L)]\n", + "\n", + " z_clean = lasagne.layers.get_output(z_clean_l, X, deterministic=False)\n", + " z_dirty = lasagne.layers.get_output(z_dirty_l, X, deterministic=False)\n", + "\n", + " # squared error\n", + " cost = lambdas[L] * T.sqr(z_clean - z_dirty).mean()\n", + " if use_extra_costs:\n", + " C_mu, C_sigma = get_mu_sigma_costs(z_clean)\n", + " cost += alphas[L] * C_mu + betas[L] * C_sigma\n", + "\n", + " rec_costs = [cost]\n", + "\n", + " dec_batchns = [x for x in dirty_net.keys() if 'dec_batchn_z' in x][::-1]\n", + "\n", + " for l, name in enumerate(dec_batchns):\n", + " z_clean_l = clean_net['enc_batchn_{}_norm'.format(l)]\n", + " z_dirty_l = dirty_net[name]\n", + "\n", + " z_clean = lasagne.layers.get_output(z_clean_l, X, deterministic=False)\n", + " z_dirty = lasagne.layers.get_output(z_dirty_l, X, deterministic=False)\n", + "\n", + " cost = lambdas[l] * T.sqr(z_clean - z_dirty).mean()\n", + " if use_extra_costs:\n", + " C_mu, C_sigma = get_mu_sigma_costs(z_clean)\n", + " cost += alphas[l] * C_mu + betas[l] * C_sigma\n", + "\n", + " rec_costs.append(cost)\n", + "\n", + " return rec_costs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# %load utils.py\n", + "import numpy as np\n", + "import gzip\n", + "import cPickle as pickle\n", + "\n", + "import lasagne\n", + "import theano.misc.pkl_utils\n", + "import theano.tensor as T\n", + "\n", + "def pickle_load(f, encoding):\n", + " return pickle.load(f)\n", + "\n", + "\n", + "def load_data(shared_var=True):\n", + " \"\"\"Get data with labels, split into training, validation and test set.\"\"\"\n", + " with gzip.open('./mnist.pkl.gz', 'rb') as f:\n", + " data = pickle_load(f, encoding='latin-1')\n", + " X_train, y_train = data[0]\n", + " X_valid, y_valid = data[1]\n", + " X_test, y_test = data[2]\n", + "\n", + " if shared_var:\n", + " return dict(\n", + " X_train=theano.shared(lasagne.utils.floatX(X_train)),\n", + " y_train=T.cast(theano.shared(y_train), 'int32'),\n", + " X_valid=theano.shared(lasagne.utils.floatX(X_valid)),\n", + " y_valid=T.cast(theano.shared(y_valid), 'int32'),\n", + " X_test=theano.shared(lasagne.utils.floatX(X_test)),\n", + " y_test=T.cast(theano.shared(y_test), 'int32'),\n", + " num_examples_train=X_train.shape[0],\n", + " num_examples_valid=X_valid.shape[0],\n", + " num_examples_test=X_test.shape[0],\n", + " input_dm=X_train.shape[1],\n", + " output_dim=10,\n", + " )\n", + " else:\n", + " return dict(\n", + " X_train=np.float32(X_train),\n", + " y_train=np.int32(y_train),\n", + " X_valid=np.float32(X_valid),\n", + " y_valid=np.int32(y_valid),\n", + " X_test=np.float32(X_test),\n", + " y_test=np.int32(y_test),\n", + " num_examples_train=X_train.shape[0],\n", + " num_examples_valid=X_valid.shape[0],\n", + " num_examples_test=X_test.shape[0],\n", + " input_dm=X_train.shape[1],\n", + " output_dim=10,\n", + " )\n", + "\n", + "\n", + "def softmax(vec, axis=1):\n", + " \"\"\"\n", + " The ND implementation of softmax nonlinearity applied over a specified\n", + " axis, which is by default the second dimension.\n", + " \"\"\"\n", + " xdev = vec - vec.max(axis, keepdims=True)\n", + " rval = T.exp(xdev)/(T.exp(xdev).sum(axis, keepdims=True))\n", + " return rval\n", + "\n", + "unzip = lambda zipped: zip(*zipped)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2016-07-03 19:39:16-- http://deeplearning.net/data/mnist/mnist.pkl.gz\n", + "Resolving deeplearning.net (deeplearning.net)... 132.204.26.28\n", + "Connecting to deeplearning.net (deeplearning.net)|132.204.26.28|:80... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 16168813 (15M) [application/x-gzip]\n", + "Saving to: ‘mnist.pkl.gz’\n", + "\n", + "100%[======================================>] 16.168.813 5,71MB/s in 2,7s \n", + "\n", + "2016-07-03 19:39:19 (5,71 MB/s) - ‘mnist.pkl.gz’ saved [16168813/16168813]\n", + "\n" + ] + } + ], + "source": [ + "!wget -N http://deeplearning.net/data/mnist/mnist.pkl.gz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "# %load train_ladder_nets.py\n", + "from __future__ import print_function\n", + "from utils import load_data\n", + "from ladder_nets import *\n", + "import time\n", + "import theano.misc.pkl_utils\n", + "import lasagne\n", + "import cPickle\n", + "import numpy as np\n", + "\n", + "LEARNING_RATE = 0.1\n", + "LR_DECREASE = 1.\n", + "BATCH_SIZE = 100\n", + "INPUT_SHAPE = [1, 28, 28]\n", + "NUM_EPOCHS = 15\n", + "COMBINATOR_TYPE = 'milaUDEM' # or 'curiousAI'\n", + "DROPOUT = 0.3\n", + "EXTRA_COST = False # True\n", + "ALPHAS = None # [0.1]*3\n", + "BETAS = None # [0.1]*3\n", + "NUM_LABELED = None\n", + "PSEUDO_LABELS = None\n", + "CONV = True # False\n", + "POOL = True # False\n", + "\n", + "print (\"Loading data...\")\n", + "dataset = load_data()\n", + "\n", + "def get_encoder_settings(convolution, pooling):\n", + " if convolution and pooling:\n", + " settings = [('conv', (40, 8, 1, 0)), ('pool', (0, 2, 2, 0)),\n", + " ('conv', (10, 8, 1, 0)), ('pool', (0, 2, 2, 0))]\n", + " elif convolution:\n", + " settings = [('conv', (40, 15, 1, 0)), ('conv', (10, 14, 1, 0))]\n", + " else:\n", + " settings = [('dense', 500), ('dense', 10)]\n", + "\n", + " return settings\n", + "\n", + "def get_decoder_settings(convolution, pooling):\n", + " if convolution and pooling:\n", + " settings = [('unpool', 'enc_3_pool'), ('deconv', (40, 8, 1, 0)),\n", + " ('unpool', 'enc_1_pool'), ('deconv', (1, 8, 1, 0))]\n", + " elif convolution:\n", + " settings = [('deconv', (40, 14, 1, 0)), ('deconv', (1, 15, 1, 0))]\n", + " else:\n", + " settings = [('dense', 10), ('dense', 784)]\n", + "\n", + " return settings\n", + "\n", + "# build model\n", + "encoder_specs = get_encoder_settings(convolution=CONV, pooling=POOL)\n", + "decoder_specs = get_decoder_settings(convolution=CONV, pooling=POOL)\n", + "LAMBDAS = [1] * (len(decoder_specs) + 1)\n", + "input_shape = INPUT_SHAPE if CONV else np.prod(INPUT_SHAPE)\n", + "\n", + "print (\"Building model ...\")\n", + "train_output_l, eval_output_l, dirty_encoder, dirty_decoder, clean_encoder = \\\n", + " build_model(encoder_specs, decoder_specs, DROPOUT, DROPOUT,\n", + " input_shape=input_shape, combinator_type=COMBINATOR_TYPE)\n", + "\n", + "print (map(lambda x: (x.name, x.output_shape), dirty_encoder.values()))\n", + "print (map(lambda x: (x.name, x.output_shape), dirty_decoder.values()))\n", + "\n", + "# set up input/output variables\n", + "X = T.ftensor4('x') if CONV else T.fmatrix('X')\n", + "y = T.ivector('y')\n", + "y_onehot = lasagne.utils.one_hot(y, 10)\n", + "\n", + "# training output\n", + "output_train = lasagne.layers.get_output(train_output_l, X,\n", + " deterministic=False).flatten(2)\n", + "\n", + "# evaluation output. Also includes output of transform for plotting\n", + "output_eval = lasagne.layers.get_output(eval_output_l, X,\n", + " deterministic=True).flatten(2)\n", + "\n", + "# set up (possibly amortizable) lr, cost and updates\n", + "sh_lr = theano.shared(lasagne.utils.floatX(LEARNING_RATE))\n", + "\n", + "print (\"Building costs and updates ...\")\n", + "class_cost, stats = build_costNstats(y_onehot, output_train, output_eval,\n", + " NUM_LABELED, PSEUDO_LABELS)\n", + "\n", + "rec_costs = build_rec_costs(X, clean_encoder, dirty_decoder, decoder_specs,\n", + " lambdas=LAMBDAS, alphas=ALPHAS, betas=BETAS,\n", + " use_extra_costs=EXTRA_COST)\n", + "\n", + "cost = class_cost + T.sum(rec_costs)\n", + "net_params = lasagne.layers.get_all_params(train_output_l, trainable=True)\n", + "updates = lasagne.updates.adam(cost, net_params, learning_rate=sh_lr)\n", + "\n", + "# get training and evaluation functions, cost = class_cost + T.sum(rec_costs)\n", + "batch_index = T.iscalar('batch_index')\n", + "batch_slice = slice(batch_index * BATCH_SIZE, (batch_index + 1) * BATCH_SIZE)\n", + "\n", + "print (\"Compiling functions...\")\n", + "train = theano.function([batch_index], [cost] + rec_costs,\n", + " updates=updates, givens={\n", + " X: dataset['X_train'][batch_slice].reshape(\n", + " (-1,) + tuple(input_shape)\n", + " ),\n", + " y: dataset['y_train'][batch_slice],\n", + " })\n", + "\n", + "eval = theano.function([batch_index], [cost] + stats, givens={\n", + " X: dataset['X_valid'][batch_slice].reshape(\n", + " (-1,) + tuple(input_shape)\n", + " ),\n", + " y: dataset['y_valid'][batch_slice],\n", + " })\n", + "\n", + "network_dump = {'train_output_layer': train_output_l,\n", + " 'eval_output_layer': eval_output_l,\n", + " 'dirty_net': dirty_decoder,\n", + " 'clean_net': clean_encoder,\n", + " 'x': X,\n", + " 'y': y,\n", + " 'output_eval': output_eval\n", + " }\n", + "\n", + "def save_dump(filename,param_values):\n", + " f = file(filename, 'wb')\n", + " cPickle.dump(param_values,f,protocol=cPickle.HIGHEST_PROTOCOL)\n", + " f.close()\n", + "\n", + "\n", + "def train_epoch():\n", + " costs = []\n", + " rec_costs = []\n", + " for b in range(num_batches_train):\n", + " train_out = train(b)\n", + " train_cost = train_out[0]\n", + " rec_cost = train_out[1:]\n", + "\n", + " costs.append(train_cost)\n", + " rec_costs.append(rec_cost)\n", + "\n", + " return (np.mean(costs), np.mean(rec_costs, axis=0))\n", + " \n", + "\n", + "def eval_epoch():\n", + " costs = []\n", + " accs = []\n", + " for b in range(num_batches_valid):\n", + " eval_cost, eval_acc = eval(b)\n", + " costs.append(eval_cost)\n", + " accs.append(eval_acc)\n", + "\n", + " return np.mean(eval_cost), np.mean(eval_acc)\n", + "\n", + "\n", + "num_batches_train = dataset['num_examples_train'] // BATCH_SIZE\n", + "num_batches_valid = dataset['num_examples_valid'] // BATCH_SIZE\n", + "\n", + "train_costs, valid_costs, valid_accs = [], [], []\n", + "\n", + "print (\"Starting training...\")\n", + "now = time.time()\n", + "\n", + "try:\n", + " for n in range(NUM_EPOCHS):\n", + " train_cost, rec_costs = train_epoch()\n", + " eval_cost, acc = eval_epoch()\n", + " \n", + " train_costs.append(train_cost)\n", + " valid_costs.append(eval_cost)\n", + " valid_accs.append(acc)\n", + "\n", + " print (\"Epoch %d took %.3f s\" % (n + 1, time.time() - now))\n", + " now = time.time()\n", + " print (\"Train cost {}, val cost {}, val acc {}\".format(train_costs[-1],\n", + " valid_costs[-1],\n", + " valid_accs[-1]))\n", + " print ('\\n'.join(['Layer #{} rec cost: {}'.format(i, c) for i, c\n", + " in enumerate(rec_costs)]))\n", + "\n", + " if (n+1) % 10 == 0:\n", + " new_lr = sh_lr.get_value() * LR_DECREASE\n", + " print (\"New LR:\", new_lr)\n", + " sh_lr.set_value(lasagne.utils.floatX(new_lr))\n", + "except KeyboardInterrupt:\n", + " pass\n", + "\n", + "# uncomment if to save the learning curve\n", + "# save_dump('final_epoch_{}_accs_ladder_net_mnist.pkl'.format(n),\n", + "# zip(train_cost, valid_cost))\n", + "\n", + "# uncomment if to save the params only\n", + "# save_dump('final_epoch_{}_ladder_net_mnist.pkl'.format(n),\n", + "# lasagne.layers.get_all_param_values(output_layer))\n", + "\n", + "# uncomment if to save the whole network\n", + "# save_dump('final_epoch_{}_ladder_net_mnist.pkl'.format(n),\n", + "# network_dump)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3QAAAGoCAYAAAAO86fGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4VeW5///3HSZlCBDCZCBhSEUGFajiLEFkqEerx1aP\noIdqnS5bW7XtaS3Wiu3POtb28LWDtIqodahWe7QKYTI4IEIVEVAQmWeQKcxD8vz+WHvHnZCQae+9\nhnxe15WLvdd4L4iuda/7Gcw5h4iIiIiIiIRPht8BiIiIiIiISN0ooRMREREREQkpJXQiIiIiIiIh\npYROREREREQkpJTQiYiIiIiIhJQSOhERERERkZCqNqEzs2Zm9oGZzTezxWb2m0q2GW1mC2I/75rZ\nKQnrVsWWzzezucm+ABERkVQzs5FmtsTMPjezn1Wyvo2ZvRK7380xsz4J624zs4Wxn9sSlt9jZuvM\n7KPYz8h0XY+IiERH4+o2cM4dNLMhzrl9ZtYIeM/MznHOvZew2QrgfOfcrtgNaQJwZmxdKVDgnNuR\n9OhFRERSzMwygMeAocAGYJ6Z/Z9zbknCZmOB+c65y82sF/AH4EIz6wtcD5wGHAGmmNnrzrkVsf0e\ndc49mraLERGRyKlRk0vn3L7Yx2axfXZUWD/HObcr9nUOkJOw2mp6HhERkQAaBCxzzq12zh0GXgAu\nrbBNH2AmgHNuKdDNzNoDvYEPnHMHnXMlwCzg8oT9LOXRi4hIpNUo0TKzDDObD2wCipxznx5j8xuA\nyQnfHTDNzOaZ2Y11D1VERMQXOcDahO/rKP/iEmABsUTNzAYBuUAXYBFwnpm1NbPmwEVA14T9bjWz\nj83sr2bWOlUXICIi0VXTCl2pc24A3s3pfDMbXNl2ZjYEuA5I7F9wjnNuIN5N7Ptmdm49YxYREQma\nB4C2ZvYR8H1gPlASa5b5IDANeDO+PLbPH4Eezrn+eC9M1fRSRERqrdo+dImcc8Vm9gZeX4BZieti\nA6FMAEYm9pdzzm2M/bnVzF7Fa7rybsVjm5mrffgiIhJGzrkwNTVcj1dxi+sSW1bGObcb+G78u5mt\nxOtfjnNuIjAxtvw+YtU+59zWhEP8BXi9spPr/igi0rDU9h5Zk1Eus+PNQMzseGAY8HGFbXKBfwD/\n7ZxbnrC8uZm1jH1uAQzHa35SVfCR+7nnnnt8j0HXpWvTdYXzJ6rXFkLzgHwzyzOzpsBVwGuJG5hZ\nazNrEvt8IzDLObcn9r197M9c4D+B52LfOyUc4nJ0f4zMT1SvLarXFeVr03WF76cualKh6wxMMrP4\n4CbPOOdmmNnN3j3GTQDuBrKAP8a2O+ycGwR0BF6NvV1sDPzNOTe1TpGKiIj4wDlXYma3AlPx7oNP\nOOc+q3Af7I13rywFFuONbBn3DzPLAg4D33POFceWP2Rm/fFGg14F3JyeKxIRkSipybQFC4GBlSx/\nPOHzjcBRA54451YC/esZo4iIiK+cc1OAXhWWJd4H51Rcn7Du/CqWj0lmjCIi0jBpOoEUKygo8DuE\nlIjqdUF0r03XFT5RvjaRKP9+R/XaonpdEN1r03U1DFbXtprJZmYuKLGIiEjqmBkuXIOi+Er3RxGR\nhqMu98hajXIpIlKdbt26sXr1ar/DkADIy8tj1apVfochInIU3avEb8m8R6pCJyJJFXuz5HcYEgBV\n/S6oQlc7uj+KJJ/uVeK3ZN4j1YdOREREREQkpJTQiYiIiIiIhJQSOhERERERkZBSQiciUgu33HIL\n9913X9K3FRERqY9Zs2bRtWvXsu/9+vXj7bffrtG2taX7W7BolEsRaTC6d+/OE088wQUXXFDnY/zp\nT39KybZ+y8jI4IsvvqBHjx5+hyIiInVk9tVYGosWLarxtscyadIk/vrXv/LOO++ULQvT/a0hUIVO\nRCSmpKTE7xB8U9Mbu4iINCzOuQZzjwjrc4ASOhFpEMaMGcOaNWu45JJLyMzM5JFHHmH16tVkZGTw\n5JNPkpeXx9ChQwG48sor6dy5M23btqWgoIBPP/207DjXXXcdv/zlL4Gvmqw8+uijdOzYkZycHJ56\n6qk6bbt9+3YuueQSWrduzRlnnMHdd9/NeeedV+X1vPvuu5xzzjm0bduWvLw8nn76aQCKi4sZM2YM\nHTp0oHv37uWaxCxfvpyCggLatGlDhw4dGDVqFACDBw/GOccpp5xCZmYmL730Uv3+skVEpE4eeugh\nrrjiinLLbrvtNm6//XYAnnrqKfr06UNmZib5+flMmDChymN1796dmTNnAnDgwAGuvfZasrKy6Nev\nH/PmzSu37YMPPkh+fj6ZmZn069ePf/7znwAsWbKEW265hffff59WrVqRlZUFlL+/AfzlL3/ha1/7\nGtnZ2Vx22WVs3LixbF1GRgaPP/44J554IllZWdx6661Vxjxv3jzOPvts2rZtS05ODj/4wQ84cuRI\n2frFixczfPhw2rVrR+fOnXnggQcAKC0t5Te/+U3ZNZx++umsX7++7D5fWlpadowhQ4bw5JNPAl71\n8dxzz+VHP/oR2dnZ3HvvvaxYsYKhQ4eSnZ1Nhw4duOaaayguLi7bf926dXzrW9+iQ4cOtG/fnh/+\n8IccPnyYdu3asXjx4rLttm7dSosWLdi2bVuV15ssSuhEpEF4+umnyc3N5V//+hfFxcX85Cc/KVv3\n9ttvs2TJEgoLCwG46KKLWL58OVu2bGHgwIFcffXVVR5306ZN7N69mw0bNvDXv/6V73//++zatavW\n237ve9+jVatWbNmyhaeeeopJkyZV+UZ0zZo1XHTRRdx22218+eWXfPzxx/Tv3x+AW2+9ld27d7Nq\n1SqKiop4+umnmThxIgB33303I0aMYOfOnaxbt44f/OAHgJdsAixcuJDi4uKjHiZERCQ9rrrqKiZP\nnszevXsBL1F56aWXyu5DHTt25M0336S4uJiJEydyxx138PHHH1d73HHjxrFy5UpWrlxJYWEhkyZN\nKrc+Pz+f9957j+LiYu655x6uueYaNm/ezEknncSf//xnzjrrLHbv3s327duPOvbMmTMZO3YsL7/8\nMhs3biQ3N5errrqq3DZvvPEGH374IQsWLODvf/87U6dOrTTORo0a8fvf/57t27fz/vvvM3PmTP74\nxz8CsGfPHoYNG8ZFF13Exo0b+eKLL8pexP72t7/lxRdfZMqUKRQXF/Pkk0/SvHlzoPoWKB988AH5\n+fls2bKFu+66C+ccY8eOZdOmTXz22WesW7eOcePGAd6/x8UXX0z37t1Zs2YN69ev56qrrqJJkyaM\nGjWKZ599tuy4zz//PBdeeCHt2rU75vmTwjkXiB8vFBEJuyD/t9ytWzc3Y8aMsu+rVq1yGRkZbtWq\nVVXus2PHDmdmrri42Dnn3LXXXuvuvvtu55xzRUVFrnnz5q6kpKRs+w4dOrgPPvigVtuWlJS4Jk2a\nuGXLlpWt+8UvfuHOO++8SmO6//773eWXX37U8pKSEte0aVO3ZMmSsmWPP/64GzJkiHPOuTFjxrib\nb77ZrVu37qh9zcwtX768yr+HuqjqdyG23Pf7Tlh+gvzflEhYBfm/q/POO88988wzzjnnpk6d6vLz\n86vc9rLLLnPjx493znn3ma5du5atS7zn9ejRw02dOrVs3YQJE8ptW1H//v3da6+95pxz7qmnnjrq\nfpR4f7v++uvdz372s7J1e/bscU2aNHGrV692znn3l9mzZ5etv/LKK92DDz54jL+Br/z+978vu989\n//zzbuDAgZVu16tXL/f6668ftTx+n0+89xYUFLgnnnii7Nry8vKOGcM///nPsvPOnj3bdejQodzx\n4ubMmeNyc3PLvp922mnupZdeqvK4ybxHqkInImlllpyfZOrSpUvZ59LSUu68807y8/Np06YN3bt3\nx8z48ssvK923Xbt2ZGR89b/S5s2bs2fPnlptu3XrVkpKSsrFcazRx9auXUvPnj2PWv7ll19y5MgR\ncnNzy5bl5eWxfv16wGvKU1payqBBgzj55JPLKnciIlKen/eqUaNG8fzzzwNelWf06NFl6yZPnsxZ\nZ51Fu3btaNu2LZMnT67y/pRow4YN5e4xeXl55dY//fTTDBgwgLZt29K2bVsWL15co+PGj514vBYt\nWtCuXbuyew94lcW4Y90nly1bxiWXXELnzp1p06YNd911V1kcVd374uvqOqhXxfvtli1bGDVqFF26\ndKFNmzZcc801ZTGsW7eOvLy8cvfyuDPOOIPmzZsza9Ysli5dyvLly/nmN79Zp5hqSwmdiKSVc8n5\nqYuqml0kLn/uued4/fXXmTlzJjt37mTVqlWJlZKUaN++PY0bN2bdunVly9auXVvl9l27duWLL744\nanl2djZNmjRh9erVZctWr15NTk4O4N1QJ0yYwPr16/nzn//M9773PVasWJHEKxERiQY/71VXXHEF\nRUVFrF+/nldffbUsoTt06BDf/va3+elPf8rWrVvZsWMH3/jGN2p0f+rcuXO5+0rifWLNmjXcdNNN\n/PGPf2THjh3s2LGDvn37lh23uiaLJ5xwQrnj7d27l23btpVLIGvqlltuoXfv3ixfvpydO3dy3333\nlcXRtWtXli9fXul+ubm5la5r0aIFAPv27StbtmnTpnLbVLy+sWPHkpGRweLFi9m5cyfPPvtsuRjW\nrFlTrk9eou985zs888wzPPPMM3z729+madOmNbzy+lFCJyINRqdOnY5KYCreCHfv3k2zZs1o27Yt\ne/fu5ec//3nKR/fKyMjg8ssvZ9y4cezfv58lS5aUDXJSmauvvpoZM2bw8ssvU1JSwvbt21mwYAEZ\nGRlceeWV3HXXXezZs4fVq1fzu9/9jv/+7/8G4OWXXy57Y9qmTRsyMjLK3jJW9ncjIiLpl52dzeDB\ng7nuuuvo0aMHvXr1AryE7tChQ2RnZ5ORkcHkyZOr7ItW0ZVXXsn9999f1of6scceK1u3d+9eMjIy\nyM7OprS0lIkTJ5ab8qBjx46sW7eOw4cPV3rsUaNGMXHiRD755BMOHjzI2LFjOfPMM+s0z93u3bvJ\nzMykefPmLFmypNz0CBdffDGbNm1i/PjxHDp0iD179jB37lwArr/+eu6+++6yl50LFy5kx44dZGdn\nk5OTw7PPPktpaSlPPvlklUlhYgwtW7akVatWrF+/nocffrhs3aBBg+jcuTN33nkn+/bt4+DBg8ye\nPbts/dVXX82rr77K3/72N8aMGVPr668rJXQi0mDceeed/PrXvyYrK4tHH30UOPrN3JgxY8jNzSUn\nJ4d+/fpx9tln1+octUn+Erf9f//v/7Fz5046d+7Md77zHUaPHk2zZs0q3a9r1668+eabPPLII2Rl\nZTFgwAA++eQTAMaPH0/z5s3p0aMH559/Ptdccw3XXXcd4I0edsYZZ5CZmclll13G+PHj6datG+B1\nmB8zZgxZWVm8/PLLtbpmERFJrtGjRzNjxoxyg3K1bNmS8ePHc8UVV5CVlcULL7zApZdeWuUxEu8x\n99xzD7m5uXTv3p2RI0eWSzZ69+7Nj3/8Y84880w6derE4sWLOffcc8vWX3DBBfTt25dOnTrRoUOH\no84zdOhQfv3rX3P55ZeTk5PDypUreeGFFyqNo7LviR555BH+9re/kZmZyc0331xucJWWLVsybdo0\nXnvtNTp16sSJJ55IUVERAD/60Y+48sorGT58OK1bt+aGG25g//79AEyYMIGHHnqI7OxsPvvsM845\n55wqzx//u/rwww9p06YNl1xyCd/61rfK1mVkZPD666+zbNkycnNz6dq1K3//+9/L1nft2pUBAwZg\nZuX+DlPNUtmMqDbMzAUlFhGpOzNLafPEhuLOO+9k8+bNoe7nVtXvQmx5w5jUKAl0fxRJPt2rJFVu\nuOEGTjjhBH71q18dc7tk3iMb1y5EERFJhaVLl3Lo0CFOPvlk5s6dyxNPPFE2T46IiIgE3+rVq3nl\nlVeYP39+Ws+rJpciIgGwe/duLr/8clq2bMmoUaP4n//5Hy655BK/wxIREZEa+OUvf8nJJ5/MT3/6\n06NGEU01NbkUkaRSMxaJU5PL5ND9UST5dK8SvyXzHqkKnYiIiIiISEgpoRMREREREQkpJXQiIiIi\nIiIhpYROREREREQkpDRtgYiIiIg0KHl5ecec4Fok1ZI5EqYqdCIi1Zg1axZdu3Yt+96vXz/efvvt\nGm1bW7fccgv33XdfnfcXEZHqrVq1CuecfvTj28+qVauS9vushE5EpAYS3+QuWrSI888/v0bbHsuk\nSZM477zzyi3705/+xF133VW3INOosthFREQk/ZTQiYj4xDkX2iY/YY5dREQkSpTQiUiD8NBDD3HF\nFVeUW3bbbbdx++23A/DUU0/Rp08fMjMzyc/PZ8KECVUeq3v37sycOROAAwcOcO2115KVlUW/fv2Y\nN29euW0ffPBB8vPzyczMpF+/fvzzn/8EYMmSJdxyyy28//77tGrViqysLACuu+46fvnLX5bt/5e/\n/IWvfe1rZGdnc9lll7Fx48aydRkZGTz++OOceOKJZGVlceutt1YZc2lpKb/5zW/Iz8+ndevWnH76\n6axfvx6A2bNnM2jQINq2bcsZZ5zB+++/X7bfU089Rc+ePcnMzKRnz548//zzVcYuIiIi6aeETkQa\nhKuuuorJkyezd+9ewEtwXnrpJa6++moAOnbsyJtvvklxcTETJ07kjjvu4OOPP672uOPGjWPlypWs\nXLmSwsJCJk2aVG59fn4+7733HsXFxdxzzz1cc801bN68mZNOOok///nPnHXWWezevZvt27cfdeyZ\nM2cyduxYXn75ZTZu3Ehubi5XXXVVuW3eeOMNPvzwQxYsWMDf//53pk6dWmmcv/3tb3nxxReZMmUK\nu3bt4sknn6R58+bs2LGDiy++mNtvv51t27Zxxx138B//8R/s2LGDffv2cdttt1FYWEhxcTGzZ8+m\nf//+NYpdRERE0iNQCd3+/X5HICJRlZuby8CBA3n11VcBmDFjBi1atOD0008H4Bvf+AbdunUD4Lzz\nzmP48OG888471R73pZde4he/+AWtW7cmJyeHH/7wh+XWf+tb36Jjx44AXHHFFXzta19j7ty5NYr5\nueee4/rrr+fUU0+lSZMm3H///bz//vusWbOmbJuf//zntGrViq5duzJkyJAqk9AnnniC++67j/z8\nfABOPvlk2rZtyxtvvMGJJ57I6NGjycjI4KqrruKkk07i9ddfB6BRo0YsXLiQAwcO0LFjR3r37l2j\n2EVERCQ9AjVtwfr1EHvWEJGIsnuT0+/K3eNqvc+oUaN4/vnnueaaa3j++ecZPXp02brJkyfzq1/9\nis8//5zS0lL279/PKaecUu0xN2zYQJcuXcq+VxyG+Omnn+Z3v/td2WhWe/fu5csvv6xRvBs2bODr\nX/962fcWLVrQrl071q9fT25uLkBZsgjQvHlz9uzZU+mx1q5dS48ePSo9R8WY8/LyWL9+Pc2bN+fF\nF1/k4Ycf5rvf/S7nnnsujzzyCL169apR/CIiIpJ6gUro1q1TQicSdXVJxJLliiuu4Cc/+Qnr16/n\n1VdfZc6cOQAcOnSIb3/72zz77LNceumlZGRk8J//+Z84V32snTt3Zu3atWWVq9WrV5etW7NmDTfd\ndBNvvfUWZ511FgADBgwoO251g4qccMIJ5Y63d+9etm3bVi6BrKnc3FyWL19Onz59jjrHP/7xj3LL\n1qxZwze+8Q0Ahg0bxrBhwzh48CB33XUXN910E7NmzdKAKCIiIgERqCaXsf75IiIpkZ2dzeDBg7nu\nuuvo0aNHWaXp0KFDHDp0iOzsbDIyMpg8eXKVfdEquvLKK7n//vvZuXMn69at47HHHitbt3fvXjIy\nMsjOzqa0tJSJEyeyaNGisvUdO3Zk3bp1HD58uNJjjxo1iokTJ/LJJ59w8OBBxo4dy5lnnlmnee6u\nv/567r77br744gsAFi5cyI4dO7joootYtmwZL7zwAiUlJbz44ot89tlnXHzxxWzZsoXXXnuNffv2\n0aRJE1q2bElGRkaNYhcREZH0CFRCt26d3xGISNSNHj2aGTNmlA2GAtCyZUvGjx/PFVdcQVZWFi+8\n8AKXXnpplcdIrE7dc8895Obm0r17d0aOHMmYMWPK1vXu3Zsf//jHnHnmmXTq1InFixdz7rnnlq2/\n4IIL6Nu3L506daJDhw5HnWfo0KH8+te/5vLLLycnJ4eVK1fywgsvVBpHZd8T/ehHP+LKK69k+PDh\ntG7dmhtuuIH9+/eTlZXFv/71Lx555BGys7N55JFHeOONN8jKyqK0tJRHH32UnJwcsrOzefvtt/nT\nn/5Uo9hFREQkPawmTYrSwczcD37gGD/e70hEpD7MrEZNFSX6qvpdiC1Xm80aMjOn/6ZERBqGutwj\nVaETEREREREJKSV0IiIiIiIiIRWohE6DooiIiIiIiNRcoBK6rVvhyBG/oxARESnPzEaa2RIz+9zM\nflbJ+jZm9oqZLTCzOWbWJ2HdbWa2MPbzw4Tlbc1sqpktNbNCM2udrusREZHoCFRCl50Nmzb5HYWI\niMhXzCwDeAwYAfQFRpnZSRU2GwvMd86dCnwHGB/bty9wPXAa0B+4xMziM7zfCUx3zvUCZgI/T/W1\niIhI9AQqoevSRf3oREQkcAYBy5xzq51zh4EXgIrzWvTBS8pwzi0FuplZe6A38IFz7qBzrgSYBVwe\n2+dSYFLs8yTgstRehoiIRFGgErqcHCV0IiISODnA2oTv62LLEi0glqiZ2SAgF+gCLALOizWvbA5c\nBMRnhu/onNsM4JzbBGhCPxERqbXGfgeQqEsXDYwiEnZ5eXnHnOBaGo68vDy/Q0inB4D/NbOPgIXA\nfKDEObfEzB4EpgF74surOIYmmxORpJkzx3u27tLF70gk1QKX0KlCJxJuq1at8jsEkWRbj1dxi+sS\nW1bGObcb+G78u5mtBFbE1k0EJsaW38dX1b5NZtbRObfZzDoBW6oKYNy4cWWfCwoKKCgoqPvViEiD\ncPvt8F//BXfc4XckcixFRUUUFRXV6xjmXDBeCJqZe+YZx5tvwnPP+R2NiIikipnhnAtNGdfMGgFL\ngaHARmAuMMo591nCNq2Bfc65w2Z2I3COc+7a2Lr2zrmtZpYLTAHOdM4Vxyp3251zD8ZGzmzrnLuz\nkvO7oNyrRSQctm2D9u3hllvgD3/wOxqpjbrcI1WhExEROQbnXImZ3QpMxet7/oRz7jMzu9lb7Sbg\nDX4yycxKgcV4I1vG/cPMsoDDwPecc8Wx5Q8Cfzez7wKrgSvTdEkiEnHTp0PLlrB8ud+RSDoEqkK3\nbJlj+HBYscLvaEREJFXCVqHzmyp0IlJb3/0utGgBkyfDF1/4HY3URl3ukYFK6Pbtc7RtC/v3g8ZU\nEBGJJiV0taOETkRqwzmv1VthIXz967BnDzRp4ndUUlN1uUcGatqC44/33iZ8+aXfkYiIiIiIhM/i\nxdCsGfTtCyecAGvW+B2RpFqgEjrw5qLT1AUiIiIiIrU3ZQqMGOG1dsvPV5PLhiBwCZ0GRhERERER\nqZvCQi+hA+jZUwOjNARK6EREREREImDfPm9C8Qsu8L6rQtcwBC6hU5NLEREREZHamzULBg6EzEzv\ne8+eSugagsAldKrQiYiIiIjUXrz/XFx+vppcNgTVJnRm1szMPjCz+Wa22Mx+U8k2o81sQeznXTM7\nJWHdSDNbYmafm9nPqjtfTo4SOhERERGR2krsPwfQowesXAmlpf7FJKlXbULnnDsIDHHODQBOAS4w\ns3MqbLYCON85dyrw/wETAMwsA3gMGAH0BUaZ2UnHOl+XLmpyKSIiIiJSG6tXw/btMGDAV8tatIA2\nbWDDBv/iktSrUZNL59y+2MdmsX12VFg/xzm3K/Z1DpAT+zwIWOacW+2cOwy8AFx6rHOpyaWIiIiI\nSO0UFsKwYZBR4eleA6NEX40SOjPLMLP5wCagyDn36TE2vwGYHPucA6xNWLeOr5K9SrVuDSUlUFxc\nk8hERERERGTKFBg58ujlSuiir6YVutJYk8suwPlmNriy7cxsCHAdUG1fuaqYqdmliIiIiEhNHT4M\nM2fC8OFHr9NcdNHXuDYbO+eKzewN4DRgVuK62EAoE4CRzrl4k8z1QG7CZl1iyyo1btw4AA4dgjfe\nKKB374LahCciIgFUVFREUVGR32GIiETWBx9A9+7QsePR6/Lz4eWX0x+TpI855469gVk2cNg5t8vM\njgcKgXudczMStskFZgD/7Zybk7C8EbAUGApsBOYCo5xzn1VyHhePZcwYb0LEa6+t59WJiEjgmBnO\nOfM7jrBIvD+KiFTm7ru9Kt0DDxy97t//hhtvhPnz0x+X1F5d7pE1qdB1BiaZmeE10XzGOTfDzG4G\nnHNuAnA3kAX8MbbdYefcIOdciZndCkyN7ftEZclcRRoYRURERESkZqZMgYcfrnxdvMmlc17XJome\nait06ZL4BvIPf4BFi+BPf/I5KBERSTpV6GpHFToROZYvv/SStq1boWnTyrfJzoZPP4UOHdIbm9Re\nXe6RNRoUJd1UoRMRERERqd60aTB4cNXJHGhglKgLZEKXk6OETkRERESkOoWFMGLEsbfR1AXRFsiE\nTtMWiIiIiIgcm3NeQlfZ/HOJlNBFWyATug4dYOdOOHjQ70hERERERILpk0+gRQuvSeWxqMlltAUy\nocvIgM6dYcMGvyMREREREQmmmjS3BFXooi6QCR1oYBQRERERkWOpaULXs6cSuigLbEKngVFERERE\nRCq3Zw/MnQtDhlS/bYcOXlemnTtTH5ekX2ATOg2MIiIiIiJSuaIiOO00aNWq+m3NvGaX6kcXTYFO\n6FShExERERE5Wk2bW8ZpYJToCmxCl5OjCp2IiIiISGVqm9BpYJToCmxCpwqdiIiIiMjRVqyA4mI4\n9dSa76OELrqU0ImIiIiIhEhhIQwf7k31VVNqchldgU3oOneGzZuhpMTvSEREREREgqO2zS1BFboo\nC2xC17QpZGV5SZ2IiIiIiMDhw94Il8OG1W6/nBzYvh327UtJWOKjwCZ0oIFRREREREQSvf++V23r\n0KF2+2UIIzozAAAgAElEQVRkQPfuXv87iZZAJ3TqRyciIiIi8pW6NLeMU7PLaFJCJyIiIiISElOm\n1D2h08Ao0RTohE5NLkVEREREPFu2eAnZWWfVbX9V6KIp0AmdKnQiIiIiIp5p02DIEGjSpG77K6GL\npkAndKrQiYiIiIh46tN/DtTkMqoCndCpQiciIiIiAqWl9U/o8vJgwwY4dCh5cYn/Ap3Q5eR4CZ1z\nfkciIiIiIuKfBQugdWtv6oG6atLEe75etSppYUkABDqha9kSmjWDHTv8jkRERERExD+FhTByZP2P\nk5+vZpdRE+iEDtTsUkRERESkvs0t4zQwSvQEPqGLN7sUERHxi5mNNLMlZva5mf2skvVtzOwVM1tg\nZnPMrE/Cup+b2WIz+8TM/mZmTWPL7zGzdWb2UewnCe/eRSSKdu+Gf/8bCgrqfywNjBI9gU/ounTR\nSJciIuIfM8sAHgNGAH2BUWZ2UoXNxgLznXOnAt8Bxsf2zQNuBAY4504BGgNXJez3qHNuYOxnSoov\nRURC6q23YNAgaNGi/sdShS56QpHQqUInIiI+GgQsc86tds4dBl4ALq2wTR9gJoBzbinQzczaA8XA\nIaCFmTUGmgMbEvazVAcvIuGXrP5z4FXolNBFS+ATOs1FJyIiPssB1iZ8XxdblmgBcDmAmQ0CcoEu\nzrkdwG+BNcB6YKdzbnrCfrea2cdm9lcza52qCxCRcEtW/zmAHj1g9WooKUnO8cR/gU/oVKETEZEQ\neABoa2YfAd8H5gMlZtYDuAPIA04AWprZ6Ng+fwR6OOf6A5uAR9MftogE3RdfwL59cPLJyTne8cdD\ndraer6Oksd8BVEeDooiIiM/W41Xc4rrElpVxzu0Gvhv/bmYrgBXARcB7zrntseWvAGcDzznntiYc\n4i/A61UFMG7cuLLPBQUFFCRjZAQRCYXCQhg+HCyJDbTjzS7z8pJ3TKmboqIiioqK6nUMcwGZtdvM\nXGWxbNvmdd7UXHQiItFgZjjnQtN3zMwaAUuBocBGYC4wyjn3WcI2rYF9zrnDZnYjcI5z7lozOxV4\nFjgdOAhMBOY55/5gZp2cc5ti+98BnO6cG00FVd0fRaRh+OY3YfRouOqq6retqRtu8AZZuemm5B1T\nkqMu98jAV+iysuDAAdi7Nzkj+4iIiNSGc67EzG4FpuJ1VXjCOfeZmd3srXYTgN7AJDMrBRYD18f2\nXWBmTwMfAiV4TTEnxA79kJn1B0qBVcDNabwsEQmBQ4dg1iyYODG5x9VIl9ES+ITO7KuBUU480e9o\nRESkIYpNKdCrwrLHEz7Pqbg+Yd3DwMOVLB+T5DBFJGJmz4ZevaBdu+Qet2dPmDcvuccU/wR+UBTQ\nwCgiIiIi0vBMmZK80S0TqUIXLaFI6DQwioiIiIg0NMmcfy5Rz56wfDmoe240hCKh69JFc9GJiIiI\nSMOxeTOsWgVnnJH8Y2dmQvPm3jkk/EKT0KlCJyIiIiINxdSpcMEF0DhFI16o2WV0hCKhiw+KIiIi\nIiLSEKSq/1xcvNmlhF8oEjpV6ERERESkoSgthWnTUpvQqUIXHUroREREREQCZP58b6qCvLzUnUMJ\nXXSEIqHr2BG2b/cmVxQRERERibLCwtRW50BNLqMkFAldo0ZeUrdxo9+RiIiIiIikVqr7z4EqdFES\nioQONDCKiIiIiERfcbHX5HLw4NSep107KCnxWsFJuIUmoVM/OhERERGJupkz4ayzvHniUsnMq9Kp\n2WX4KaETEREREQmIdPSfi1Ozy2gITUKnJpciIiIiEmXOpaf/XJwGRomG0CR0qtCJiIiISJQtWwaH\nD0Pfvuk5nyp00RCahE4VOhERERGJsnhzS7P0nE8JXTSEJqFThU5EREREoiyd/edATS6jwpxzfscA\ngJm5Y8Vy4AC0bg3790NGaNJQERGpyMxwzqXp/XP4VXd/FJFoOHgQ2reHVasgKys95ywthZYtYcsW\n70/xX13ukaFJjY47DjIzYetWvyMREREREUmud9/1+s6lK5kDr0jSo4eqdGEXmoQO1OxSRERERKIp\n3c0t4zQXXfiFKqHTwCgiIiIiEkV+JnQaGCXcQpXQqUInIiIiIlGzYQOsXQunn57+c2tglPBTQici\nIiIi4qOpU2HoUGjcOP3nVoUu/EKV0KnJpYiIiIhETWEhjBzpz7mV0IVftQmdmTUzsw/MbL6ZLTaz\n31SyTS8zm21mB8zsRxXWrTKzBbH959YnWFXoRERERCRKSkpg2jR/+s8BdO0Kmzd70yZIOFVb2HXO\nHTSzIc65fWbWCHjPzM5xzr2XsNk24AfAZZUcohQocM7tqG+wOTlK6EREREQkOj76CDp29AoXfmjc\nGHJzYeVKOOkkf2KQ+qlRk0vn3L7Yx2axfXZUWP+lc+5D4Eglu1tNz1OdLl28JpeaX1VEREREomDK\nFP+qc3E9e6rZZZjVKNEyswwzmw9sAoqcc5/W4hwOmGZm88zsxroEGZeZ6U2AuGtXfY4iIiIiIhIM\nfvafi9NcdOFW0wpdqXNuANAFON/MBtfiHOc45wYCFwHfN7Nz6xBnGQ2MIiIiIiJRsGsXfPIJnHee\nv3FoYJRwq9XgqM65YjN7AzgNmFXDfTbG/txqZq8Cg4B3K9t23LhxZZ8LCgooKCg4apv4wCh9+9Ym\nchER8UtRURFFRUV+hyEiEjgzZsDZZ8Pxx/sbR8+eXqVQwqnahM7MsoHDzrldZnY8MAy491i7JOzb\nHMhwzu0xsxbA8GPtm5jQVUUDo4iIhEvFF3T33nusW4iISMNRWOh//zlQhS7salKh6wxMMrP44CbP\nOOdmmNnNgHPOTTCzjsC/gVZAqZndBvQB2gOvmpmLnetvzrmp9Qk4PjCKiIiIiEhYOecNiHL77X5H\nAt27w5o1cOSIP5ObS/3UZNqChcDASpY/nvB5M9C1kt33AP3rE2BFXbp4w7uKiIiIiITV0qVeUheE\nqQKaNfOmTli71kvuJFySMp1AOmlQFBEREREJu3hzS7Pqt00HNbsMr9AldPFBUUREREREwioo/efi\nNBddeCmhExERERFJowMH4N134cIL/Y7kK5qLLrxCl9BlZ8PevbB/v9+RiIiIiIjU3jvvwMknQ5s2\nfkfyFTW5DK/QJXRmcMIJ6kcnIiIiIuEUtOaW4DW5VIUunEKX0IEGRhERERGR8ApyQlda6nckUluh\nTOjUj05EREREwmjdOti4EU47ze9IymvZEjIzvdgkXJTQiYiIiIikydSp3mAojRr5HcnRNDBKOIUy\noVOTSxERSSczG2lmS8zsczP7WSXr25jZK2a2wMzmmFmfhHU/N7PFZvaJmf3NzJrGlrc1s6lmttTM\nCs2sdTqvSUT8EcTmlnEaGCWcQpnQqUInIiLpYmYZwGPACKAvMMrMTqqw2VhgvnPuVOA7wPjYvnnA\njcAA59wpQGPgqtg+dwLTnXO9gJnAz1N9LSLir5ISmD4dhg/3O5LKaS66cAplQqcKnYiIpNEgYJlz\nbrVz7jDwAnBphW364CVlOOeWAt3MrD1QDBwCWphZY6A5EL+DXQpMin2eBFyW0qsQEd/Nm+c9x+bk\n+B1J5dTkMpxCmdCpQiciImmUA6xN+L4utizRAuByADMbBOQCXZxzO4DfAmvwErmdzrkZsX06OOc2\nAzjnNgEdUnYFIhIIQW5uCWpyGVaN/Q6gLjp1gq1b4cgRaBzKKxARkYh5APhfM/sIWAjMB0rMrAdw\nB5AH7AJeNrPRzrnnKjmGS1u0Dcy118LDD0P79n5HIkFz5Ig3QMm2bek535o18Oqr6TlXXcSnLnDO\nm/s5iD7+GJ5/Hh580O9IgiOU6VCTJpCdDZs2edU6ERGRFFqPV3GL68JXzSYBcM7tBr4b/25mK4AV\nwEXAe8657bHlrwBnA88Bm82so3Nus5l1ArZUFcC4cePKPhcUFFBQUFC/K2pA1q2DSZPgmmu8B3eR\nRHPnwpdfeglCOjRqBL17p+dcdZGVBRkZXoKbne13NJV78UX4wx/g17+Gpk39jqb+ioqKKCoqqtcx\nQpnQwVfNLpXQiYhIis0D8mMDnGzEG9RkVOIGsREq9znnDpvZjcDbzrk9ZrYUuNvMjgMOAkNjxwN4\nDbgWeBBvIJX/qyqAxIROaqew0Ptz+XIldHK0KVPgP/4DTj7Z70iCIz4wSlATuilTvKTzvfdgyBC/\no6m/ii/p7r333lofI5R96EADo4iISHo450qAW4GpwGLgBefcZ2Z2s5ndFNusN7DIzD7DGw3ztti+\nC4CngQ/x+tkZMCG2z4PAsFjSNxSv2aYkWWEh9OqlfkFSucJCGDnS7yiCJcgDo2zeDKtWwfe//9XL\nGolAhU5ERCTVnHNTgF4Vlj2e8HlOxfUJ6x4GHq5k+XZANaMUig8RP24c1LNFk0TQtm2wZAmcc47f\nkQRLkAdGmToVLrjAq6r+4AfwgF6DASGu0CmhExERkWOJDxF//vnBfUAV/0yf7v1uRKEfVjLFB0YJ\novgooWecAStXeuNpSIgTOjW5FBERkWOZMsV7+EscuU8kLv77IeUFtUJXWupV6EaM8AZIvOAC77uE\nOKFThU5ERESOJd4/qlUraNlSb/PlK855yYD6zx0tqAnd/PnQrh3k5XnfR45UP7q40CZ0OTlK6ERE\nRKRyO3bA4sVw7rne96A+pIo/Fi2C447zfi+kvI4dYd8+KC72O5LyKk7KPmIETJvmVe4aulAndBs2\nqPmEiIiIHG36dC+ZO+4473t8KHYRODo5kK+YBbMfXcV/s7w8b968+fP9iykoQpvQNW8OLVp4k0GK\niIiIJKrYPyrIQ7FL+qn/3LEF7QVIcTF89BEMHlx++YgR3r9lQxfahA40MIqIiIgczbmj5xdTk0uJ\n27sXPvjAG1RDKhe0FyAzZ8KZZ3oFnUTqR+cJdUKngVFERESkok8/hcaN4cQTv1oWxCZk4o9Zs+Dr\nX/cGy5HKBe0FSFUTwA8e7DW5DFp/v3QLdUKngVFERESkonhfG7OvluXnw7Jl6nsv6j9XE0F6AeJc\n1U1kmzf3KnczZ6Y/riAJdULXpYuaXIqIiEh5lT2wZ2V5f27fnv54JFjUf656QarQLVsGhw9D376V\nr1c/uggkdKrQiYiISNy+fTB7NgwdWn65WfD6BUn6rVoFO3dC//5+RxJsXbrA1q2wf7/fkXgvaIYP\nL19xTxTvR9eQq++hTug0KIqIiIgkevtt72G9deuj1wWp6iD+iCcHGaF+Ak69Ro2gWzdYudLvSKru\nPxfXt69XwVu2LH0xBU2of51VoRMREZFEx+ofFaR+QeIP9Z+ruSC8ADl40HtJc+GFVW9j5iXpDXm0\nSyV0IiIiEhnHemAPwgOq+OfwYW/wjOHD/Y4kHIIwF92770KfPl/1ga3KiBFK6EKrdWsoKdFQpSIi\nIgJr1nj9fr7+9crXB+EBVfwzZ473O9Chg9+RhEMQ+pzWtKJ64YVeJe/gwdTHFEShTujMNNKliIiI\neAoLYdiwqvtHBeEBVfyj5pa1E4SKdnX95+LatfMqee++m/qYgijUCR1oYBQRERHxVPfA3rkz7N7t\n/UjDo4Sudvzuc7phA6xdC6efXrPtG3Kzy9AndOpHJyIiIkeOwIwZx+4fZQY9eqhK1xBt3Qqffw5n\nneV3JOHRrZv3jH34sD/nnzrVm36kceOaba+ELsSU0ImIiMgHH0BenleFOxY1u2yYpk2DIUOgaVO/\nIwmPpk29/55Wr/bn/LWtqA4a5FX0NmxIXUxBFfqETk0uRUREpKYPf0HoFyTpp+aWdePXC5CSEi8J\nr82/WePGXkVv6tTUxRVUoU/oVKETERGRmj6w+90vSNLPOe8hXwld7fn1AuSjj6BjR+jatXb7NdRm\nl6FP6FShExERadi2bYPPPoNzzql+W1XoGp5PPoGWLb3+k1I7fk31UdeK6ogRXmWvpCT5MQVZ6BM6\nVehEREQatmnTYPBgaNas+m01F13DM2VKzYa+l6P51eRyypS6JXRdu3qVvQ8/TH5MQRb6hK5DB9i5\ns+FOJCgiItLQ1eZtfteusGULHDiQ2pgkONR/ru78qGjv2gULFsD559dt/4bY7DL0CV1GhjcCT0Mc\n0UZERKShq23/qMaNvdEwV65MbVwSDHv2wLx5UFDgdyTh1KOH999KaWn6zjljhtd8+vjj67a/ErqQ\nUrNLERGRhmnRIq+pZX5+zfdRs8uGo6jIm5i6ZUu/Iwmn5s0hKyu941XUt6J6/vlehW/nzuTFFHSR\nSOg0MIqIiEjDFO8fZVbzfTQXXcOh/nP1l84XIM7Vvf9c3PHHexW+GTOSF1fQRSKhU4VORESkYarL\n23yNdNlwqP9c/aXzBcjSpV7zzt6963echtbsUgmdiIiIhNLevfDBBzBkSO3201x0DcOKFV4fulNO\n8TuScEvnC5DCwtpX3CsTT+icS05cQReJhE5NLkVERBqeWbNg4EDIzKzdfqrQNQyFhTB8eP2Tg4Yu\nnU0uk1VR7d3bq/QtXVr/Y4VBJBI6VehEREQanrr2j+rWDdasgSNHkh6SBIj6zyVHuppcHjgA77wD\nQ4fW/1hm3r/9lCn1P1YYRCKhU4VORESk4anr2/xmzbwpj9asSX5MEgyHDnkjXA4b5nck4Rev0KW6\n+eI778DJJ0Pbtsk5XkPqRxeJhO6EE2DTJigp8TsSERERSYdVq7xhyfv3r9v+anYZbe+/DyeeCNnZ\nfkcSfm3aeC9Btm5N7Xni/eeSZehQePddr/IXdZFI6Jo29ebI2LzZ70hEREQkHQoLvepLRh2fZDQX\nXbRpdMvkSscLkGT/m7Vt61X83nkneccMqkgkdKBmlyIiIg1JfftHaS66aFP/ueRK9QuQdetgwwY4\n7bTkHreh9KOLTEKngVFEREQahsOH4a23vBEM60pNLqNr82ZYuRLOOMPvSKIj1S9Apk6FCy+ERo2S\ne9yG0o8uMgldTo4SOhERkYZgzhzo0QM6dKj7MTQXXXRNm+bNTdikid+RREeqX4Aku/9c3GmnwcaN\n0c8RIpPQdemiJpciIiINQTL62vTs6U08XVqanJgkONR/LvlS+QKkpASmT69fxb0qjRp5lb+pU5N/\n7CCJVEIX9exbREREktM/qkULaN3a67cj0VFaqoQuFVJZoZs3zxuxPicnNcdvCP3oqk3ozKyZmX1g\nZvPNbLGZ/aaSbXqZ2WwzO2BmP6qwbqSZLTGzz83sZ8kMPpEGRRERkVSp7l5mZm3M7BUzW2Bmc8ys\nT2z5ibH750exP3eZ2Q9j6+4xs3WxdR+ZmYZwqIGtW2HZMjjrrPofSwOjRM/HH3sjn3fr5nck0dK+\nvTe3386dyT92qhPw4cO9CmCUpzerNqFzzh0EhjjnBgCnABeY2TkVNtsG/AB4OHGhmWUAjwEjgL7A\nKDM7KRmBV6QKnYiIpEIN72VjgfnOuVOB7wDjAZxznzvnBjjnBgJfB/YCryTs96hzbmDsJ+LvkJNj\n2jQoKPCmLKovDYwSParOpYZZ6ppdpqr/XFxOjvczb17qzuG3GjW5dM7ti31sFttnR4X1XzrnPgSO\nVNh1ELDMObfaOXcYeAG4tH4hVy4+KEqqZ7EXEZEGpyb3sj7ATADn3FKgm5m1r7DNhcBy51zi60dL\nUcyRlcwHds1FFz1K6FInFS9AduyARYvg3HOTe9yKoj7aZY0SOjPLMLP5wCagyDn3aQ2PnwOsTfi+\nLrYs6Vq29Gax37Gj+m1FRERqoSb3sgXA5QBmNgjIBbpU2Oa/gOcrLLvVzD42s7+aWevkhRxN8f5R\nyXqbryaX0VJcDB9+CIMH+x1JNKXiBcj06V4yd9xxyT1uRVHvR1fTCl1prMllF+B8Mwvkfypqdiki\nIj55AGhrZh8B3wfmA2U9NsysCfBN4KWEff4I9HDO9cd7Yfpo+sINp08+gVatvCkLkkFNLqPlrbfg\nzDO9AW8k+VLxAiRdFdVzz4XFi6Nb+Glcm42dc8Vm9gZwGjCrBrusx3tLGdcltqxS48aNK/tcUFBA\nQUFBbcIrGxjllFNqtZuIiKRQUVERRUVFfodRH9Xey5xzu4Hvxr+b2UpgRcIm3wA+dM5tTdhna8L6\nvwCvVxVAfe+PUZHsh794nyDnvD5CEm5qbpla+fnw9NPJO55z3r/Z//xP8o5ZleOO85K66dPhiitS\nf77aSMY90lw1nc7MLBs47JzbZWbHA4XAvc65GZVsew+wxzn329j3RsBSYCiwEZgLjHLOfVbJvq66\nWKpz/fXem5kbb6zXYUREJIXMDOdcaB6fa3IvizWX3OecO2xmNwLnOOeuTVj/PDDFOTcpYVkn59ym\n2Oc7gNOdc6MrOX+9749RccEFcMcdcMklyTtmu3awZIk3ip+El3Negv5//wcnn+x3NNG0dq33nJ2s\nUeU//RQuughWrkzPC5X//V9YuBD++tfUn6s+6nKPrEmFrjMwycwMr4nmM865GWZ2M+CccxPMrCPw\nb6AVUGpmtwF9nHN7zOxWYGps3ycqS+aSRU0uRUQk2ZxzJZXdyxLvg0BvvHtlKbAYuD6+v5k1xxsQ\n5aYKh37IzPoDpcAq4OaUX0yI7dnjjVI3ZEhyjxvvF6SELty++AIOHoR+/fyOJLpycmD7dti7NznN\nWqdM8Sqq6aqOjxwJDz8czYp8tQmdc24hMLCS5Y8nfN4MdK1i/ylAr3rEWGM5OTB3bjrOJCIiDUll\n97IK98E5FdcnrNsHHJUuOOfGJDnMSHvrLTj9dG8QtGSK9wtKxrx24p94c8uoPagHSUYGdO8OK1Yk\npwpaWAg3p/E11oknQuPGXmWwb9/0nTcdajQoSlioQiciIhJNqeofpYFRokH959IjWQOj7N8Ps2fD\n0KH1P1ZNmUV3+oJIJXTxQVFEREQkWlL1wJ6qyZIlfQ4ehLffhgsv9DuS6EvWC5C334b+/aF1midr\nUUIXAqrQiYiIRM/y5bB7d2pGsVaFLvzeew969/YGuJHUStZcdPH+c+k2dKhXGdy3L/3nTqVIJXRZ\nWXDggNdZU0RERKIhXp3LSMFTixK68FNzy/RJVpNLv/7NWrf2KoNvv53+c6dSpBI6MzW7FBERiZpU\nPvx16OC9DN61KzXHl9RTQpc+yXgBsnYtbNkCA48acjE9otjsMlIJHajZpYiISJQcOgRFRTBsWGqO\nb6Z+dGG2cSOsWQODBvkdScOQm+v9nR86VPdjFBbC8OHQqFHy4qoNJXQhoAqdiIhIdLz/Pnzta6md\nJy5Z/YIk/aZO9fpFNa7JzMpSb02aeMWTVavqfgy/+s/FDRzoVQjXrPEvhmSLXEKnCp2IiEh0TJni\nTQicSsnqFyTpp+aW6VefFyBHjsCMGV6Fzi+NGnnnj1KVTgmdiIiIBFY6Htg1MEo4lZbCtGlK6NKt\nPi9A5s71mm127pzcmGoras0uI5fQqcmliIhINGzeDCtWwJlnpvY86kMXTh995DXF7drV70galvq8\nACksTH3FvSaGD/cqhUeO+B1JckQuoVOFTkREJBqmTYMhQ7x+O6mkCl04BSU5aGjq0+TS7/5zcZ07\ne5XCuXP9jiQ5IpfQqUInIiISDenoPwfes8O2bbB/f+rPJckTlOSgoalrk8tt2+Czz+Ccc5IfU12M\nHOn9DkVB5BK6Tp28X5j6DKcqIiIi/iot9UYwTMcDe6NG0K2b17xTwmHXLvj4Yzj/fL8jaXh69PBG\nuSwpqd1+06d7/17NmqUkrFqLUj+6yCV0jRpBx47eHBkiIiISTh9/DG3beolWOqjZZbjMnAlnnw3H\nH+93JA3Pccd5fRdr28UpaE1kzznHqxhu2+Z3JPUXuYQO1OxSREQk7NI9HL3moguXoCUHDU1tX4A4\nF7wpJpo18yqG06f7HUn9RTKh08AoIiIi4Zau/nNxmosuPJxT/zm/1fYFyKJFXgKVn5+6mOoiKv3o\nIpnQqUInIiISXsXF3pD0gwen75xqchken3/u9d/q3dvvSBqu2r4AiVfnzFIXU12MGOH11XXO70jq\nJ5IJnSp0IiIi4fXWW3DGGdCiRfrOqbnowiOoyUFDUtsXIEFtIpuf71UOFy3yO5L6UUInIiIigeJH\nX5u8PK91j0bJDr6gJgcNSW2aXO7dC3PmeHNKBo1ZNEa7jGRCpyaXIiIi4RTvH5XuB/amTeGEE2D1\n6vSeV2rnwAF45x0YOtTvSBq2nj29aT5q0lRx1iwYOBAyM1MfV11EoR9dJBM6VehERETC6Ysv4OBB\n6Ncv/efWwCjB9+673u9G27Z+R9KwZWZ6TaI3bap+26CNblnRkCHwwQdeJTGsIpnQnXACbNjgTUoq\nIiIi4VFYCMOH+9M/SgOjBF/Qk4OGpKb9ToP+b5aZ6VUQZ83yO5K6i2RCd9xx0Lo1bN3qdyQiIiJS\nG34+/GkuuuBT/7ngqMkLkFWrYPt2GDAgLSHVWdj70UUyoQM1uxQREQmbgwe9t+TDhvlzfjW5DLb1\n672f007zOxKBmr0AiVfcMwKecYS9H13A/3rrTgOjiIiIhMt773lzi7Vr58/51eQy2KZOhQsvhEaN\n/I5EoGYvQILe3DKuf3/YudOrKIZRZBM6VehERETCxe+Hvx49vAe6khL/YpCq+f37IeVV9wLk8GGY\nOdOr0AVdRobXMiCszS6V0ImIiEgg+P3AfvzxXnVQLXyCp6QEpk8PR3LQUFTX5HLOHO8lSceO6Yup\nPsLcjy6yCZ2aXIqIiITHxo3eHHBnnOFvHBoYJZj+/W/o3Nl7YS/B0K6dNw/d9u2Vr/f7BU1tDR/u\nVRQPH/Y7ktqLbEKnCp2IiEh4TJ3qTRbduLG/cWhglGAKW3LQEJgdu9ll2P7NOnb0Kopz5vgdSe1F\nNqFThU5ERCQ8gvLwp4FRgikovx9SXlVz0X35JXz+OZx9dvpjqo+wNruMbEIXr9A553ckIiIiciyl\npTBtWjAe2NXkMnh27ICFC+G88/yORCqq6gXItGlQUABNm6Y9pHpRQhcwmZneiDW7dvkdiYiIiBzL\nR2N5gCQAACAASURBVB9B+/aQm+t3JGpyGUQzZsC558Jxx/kdiVRU1QuQKVOC8YKmts4+26ssbt3q\ndyS1E9mEDtTsUkREJAyC9PAXb0KmFj7BoeaWwVXZCxDnvD6xYfw3a9rUqyxOm+Z3JLUT6YROA6OI\niIgEX5Ae2Fu39ipBW7b4HYmAlxwE6fdDyqusyeUnn0CLFt7LkTAKY7PLSCd0qtCJiIgE265d8PHH\ncP75fkfyFQ2MEhxLlnijKfbq5XckUpnOnWH3btiz56tlhYUwcqR/MdVXPKErLfU7kpqLdEKnCp2I\niEiwzZzp9Vtp3tzvSL6igVGCY8oULzkw8zsSqYyZN9R/YrPLIDWhrouePaFVK6/SGBZK6ERERMQ3\nQXz408AowaHmlsGX+AJkzx6YNw+GDPE3pvoKW7PLSCd0anIpIiISXEHtH6Uml8Gwfz/Mnu1NOC/B\nlfgCpKgITjsNWrb0NaR6U0IXIKrQiYiIBNfnn8ORI9Cnj9+RlFfVZMmSXu+8A6ec4g1UI8GV+AIk\n7P3n4oYM8SqNiX0DgyzSCZ0qdCIiIsEVf/gLWv8oVeiCId5/ToItscllEJtQ10XLlnD66fDWW35H\nUjORTuiys72Rd/bv9zsSEREJMzMbaWZLzOxzM/tZJevbmNkrZrbAzOaYWZ/Y8hPNbL6ZfRT7c5eZ\n/TC2rq2ZTTWzpWZWaGYNrg4R1Ie/7Gyvcrhjh9+RNGxBbI4rR4s3uVyxwnvuPuUUvyNKjjA1u4x0\nQpeRoSqdiIjUj5llAI8BI4C+wCgzO6nCZmOB+c65U4HvAOMBnHOfO+cGOOcGAl8H9gKvxPa5E5ju\nnOsFzAR+nvKLCZADB+Ddd+HCC/2O5Ghmanbpt7VrvbkABw70OxKpTteusHkzvPYaDB/uPX9HgRK6\nAFFCJyIi9TQIWOacW+2cOwy8AFxaYZs+eEkZzrmlQDcza19hmwuB5c65eO/uS4FJsc+TgMtSEXxQ\nvfsu9O0Lbdv6HUnl1OzSX1Onesl+o0Z+RyLVadwYcnPhz3+OVkX1lFO8imMYXuxEPqHTwCgiIlJP\nOcDahO/rYssSLQAuBzCzQUAu0KXCNv8FPJ/wvYNzbjOAc24T0CGJMQde0AdP0Fx0/lL/uXDJz/cG\nORo+3O9IkicjIzxVusZ+B5BqqtCJiEgaPAD8r5l9BCwE5gMl8ZVm1gT4Jl4zy6q4lEYYMFOmwF/+\n4ncUVcvP96qIQbZ3r9dvKWqcgxkzYPx4vyORmurZ02se275iu4SQGzECnnoK/n/27js8qmpr4PBv\npxB675BAQu89lAQICSAoCoKo4AUEC95L0U/lil1ERbherwXFCmJFmgUEQg0ldEjooUiABAi9EyBl\nf3+cBEJIyEwyM2fKep9nHjOn7XU05syavffaHTuaHcnduX1CV726e/6xE0II4TBHMXrcMlXP2HaT\n1voSMCzzvVIqHsj69OkJbNFan8qy7YRSqpLW+oRSqjJwMrcA3nrrrZs/h4WFERYWZv1dOJFjx4wv\nW9u0MTuS3NWubXyQc2avvAJz50Lp0mZHYnvdu0OVKmZHISzVtSs0aWJ2FLZ3zz3wv//BwIH2a+Py\n5SiuXIkq0DWU1s7xhaBSStsjltmz4eefjT94QgghzKeUQmvtZIXqc6eU8gb2AhHAcWAjMEBrvSfL\nMaWAq1rrFKXUU0CI1vrxLPt/ARZpradn2TYROKu1nphRObOM1vqOHjx7PR/NNG0aLFwIM2eaHUnu\nEhMhONhIPp1V3brGv8Pmzc2ORAhhK/l5Rrp9D50MuRRCCFEQWus0pdRIYDHG3PNvtdZ7lFLDjd36\nK6ABMF0plQ7sAp7IPF8pVRSjIMrT2S49EZiplBoGHAYetv/dOAdnnz8HULUqnD9vDGssVszsaO50\n8CBcvOg+JeKFEPnn9j10CQnQrp0kdUII4SxcrYfObO7WQ5eWBhUrwrZtxrQIZ9aoEfzyi3MmTVOm\nwPr1MH163scKIVxHfp6Rbl/lsnJlOHXKWCBUCCGEEObavNmYG+XsyRzcWjDZGcmi20KITG6f0Pn6\nQvnykJRkdiRCCCGEcKVExFnXoktJgago6NbN7EiEEM7A7RM6kLXohBBCCGfhCvPnMjnrWnTr1hnJ\npruViBdC5I9HJHRSGEUIIYQw37lzsH2786/plMlZh1zKottCiKw8IqGTHjohhBDCfMuWQWgoFC5s\ndiSWcdYhl640bFUIYX+S0AkhhBDCIVwtEQkIMObgX79udiS3nDxp9Bq2a2d2JEIIZ+ERCZ0MubSt\nFfErGL1wtNlhCCGEcCFau9b8OQAfH/D3h0OHzI7kliVLoEsXo+ibEEKAhyR00kNnO4fPH2bg3IFM\njZnKjbQbZocjhBDCRezZA0pBvXpmR2IdZyuMIvPnhBDZ5ZnQKaX8lFIblFIxSqldSqn3cjnuE6XU\nfqVUrFKqRZbth5RS2zLO32jL4C0lPXS2kZySTN+Zffl3h39Tp1wdth7fanZIQgghXETmcEvlYkvK\nO1NhlPR0WLzYtYatCiHsL8+ETmt9HeiitW4BNAXClVIhWY9RSvUEammt6wDDgSlZdqcDYVrrFlrr\nYNuFbrnMhE5rM1p3D1prnvnrGeqXr89z7Z4jxD+E6CPRZoclhBDCRbja/LlMzlQYZds2KF0aatY0\nOxIhhDOxaMil1vpqxo9+Geecy3ZIb+D7jGM3AKWUUpUy9ilL27GXokWhWDE4fdrMKFzbZ5s+IzYp\nlq/v/xqllJHQJUhCJ4QQIm/JyRAdDRERZkdiPWcacumqSbEQwr4sSrSUUl5KqRggCYjSWu/Odkg1\nICHL+6MZ2wA0sEQptUkp9VRBA84vGXaZf6sPr+adVe/w2yO/UdS3KAChAaGsObIGLd2eQggh8rBq\nFTRrZvQuuRpnGnIp8+eEEDmxtIcuPWPIZXWgk1KqsxVthGitWwL3AiOUUqH5iLPApDBK/hy9eJRH\nZj/C9D7TCSoTdHO7fyl/CvsU5sBZJ/naUgghhNNy5Z6lwEA4fBjS0syN49Il2LIFOlvzCUwI4RF8\nrDlYa31RKfUX0BpYmWXXUcA/y/vqGdvQWh/P+OcppdRvQDCwJqfrv/XWWzd/DgsLIywszJrw7kp6\n6Kx3PfU6/Wb2Y1TwKO6pfeeTOLOXrk65OiZEJ4RwFVFRUURFRZkdhjBRZCRMm2Z2FPlTuDBUrAgJ\nCebOXVuxAtq2NaaQCCFEVnkmdEqp8kCK1vqCUqoI0A0Yl+2wP4ERwK9KqXbAea31CaVUUcBLa31Z\nKVUM6J7DuTdlTehsTXrorDd64Wiql6zO2NCxOe7PnEc3tMVQB0cmhHAl2b+gGzcu18eAcEMJCXDi\nBLRqZXYk+ZdZGMXMhM6VezmFEPZlyZDLKsCKjDl064E/tdbLlFLDlVJPA2itFwDxSqkDwJfAvzLO\nrQSsyXLuPK31YpvfhQUkobPO11u+Zk3CGqb1nobKpcZ0Zg+dEEIIkZvISOjWDby9zY4k/5yhMMqi\nRZLQCSFylmcPndZ6B9Ayh+1fZns/Modj4oHmlgZz5caV295nTyQUdyYWlh5TsQokHIOUNIWvt6+l\nIXmkDYkbeHX5q6wZtoYSfiVyPa5xxcYkXU7i9NXTlC9a3oERCiGEcBWRkXDffWZHUTBmF0Y5cMCo\nFNqkiXkxCCGcl1Vz6Oyt4gcVb/6cvXqi5s5qinkdk3W/1pDaFgq/m8bywcvpXFNmFeck6XISD816\niG8f+Ja65ere9VhvL2/aVm/L2oS1PFDvAQdFKIQQwlWkpsKyZfDxx2ZHUjC1a8P69ea176qLsgsh\nHMOpErorr1zJ+6B8OnfOGPv+3O9vsvDAQknocpCSlsLDsx7miRZPcH+9+y06J9TfGHYpCZ0QQojs\nNm0Cf3+oWtXsSAqmVi1ze+giI2HgQPPaF0I4N1MX/Hak0qWNbwrbVgpnWfwys8NxSi8sfoFShUvx\nRuc3LD4nJEAWGBdCCJEzdynkkZnQmbH06o0bEBVlzEMUQoiceExCp5RRGKVaejviTsdxLvmc2SE5\nle+3fc+iA4v44cEf8FKW/1q0rdaW2KRYrqVes2N0QgghXJG7FPIoUcJ4HT/u+Lajo6FBAyhXzvFt\nCyFcg8ckdGCsRXcqyY8O/h1YeXhl3id4iK3Ht/LC4hf47ZHfKF24tFXnFitUjIYVGrL52GY7RSeE\nEMIVnT0Lu3dDaKjZkdiGWcMu3aWXUwhhPx6V0GUuXRBeM5xlB2XYJcDpq6fp+2tfptw3hUYVG+Xr\nGpnz6IQQQohMS5dCx47g52d2JLaRuRado0lCJ4TIi0cldNWqwdGjEBEUwfJDy80Ox3Sp6ak8OvtR\nBjQewEMNH8r3dWQenRBCiOwiI6FHD7OjsB0z1qJLSoJDh6BtW8e2K4RwLR6V0GX20LWo3ILjl45z\n/JIJg+GdyMtLX8bby5t3wt8p0HVC/ENYm7CWdJ1uo8iEEEK4Mq3dZ/5cJjPWolu8GCIiwMepapIL\nIZyNRyZ03l7edK7ZmeXxnttL9+vOX5mzZw4/9/0Zby/vAl2rSokqlC5cmrjTcTaKTgghhCvbtQsK\nFYI6dcyOxHbMGHIpwy2FEJbwqIQuc8glQERghMcmdNtPbGfkwpHMfWQu5YrapmxWiH8I0Udk2KUQ\nQgj3XAg7c8ilo5YuSE+HJUskoRNC5M2jErrMHjowErpl8cvQZiwqY6Jzyefo+2tfPu7xMc0rN7fZ\ndUMDQlmTIIVRhBBCuN/8OYCyZY0E9exZx7QXE2MsVRAQ4Jj2hBCuy6MSuooV4fx5uH4d6pevz420\nGxw8d9DssBwmLT2Nx+Y+xv1172dgk4E2vbb00AkhhAC4ehXWrYPwcLMjsS2lHDvsctEi90uKhRD2\n4VEJnZcXVKkCx46BUorwwHCPGnb5ZtSbXE25yqRuk2x+7QYVGnA2+SxJl5Nsfm0hhBCuY+VKaNEC\nSpY0OxLbc+RadDJ/TghhKY9K6ABq1ICtW42fM4ddeoLf437nh+0/MLP/THy9fW1+fS/lRQf/DtJL\nJ4QQHs6dExFH9dBdvGgMuezUyf5tCSFcn8cldK+9Bs89Z4yBz+yhc/dy+3tO7eHpeU8zu/9sKhar\naLd2QvxlPTohhPB07jh/LpOj1qJbvhzat4eiRe3flhDC9XlcQte9Ozz4IIwYATVK16CkX0l2ndxl\ndlh2c/H6RR789UHe7/o+baq1sWtboQGhrDkihVGEEMJTHT4MZ84YQy7dkaPWonO3NfyEEPblcQkd\nwPvvQ2ws/PKLew+7TNfpDP5tMOGB4QxrMczu7bWu2ppdp3Zx5cYVu7clhBDC+URGQrduxpx1d+SI\nIZdau3cvpxDC9tz0T+7dFS0KP/4Izz4LTUuGu21C997q9zh19RQf9fjIIe0V8S1Cs0rN2Hh0o0Pa\nE0II4Vzcef4cQOXKcPkyXLpkvzb274eUFGjY0H5tCCHci0cmdACtWhkJ3YwJ4aw+vJrU9FSzQ7Kp\nBfsXMGXzFGb1n0Uh70IOa1fm0QkhhGdKSTHmfnXvbnYk9qOU/StduuOi7EII+/LYhA7gpZcg/VIF\niqbUYPOxzWaHYzMHzh5g6B9DmfnQTKqWqOrQtmUenRBCeKYNG6BmTaMXy53Ze9ilzJ8TQljLoxM6\nHx/4/ns4tzWcn9e7x7DLyzcu8+CvD/JW57cICQhxePsd/DuwPnE9aelpDm9bCCGEedx9uGUme/bQ\nXb8Oq1dD1672ub4Qwj15dEIHxh/mpyIimLZiOdevmx1NwWiteeLPJ2hTtQ3PtH7GlBgqFKtApeKV\n2HXKfSuHCiGEuJOnJHT27KFbswYaNYKyZe1zfSGEe/L4hA5g/JOdSC63gVfeTDY7lAL5YO0HHDx3\nkM/v+xxl4uD7UH8ZdimEEJ7k9GnYuxdCHD8wxOHsuRadpyTFQgjbkoQOKFW4JM2rNmHa0rWsWmV2\nNPkTfSSaD9d/yNyH51LYp7CpsYQESGEUIYR7UUr1UErFKaX2KaVeymF/aaXUXKXUNqXUeqVUwyz7\nSimlZiml9iildiml2mZsf1MplaiU2prxctlC9UuWQOfOUMhxNbhMY8+16GT+nBAiPyShy9CjbgRd\nn1rOkCFw8aLZ0VhvyuYpvNbxNfxL+ZsdihRGEUK4FaWUFzAZuAdoBAxQStXPdtgrQIzWuhkwBPgk\ny76PgQVa6wZAM2BPln0faq1bZrwW2e0m7MyTepb8/eHkSbh2zbbXPXYMEhOhTRvbXlcI4f4kocsQ\nERjBEe9ldO8Oo0ebHY11klOSmb9vPv0a9jM7FADqlK1DckoyCRcSzA5FCCFsIRjYr7U+rLVOAWYA\nvbMd0xBYDqC13gvUVEpVUEqVBDpqradl7EvVWmf92tDli9NrDYsXe05C5+0NNWpAfLxtr7t4sVEM\nxcfHttcVQrg/SegytPdvz65Tu3jjvQtER8OcOWZHZLnIvyNpWaUllYs7R61opZQMuxRCuJNqQNZv\nqBIztmW1DegLoJQKBgKA6kAgcFopNS1jWOVXSqkiWc4bqZSKVUp9o5QqZb9bsJ8dO6BoUWMooqew\nR2EUT+rlFELYliR0GQr7FKZttbZsPb2KH36Af/0Ljh83OyrLzNw1k4cbPWx2GLcJ8Q8h+ogkdEII\nj/E+UEYptRUYAcQAaYAP0BL4TGvdErgKjM0453MgSGvdHEgCPnR41DbgifO+bF0YJS3NmIfoaf8e\nhRC2IR37WUQERrAsfhkf9bif4cNh2DBYsABMLBiZp+SUZBbsX8BHPT4yO5TbhPiH8POOn80OQwgh\nbOEoRo9bpuoZ227SWl8ChmW+V0rFAweBYkCC1npzxq7ZwEsZ55zKcomvgXm5BfDWW2/d/DksLIyw\nsDDr78JOIiPh2WfNjsKxateGfftsd70tW6BSJahe3XbXFEK4hqioKKKiogp0DUnosggPDOfJeU8C\n8PrrRvnlKVOM3jpntfDAQlpXbU3FYhXNDuU2Lau0ZN+ZfVy6fokSfiXMDkcIIQpiE1BbKVUDOA48\nCgzIekDGcMmrWusUpdRTwEqt9WXgslIqQSlVV2u9D4gAdmecU1lrnZRxib7AztwCyJrQOZMrV2Dj\nRujSxexIHKt2beMLX1uJjIQeLlvjVAhRENm/pBs3bpzV15Ahl1m0qtqKhAsJnLh8Al9f+OEHeOMN\nY20dZ+WMwy0B/Hz8aFmlJesT15sdihBCFIjWOg0YCSwGdgEztNZ7lFLDlVJPZxzWANiplNqDUQ0z\na5/VaOAnpVQsRpXL9zK2T1JKbc/Y3hn4Pwfcjk1FRUHr1lDCw763s/WQS5k/J4QoCKW1NjsGAJRS\n2hli6T2jNwMaD+DRxo8C8PnnMG0arF0Lvr4mB5fN1ZSrVP1vVfaP2k+FYhXMDucOryx7BV8vX8Z1\nsf6bBiGE+1JKobV24sHszsVZno85GTUKqlWDsWPzPtadXL8OpUrBpUsF/2xw/vytpRCKFMn7eCGE\ne8vPM1J66LIJrxnOsoPLbr7/5z+hfHl45x0Tg8rFwv0LCa4W7JTJHGQURpFKl0II4bY8tWfJzw+q\nVIEjRwp+rWXLjCkekswJIfJLErpsIoIiWH5o+c33SsHUqfDll7DeyUYPztztnMMtM3Xw78DGoxtJ\nTU81OxQhhBA2Fh8PFy5As2ZmR2KOWrXg778Lfh2ZPyeEKChJ6LJpVKERl29c5tD5Qze3ValiDL0c\nNAguXzYvtqyu3LhC5IFIHqz/oNmh5KpMkTIElApgW9I2s0MRQghhY5m9c14e+knCFmvRae25vZxC\nCNvx0D/DuVNKER54+7BLgL59jSERL7xgUmDZLNi/gHbV21GuaDmzQ7krGXYphBDuyRPXn8vKFoVR\n4uKMpK5+fdvEJITwTJLQ5SAi8PZhl5k++QQWL4b5800IKpuZu2fSv2F/s8PIU2hAKGuOrDE7DCGE\nEDaUkmJUuOzWzexIzFO7dsGHXGb2zjnzerdCCOcnCV0OIgIjWB6/nOxVxUqWhO+/h6efNqpRmeXy\njcss/nsxfer3MS8IC4UEGD10zlqhTQghhPXWrTN6qCo61xKoDmWLIZcyf04IYQuS0OUgsEwghX0K\ns/vU7jv2dewIgwcbSZ1ZOcpf+/6ig38Hpx9uCRBYOhCt9W1zEoUQQrg2mfcFQUFGYZj09Pydn5wM\n0dEQEWHbuIQQnkcSulxk9tLlZNw4OHTIqH5phpm7Z/JwQ+etbpmVUupmL50QQgj3sGiR9CwVKwal\nS8OxY/k7f/VqaNLEuIYQQhSEJHS5iAiMYFn8shz3+fnBjz8aC6naomSxNS5dv8TSg0tdYrhlplB/\nmUcnhBDu4uRJ49nXvr3ZkZivIMMupZdTCGErktDloktgF1YeXpnrGmqNG8MrrxjDL1MduMza/H3z\nCQ0IpUyRMo5rtICkh04IIdzHkiUQFga+vmZHYr6CrEUn8+eEELYiCV0uKhevTLUS1Yg5HpPrMc8+\nC4ULw6RJjovLlYZbZmpWqRmHzh/iXPI5s0MRQghRQNKzdEt+e+gSEyEpCVq1sn1MQgjPIwndXdxt\n2CUYi6l+9x189BFs2WL/eC5ev8jy+OX0rt/b/o3ZkK+3L8HVglmXuM7sUIQQQhRAerqxfI8kdIb8\nrkUXGQldu4K3t+1jEkJ4Hkno7iI8MPyuCR2Avz98/DH84x9GxSp7mrd3Hp1qdKJ0YdebQR3iH0L0\nERl2KYQQrmzbNmMJn6AgsyNxDvldi056OYUQtiQJ3V10rtmZ9YnruZ56/a7HDRgAzZvDSy/ZNx5X\nHG6ZKTQglDUJUhhFCCFcmSQit8vsobNmGaO0NFi6FLp3t19cQgjPIgndXZQuXJqGFRpaNFTw88/h\n99+NoSj2cPH6RaIORfFAvQfs04Cdtaveji3HtnAj7YbZoXikC9cusP/Mfvn3L4QoEEnoblemjFEc\n5vRpy8/ZtAmqV4dq1ewXlxDCs/iYHYCzC68ZzrKDywirGXbX48qUgWnTYMgQ2L4dypa1bRx/7v2T\nsJphlCpcyrYXdpCSfiWpU64OW49vpV31dmaH41EW7F/Ak38+iZ+PH8cuHaNSsUoElQkisEwgQaUz\n/lkmiKAyQVQqVgmllNkhCyGc0KVLsHmzUeFS3JJZGKVCBcuOX7RIkmIhhG1JQpeHiKAI3ox6k/GM\nz/vYCOjfH555Bn79FWz5uXjmrpk80ugR213QBJnz6CShc4wrN67w4uIXWXhgIb/0+4XONTuTmp5K\n4sVEDp47SPy5eA6eO8hf+/+6+fPlG5cJLBNIYOlbSV7mz4FlAileqLjZtyWEMMmKFRAcDMXlz8Bt\nMhM6S9fli4yE8Xl/pBBCCItJQpeHEP8QtiVt49L1S5TwK5Hn8e+9B61bw08/GYVSbOH8tfOsPLyS\nH/v+aJsLmiQ0IJSZu2byAi+YHYrb23h0I4N+G0Tbam3Z9sy2mz27Pl4+1Cxdk5qla0LgneddvnH5\nZnIXf97459KDS4k/H0/8uXiKFyp+q0cvS+9eYOlA/Ev54+Mlf1KEcFcy3DJn1qxFd+4c7NoFoaH2\njUkI4Vnk01ceivgWoU21Nqw6vIr76t6X9/FF4McfjcnOnTpBQEDBY/hz7590qdmFkn4lC34xE4X4\nhzB64Wi01jKsz05S01N5b/V7fLbpMyb3nEz/Rv2tOr94oeI0qdSEJpWa3LFPa82JKyc4eO7gzR6+\n6IRoftz+IwfPHeTElRNUL1mdwNKBDG0+lMeaPmar2xJCOIHISJgzx+wonE/t2pbPn1+6FDp2NNaw\nFUIIW5GEzgIRgREsj19uUUIH0KIFvPCCMZ9u2TJjvbqCmLlrJgObDCzYRZyAfyl/CvsU5sDZA9Qp\nV8fscO6QkpZCj5960LlGZ55v/7zLDS88cPYAg34bRIlCJdj69FaqlbTtjHulFJWLV6Zy8cp08O9w\nx/7rqdc5cuEIsUmxjFw4kv6N+lPIu5BNYxBCmOPAAbhyBZo2NTsS52PNWnQyf04IYQ9S5dICeS0w\nnpMxY4zSxE8+aUwkz69zyedYfWQ199e9P/8XcSKhAaGsOeKcyxf8vONnrqZcZe+ZvdT9tC5TNk0h\nJS3F7LDypLXm6y1f0+6bdgxoPIBF/1hk82TOEn4+ftQpV4f+jfpTv3x9/tr3l8NjEELYR+ZwSxlc\ncSdL16LTWoatCiHsQxI6C7Su2pr48/GcunLK4nO8vWHePOPh16wZREXlr+0/9v5BRGCERfP3XEGI\nfwjRCc63wHhqeirvrn6XCRET+KnvT8wfOJ/f4n6j0eeNmL17NtqaRYYc6OSVk/T5tQ+fb/6clY+v\nZHTb0Xgp8/+3HtZ8GN/GfGt2GEIIG5FEJHcVK8K1a3Dhwt2P270bfHygbl3HxCWE8Bzmf/JzAb7e\nvnQM6EjUoSirzitVCr79Fj75BB57DJ57Dq5eta7tmbtm8nAj11xMPCfO2kM3Y+cMKhevTOcanQFo\nWaUliwct5rN7P+O91e/R7tt2rDy00uQobzd/33yafdGMBuUbsOHJDTSq2MjskG56qOFDRCdEc+zS\nMbNDEUIU0I0bsHIldOtmdiTOSSnLCqNERkKPHtLLKYSwPUnoLJSfYZeZevUy1qY7edKYX7cu73XK\nATibfJbohGh61e2Vr3adUeOKjUm6nGRVb6e9paWn8c6qd3ij8xt3FGvpVqsbm5/ezHNtn2PoH0Pp\n9XMvdpzYYVKkhis3rjB83nBGLRzFrw/9yvtd33e6uWrFChXjoQYP8f22780ORQhRQGvXQr16UL68\n2ZE4r8ylC+5G5s8JIewlz4ROKeWnlNqglIpRSu1SSr2Xy3GfKKX2K6VilVLNs2zvoZSKU0rtU0q9\nZMvgHSk8MDzfCR1AuXLw88/GsgYPPghjx8L163c/5/e43+kW1M3linPcjbeXN+2qt2NtwlqzQ7lp\n1u5ZlCtajojAiBz3eykvBjQZwJ4Re+heqztdf+jK478/zpELRxwcKWxI3EDzL5tzLe0ascNjNPBV\nggAAIABJREFU6VSjk8NjsNQTLZ9gasxUpx2uKoSwjCQiecsrobt61fgyNzzccTEJITxHngmd1vo6\n0EVr3QJoCoQrpUKyHqOU6gnU0lrXAYYDX2Rs9wImA/cAjYABSqn6tr0Fx2hSqQnnr50v8If4fv2M\n3rp9+4z16rZuzf1YdxtumcmZ5tGl63TGrxrPG53u7J3Lzs/Hj9FtR7N/1H78S/rT4ssWjFk8hrPJ\nZ+0eZ2p6Km9FvcUDMx5gQsQEpveZfnNtOWfVtlpbfLx8nHKIrRDCcjJ/Lm95DblctQqaNzemYggh\nhK1ZNORSa50588sv45xz2Q7pDXyfcewGoJRSqhIQDOzXWh/WWqcAMzKOdTleyovwwHCWxy8v8LUq\nVjTW8hk71hhPP24cpGQrpnjm6hnWJa7jvjqWLZXgSkICnCehm7N7DsULFad7re4Wn1PSryTjw8ez\n8587uXTjEvUm12NS9CSSU5LtEuP+M/sJmRrCusR1xAyP4aGGD9mlHVtTSjGsxTCmxk41OxQhRD6d\nOAHx8dC2rdmROLe8eugkKRZC2JNFCZ1SykspFQMkAVFa693ZDqkGJGR5n5ixLbftLim8ZsGGXWal\nlFEoJSYGNmyAdu1g585b+3+L+417at1DsULFbNKeM2lbrS2xSbFcS71mahzW9M7lpEqJKnzR6wvW\nDF3DxqMbqTu5LlNjppKWnmaT+LTWfLXlKzpM7cA/mvyDhY8tpGqJqja5tqMMajqI3/b8xqXrBVi7\nwwNprXl56cumDOsVIqvFiyEiAnx9zY7EueW1Ft2iRcYXuEIIYQ+W9tClZwy5rA50Ukp1zuMUt6zh\nFBFkLDBuyzlB1arBX3/BP/8JXbrAxInG+nWzds9yy+GWYBTMaFihIZuPbTY1jj/i/qCQdyHurXNv\nga5Tr3w9Zj88m1n9Z/Fd7Hc0/aIp8/bOK9DvyYnLJ3hgxgN8sfkLVj2+ilFtRznFcgTWqlS8El0C\nuzBz10yzQ3Epiw4sYtLaSby+4nWzQxEeTubPWaZ6dTh7NudK1keOwOnT0LKl4+MSQngGqz4haq0v\nAn8BrbPtOgr4Z3lfPWPbUSAgh+05euutt26+ovK7cJsd1SpTCx8vH/ae2WvT6yplLEC+aZMxLKNt\nl9OsPbK+wImGMwv1N3f5Aq01b696O8fKlvnVrno7Vj6+koldJ/Lyspfp/F1n1iVYWNI0i3l759H8\ny+Y0qdiE9U+up0GFBjaJzyyyJp110nU6Y5eNZVrvaSw6sIidJ3fmfZKTi4qKuu3vu3AN6emwZIkk\ndJbw8oLAQDh48M59kZHGkg9ervednBDCRfjkdYBSqjyQorW+oJQqAnQDxmU77E9gBPCrUqodcF5r\nfUIpdRqorZSqARwHHgUG5NaWsz/olVJGtcuDy6hf3va1XWrWhKVL4bEPf2NHbA++/rwoo0a550Mg\nJCCE6dumm9b+vH1GD9r9de+36XWVUvSq24uetXvy/bbveXj2w7Sp2ob3It7L83fm8o3LPB/5PEsP\nLmVW/1mEBoTaNDaz9KzTk6fnP82eU3tcPjl1hJ93/Ewx32IMajqIM1fP8OryV/nj0T/MDqtAwsLC\nCAsLu/l+3LjsjxDhjGJioGxZqFHD7EhcQ2ZhlMaNb98eGQn32/ZRI4QQt7EkVagCrMiYQ7ce+FNr\nvUwpNVwp9TSA1noBEK+UOgB8CfwrY3saMBJYDOwCZmit99jhPhwmIjCC5YcKXhglN15ecLrSTP47\n9GFmzjRKHMfH260504T4hxB9JJp0ne7wtrXWvL3Str1z2Xl7eTO0xVD2jdxH++rt6TitI8PnDc91\noe31ietp8WULUtJTiH0m1m2SOQAfLx8GNx3MtNhpZofi9K6nXue15a/xftf3UUrxzzb/JDYplugj\nzlFESHiWzIWwhWVyKoySmgrLlkF3y+tuCSGE1SxZtmCH1rql1rqF1rqZ1vqDjO1faq2/ynLcSK11\n7YxjtmbZvkhrXU9rXUdr/b59bsNxwgPDiToUZbPCF9mdunKKTUc3MaxTT1atMhYlDw6Gr74Cd1rO\nq0qJKpQpUoa403EOb3vhgYXcSLtBn/p97N5WEd8ijAkZw96ReylVuBRNpjThteWvceHaBQBS0lJ4\nc8Wb9JnRh4ldJzKt9zRK+pW0e1yONqzFML7f9j0paSl5H+zBpmyeQpNKTW6uL1jYpzDjwsYxdtlY\nWc9POJzMn7NOTgndhg1GD2eVKubEJITwDG44mM++qpaoSsViFYlNirXL9efumUvPOj0p6lsUb294\n8UVYuRK+/hp69oTERLs0a4rMXjpH0lozbuU4Xu/0ukOLjJQtUpZJ3SYRMzyGo5eOUndyXSasnkDI\n1BA2HttIzPAY+jbo67B4HK1e+XrULlubBfsXmB2K07pw7QIT1kxgQsSE27YPajqIs8lnWXhgoUmR\nCU908aIx5LJzXiXQxE05rUUnyxUIIRxBErp8iAiMsMl6dDmZuXsmDze8vbplw4awdi2EhhpVsr7/\n3j1660IDQlmT4NjCKIv/XszlG5fp17CfQ9vNFFAqgGm9p7F00FJ2nNzBkGZDWDBwAVVKuP/Xt7Im\n3d39Z+1/uLfOvTSuePsEHG8vb94Nf5eXl71syhBl4ZmWLzeW0yla1OxIXEdOPXSS0AkhHEESunyI\nCIyw2Xp0WZ24fIItx7bQo/adkxZ8feG114w1gf77X+jTB5KSbB6CQzm6h86s3rmcNKnUhJ/7/cyI\n4BF2m8fnbPo37M+qw6tIuuziv7h2cPzScaZsnsK4sJyLhfSu15siPkWYsXOGgyMTnkrmz1mvRg04\nehRu3DDenzkDcXEQEmJuXEII9ycJXT50rtmZtQlruZF2w6bXnbtnLvfVvY8ivkVyPaZ5c9i40aii\n1awZzHTh5b0aVGjA2eSzDvuAvzx+OWeTz9K/YX+HtCduV8KvBH3r9+WHbT+YHYrTGbdyHMOaDyOg\nVECO+5VSvN/1fV5f8brN/+4IkZ3WMn8uP3x9jfXoDh823i9ZAp06gZ+fuXEJIdyfJHT5ULZIWeqW\nq8uGxA02vW5Owy1z4ucH774Lf/4Jb7wBjzxiLFrqaryUFx38Ozikly6zd+61Tq/h7eVt9/ZEzoa1\nMNakkwIft+w9vZc5e+bwcseX73pcWM0w6pStwzdbv3FQZMJT7d8PKSnQqJHZkbierMMuZbilEMJR\nJKHLJ1sPu0y6nERsUiz31Lb8r3/btsak9erVoWlT+PFH4yHsSkIDHLPA+MrDKzl++TiPNn7U7m2J\n3HXw74BGsy7R+gXX3dWry1/lxfYvUrZI2TyPfS/iPd5Z9Q5XblxxQGTCU0VGGmX2PWQ0uE1lFkbR\n2pgiIQmdEMIRJKHLp/DAcJsWRpmzew696vaisE9hq84rUsSYU/frrzB1qrE4+fjxcOKEzUKzqxD/\nEKIT7N9D9/bKt3mt42v4ePnYvS2RO6UUw5oPY2qMFEcBY/3B9YnrGd12tEXHt6zSkk41OvHxho/t\nHJnwZNKzlH+ZPXQ7d0LhwsZ7IYSwN0no8ik0IJStx7fa7JtyS4db5qZjR6Mq2aJFxtIG9evD4MGw\naZNNwrOb1lVbs+vULrv2OKw+vJrDFw4zsMlAu7UhLDe42WDm7JnD5RuXzQ7FVFprxi4dy7iwcXed\nN5vd+C7j+XDdh5y5esaO0YnslFI9lFJxSql9SqmXcthfWik1Vym1TSm1XinVMMu+UkqpWUqpPUqp\nXUqpthnbyyilFiul9iqlIpVSpRx5Tzm5fh1WrYKuXc2OxDXVqmUkdJlzEKWXUwjhCJLQ5VOxQsVo\nVbUVq4+sLvC1jl86zo4TO+heq3uBr9WkCXz5pTHko2lT6N8f2reHX365VXnLmRTxLUKzSs3YeHSj\n3dp4e9XbvBL6Cr7evnZrQ1iuSokqdAzoyOzds80OxVSLDizixJUTDGk+xKrz6pSrw0MNH2Ji9EQ7\nRWZfaxPWmh2C1ZRSXsBk4B6gETBAKVU/22GvADFa62bAEOCTLPs+BhZorRsAzYA9GdvHAku11vWA\n5cDdJ1I6wJo1xlI55cqZHYlrql3beP5KL6cQwpEkoSuA8Jq2GXY5Z48x3NLPx3alsMqWNRYl//tv\nGDsWvvnGGI759tvOt9yBPYddrk1Yy/4z+xnUbJBdri/yJ7M4iqdKS0/jpaUvMSFiQr6GAb/R+Q2+\njfmWxIuJdojOfs4mn2XAnAFmh5EfwcB+rfVhrXUKMAPone2YhhhJGVrrvUBNpVQFpVRJoKPWelrG\nvlSt9cWMc3oD0zN+ng70sfN95EkSkYIJCoL4eNiwAbp0MTsaIYSnkISuACKCbFMYZeaumTzcKP/D\nLe/G2xt694Zly4wJ2seOQYMGMGiQsfyBM7BnYZTxq8bzSsdXKORdyC7XF/lzX5372H9mP/vO7DM7\nFFP8vONnihcqTu962XMCy1QtUZWnWj7F2yvftnFk9qO1ZugfQ+nXoJ/ZoeRHNSAhy/vEjG1ZbQP6\nAiilgoEAoDoQCJxWSk1TSm1VSn2llMocY1tRa30CQGudBFS04z1YRBK6gilSBMqXh5YtoWRJs6MR\nQngKqRBRAMHVgtl/Zj9nk89aVKEuJ0cvHmXnyZ10C+pm4+ju1LgxfPEFTJhgFFB59FGoWBFGjTKG\nZhYyKefp4N+BIb8PIS09zaZLCmw8upFdJ3fx+yO/2+yawjZ8vX0Z1HQQ02KmMaHrBLPDcahrqdd4\nfcXr/PDgDwVaVP6lkJeoO7kuL7R/gXrl69kwQvuYvHEyRy8eZeZDM/kf/zM7HHt4H/hYKbUV2AHE\nAGmAL9ASGKG13qyU+ghjqOWbQPZfgFzX83jlFbvEfJvUVEhIgOBg+7flzmrXlqRYCOFYktAVQCHv\nQoQEhBB1KIq+Dfrm6xpz9szhgXoP2HS4ZV7KlIEXXoDnnoO//oJPPzWGZw4fbryqVHFYKABUKFaB\nysUrs/PkTppVbmaz67698m3Gho516L9bYbmhLYbS9fuujA8f71HVR6dsmkLTSk3pWKNjga5TpkgZ\nXmz/Iq+teI1Z/WfZKDr72Hp8K2+vept1T6xz1f8fj2L0uGWqnrHtJq31JWBY5nulVDxwECgGJGit\nN2fsmg1kFlVJUkpV0lqfUEpVBk7mFsCGDW/d/DkwMIygoLB83srdffst+HjO/452MX481K1rdhRC\nCFcRFRVFVFRUga6hnGWBX6WUdpZYrPHB2g+IPxfPZ/d9lq/zQ6eG8krHV7i3zr02jsw6u3fD5Mkw\nYwb07AmjRxvr3DnKE388QauqrfhXm3/Z5Hpbjm2h94ze/D36b1f9AOkR2n/bnlc7vkqvur3MDsUh\nLly7QJ1P67B8yHIaV2xc4OtdTblKnU/r8Psjv9OmWhsbRGh7l65fotVXrRgXNo4BTYz5c0optNYu\nU/9PKeUN7AUigOPARmCA1npPlmNKAVe11ilKqaeAEK314xn7VgJPaa33KaXeBIpqrV9SSk0Ezmqt\nJ2ZUziyjtR6bQ/su+XwUQghhvfw8I2UOXQEVZIHxxIuJ7Dm9h65B5teHbtgQPv8cDh6E1q1h4EBj\n2M2PPxplrO0tJMC2hVHGrxrPSyEvSTLn5J5o8YRHrUk3KXoSver2skkyB1DUtyhvdHqDV5Y7YDxe\nPmit+edf/6RTjU43kzlXpLVOA0YCi4FdwAyt9R6l1HCl1NMZhzUAdiql9mBUw3w2yyVGAz8ppWIx\nqly+l7F9ItBNKZWZLL5v/7sRQgjhbqSHroDSdToV/lOB7c9sp1rJ7HPk7+6j9R+x/cR2pvZ2vg+0\naWmwcCF88gls335rOGbVqvZpb9+ZfXT7oRuHnztc4GvFJsVy70/38vfov61a30s43sXrFwn4XwD7\nRu2jYjHT60HY1bFLx2gypQmxw2PxL+Vvs+umpKXQ8POGTLlvilN8OZTVd7HfMSl6Epuf3kxR36I3\nt7taD53ZXPX5KIQQwnrSQ2cCL+VFl5pd8rV8gT2rWxaUtzf06mVUxlyxAk6fhkaNjJ67devA1p8t\n6pStQ3JKMgkXEvI+OA/jV41nTIcxksy5gJJ+JelTvw8/bv/R7FDsblzUOJ5o8YRNkzkwCsy80+Ud\nXl72Ms70oT/udBxjloxhZv+ZtyVzQgghhLAtSehsID/DLo9cOMK+M/uICIywU1S206ABfPaZsbZO\ncLCx5EHTpsY8u1mzbLOunVLKJsMud5zYwdqEtQxvPbzgQQmHyFyTzpmSEVuLOx3H3Li5jA29Y3qU\nTfRv1J+09DTm7plrl+tbKzklmUdmP8K74e/abHipEEIIIXImCZ0NhAcaC4xb84F09u7Z9KnfB19v\nXztGZlulSxuVMfftg6++gurV4fvvjfl3derAsGEwbRocOJC/HrwQ/xCijxQsoXtn9Tu80P4F6RFw\nIR0DOnIj7QYbjzrJwoh28OryV3mx/Yv5Xt4kL17KiwkRE3h1+aukpqfapQ1rvLD4BeqXr89TLZ8y\nOxQhhBDC7UlCZwN1y9UlXadz4OwBi89x5uGWefHygvbt4d//hnnzjOGYc+caxVQWL4awMKhWDR5+\n2FgSITbWmJOXl9CAUNYk5H+B8d2ndhN1KIpnWj+T72sIx1NKMaz5MLctjrI+cT0bEjcwuu1ou7bT\nvVZ3qpSowvTY6XZtJy9zds9h0YFFfNXrqwKtsyeEEEIIy0hRFBsZ8vsQ2ldvb1Eycej8Idp83YZj\nzx9zqR46S2kNhw7B6tW3XklJ0KEDdOxovNq0Ab9sBShvpN2g7MSyHH/hOCX8Sljd7sA5A2laqand\nhrUJ+zl68ShNpjQh8flEt+pd1VoTNj2MwU0H80TLJ+ze3vrE9fSf1Z99I/eZMof00PlDBH8dzPyB\n8wmulvvq1FIUxTqu/nwUQghhOSmKYqLwmuEWF0aZvXs2D9Z/0C2TOQClIDAQBg+Gr7+GuDhjmOaT\nT8LJk8awzXLloFMnePVVWLQILl40FmpvWaUl6xPXW91m3Ok4lh5cyog2I+xwR8LeqpWsRnv/9sze\nPdvsUGxq4YGFnLpyiiHNhzikvXbV29G6ams+3/S5Q9rLKiUthQFzBvDvkH/fNZkTQgghhG1JQmcj\nEUERrDi0gnSdnuexM3fNpH/D/g6IynlUrAh9+8L//gebN8Px4/Daa8bwzfffN5ZDaNkSkveG8lXk\nGk6csO76761+j2fbPpuvnj3hHNxtTbq09DTGLh3LhIgJ+Hj5OKzdd8PfZWL0RC5cu+CwNgFeX/E6\nZQqX4fn2zzu0XSGEEMLTSUJnI9VLVqdskbJsP7H9rsfFn4sn/nw8XQK7OCgy51SiBHTvDuPHQ1QU\nnDkDkydD45IhrDgQTf36ULcuPPEETJ0KW7ZAcnLO19p/Zj8LDyxkZPBIh96DsK1edXux+9Ruq+ai\nOrOfdvxECb8SPFDvAYe227BCQ3rV7cUHaz9wWJuRByL5cfuPTO8zHS8ljxUhhBDCkeTJa0OWDLuc\nvXs2fev3deg39q7Az8+YY/fh/3XgRoWNnDiVyuzZRq/dsmVGBc2yZY0kr29feOMNmDkTdu+Gd1a9\nx6jgUZQqXMrs2xAFUMi7EP9o+g++i/3O7FAK7FrqNV5f8ToTu040pTDIW2Fv8fnmzzlx2cqu7nw4\nfuk4j//xOD88+AMVilWwe3tCCCGEuJ0URbGh2btnMy12Gn8N/CvXY9p83Yb3I94nIsj5158zS+PP\nGzO9z3RaVW112/aUFNi/H3buvPWKOXSQQ92CabTsAM3qlaZxY26+atQwhnQK17Hz5E56/NiDw88d\nxtvL2+xw8u3DdR+y8vBK/nj0D9NieD7yeVLSUvj03k/t1kZaehr3/HgPIf4hjOsyzuLzpCiKddzh\n+SiEEMIy+XlGSkJnQ2euniHokyBOjzmdY8GTg+cO0u6bdhx74Zj00N3FM/OfoUH5Bjzb7tk8j33q\nz6coX7gK/cu/zY4dtyd7588ba+Q1acJtiV6lSkbhFuGc2n7Tlrc6v0XPOj3NDiVfzl87T91P67Ji\nyAoaVWxkWhynrpyiwWcN2PjURoLKBNmljXdXvUvk35EsH7Lcqr9pktBZxx2ej0IIISyTn2ekZBU2\nVK5oOYLKBLHp2CY6+He4Y/+sXbPo16CfJHN5CPEPYd6+eXkmdIfOH2Ju3Fz2j9pP2SLG8Myszp+H\nXbtuJXi//w47dhjJXGZyl5nsNWpkLJwuzDes+TCmxk512YRuUvQk7q97v6nJHECFYhUY3XY0b6x4\ngx/7/mjz6685soZPN37K5qc3y980IYQQwkTSQ2djLy5+kVJ+pXi98+t37Gv1VSs+6PaBxxdEycvB\ncwfpOK0jif+XeNf5R8/Mf4ZyRcrxbsS7Fl9bazhx4vaevJ07jcSvdOlbyV2jRkbvXsOGRgEX4TgX\nrl2gxkc1ODD6AOWLljc7HKscvXiUpl80JXZ4LP6l/M0Oh0vXL1Hn0zpE/iOSZpWb2ey6Z5PP0uLL\nFnx272f0qtvL6vOlh8467vJ8FEIIkTcZcukEFu5fyMToiUQ9HnXb9gNnDxA6NZSjzx916blBjqC1\nptqH1YgeFk1gmcAcj0m4kEDzL5uzd+Rem3zoT0+Hw4dvJXe7dhkFV+LioHx5I7HLTPQaNYIGDSTR\ns6dBvw2iVZVWPNfuObNDscrT856mdOHSTOo2yexQbvpkwycs/nsx8wfOt8n1tNb0+bUPQaWD+F+P\n/+XrGpLQWcddno9CCCHyJgmdE7h84zKVP6jMyTEnKepb9Ob2CasnkHgxkc/u+8zE6FxH/1n96V2v\nN/9o+o8c94/4awTFCxVnYreJdo0jLQ0OHTKSu5wSvcyePEn0bCvqUBSjF45m2zPbTKkSmR9xp+Po\nOK0je0fupWyRsmaHc9P11OvUm1yPHx78gY41Ohb4ep9u+JTp26YTPSwaPx+/fF1DEjrruMvzUQgh\nRN5kDp0TKF6oOM0rNyf6SDTdanW7uX3m7pl8dM9HJkbmWkL9Q1lzZE2OCd3Ri0f5ZecvxI2Ms3sc\n3t5Qq5bxuv/+W9szE73MBG/5cvj0UyPRq1Dh9mGbmf8sXtzu4bqNTjU6cSXlCluOb6F11dZmh2OR\nV5e/ypgOY5wqmQPw8/Hj7S5v8/Kyl1k9dHWBEuStx7fy9qq3WffEunwnc0IIIYSwLSnqbgcRgREs\ni1928/2+M/tIupxEaECoiVG5lpCAEKITonPcNyl6EsNaDKNisYoOjuqWzETvgQdg7Fj44QfYuhUu\nXTKSu2eeMdbNy/y5YkVjGYV774UxY2DaNNi4Ea5dM+0WnJqX8mJo86FMjZlqdigWWZ+4no1HNzIq\neJTZoeTosSaPcf7aef7an/uSKnm5dP0Sj85+lE96fELtsrVtGJ0QQgghCkKGXNrBykMrGbNkDBuf\n2ggYpb2TLifZdT0od5OankrZiWU5/NxhyhQpc3P78UvHafR5I3aP2E3l4pVNjNA6WXv0Mnv1du6E\nffugeXMICYHQUGNx9fKuVQfEbjLnSSb+XyJFfIuYHU6utNZ0/q4zjzd/nGEthpkdTq7+3Psnry5/\nldjhsVbP49VaM/j3wfh5+/HNA98UOBYZcmkdd3o+CiGEuLv8PCOlh84O2lVvR9zpOM5fOw8Ywy0f\nbvSwyVG5Fh8vH9pUa8O6xHW3bf/P2v8wuNlgl0rm4PYevZdfNnr0YmLg5EkYP96Yd/fZZ8YxDRrA\nk0/Cd98ZC6m70uc4reHoUThypODX8i/lT3C1YObumVvwi9nRgv0LOJN8hsHNBpsdyl3dX/d+SvqV\n5Jedv1h97vRt09lybAsf9/jYDpEJIYQQoiBkDp0d+Pn40d6/PVGHoqhfvj6nrpwiJCDE7LBcToh/\nCNFHorm3zr0AnLh8gu9iv2Pnv3aaHJntFCsG4eHGC4yevB07IDoaIiPh9dfhxg2j9y6zF69FC/C9\nc916h0pLg/h42LPH6G3cs8d4xcVBkSJG1dBy5aBnT+PVqRP45WPK1bDmw/hiyxc81vQx29+EDaSl\npzF22VgmRExw+rXYlFK8H/E+g38fzMONHqaQdyGLzos7HceYJWNYMWQFxQoVs3OUQgghhLCWDLm0\nk4lrJnL00lEqFK3Aqaun+KTnJ2aH5HIW/72Yd1e/y8rHVwLw7yX/Jjkl2eOGrh45AmvWGEnemjVw\n8CC0bn0rwWvfHkqVsk/b168bw0KzJ24HDhjzAhs2NHoUs77KljUSuq1bYeFC47VzJ3TufCvBC8x5\nNYo720+9TvX/VWfjkxtzXcLCTNNjp/P11q8LXGzEke77+T561u7JyOCReR6bnJJMu2/bMaLNCJ5u\n9bTNYpAhl9Zxt+ejEEKI3MmyBU5k87HNDPl9CF7Kiyn3TZGCKPlw8fpFqv63KmdfOsuFaxeoN7ke\n2/+5neolq5sdmqkuXIB1624leJs3Q1DQrQQvJAQCAsCa/OLiRaN3LTNhy0zeEhKM5KtBg9uTt3r1\njN5FS505A0uWGMndokVQpsztvXeFC+d+7rMLn6V04dKM6zLO8gYd4FrqNep+Wpdf+v3iUj3w25K2\n0eOnHuwftZ/ihe5eevVff/2L01dP8+tDv9o0YZWEzjru9nwUQgiRO0nonEhaehoV/lOBIr5FSPi/\nBLyUTFfMjxZftmDKfVP4I+4Pzl87z5ReU8wOyemkpBjz8TITvOhoY0hm1gSvaVPw8oJTp24lbVkT\nt3PnjCQtM2HLTN5q1YJClo3Ms1h6uhFvZu/djh1GUpeZ4AUF3X78tqRt3P/L/cQ/G291MQ97+u/a\n/7L6yGp+f/R3s0Ox2mNzH6NB+Qa81um1XI+Zs3sOY5aMIWZ4DKUK27YLWBI667jb81EIIUTuJKFz\nMn1/7Yt/SX8+7imFBPJr5IKRlPQryZdbviRmeAwBpQLMDsnpaQ1//337MM1jx8DHx9iXfYhkw4ZG\nj56XSd85nD0Lixffvfeu9VeteS/iPbrX6m5OkNmcv3aeup/WJerxKBpWaGh2OFb7++yRN8HaAAAM\nl0lEQVTftP2mLXEj4yhf9M6yqofOHyL462DmD5xPcLVgm7cvCZ113PH5KIQQImeS0DmZY5eOUbxQ\ncUr6lTQ7FJc1Y+cMBv02iKHNh/LV/V+ZHY7LOnPG6MmrVMm6oZiOllvvXbGwz7lSfhXzH59hdogA\nvLz0ZU5eOcm3vb81O5R8G/HXCAr7FOa/9/z3tu0paSl0+q4T/Rr048UOL9qlbUnorOOOz0chhBA5\nk4ROuJ3Ei4nU/qQ2e0bsccqiGMK+zp415t79HnmOXysHEjTvIL0iytKzp1Fk5W5z7+zl6MWjNJnS\nhG3PbMO/lL9NrpmWZiTcWV9pacaahLYe8prp+KXjNJ7S+I6e77FLx7L9xHbmD5xvt6HiktBZR56P\nQgjhOSShE27pXPK52xYXF55p4JzHqE47SsWNYuFC2L4dOnY05gn6+hq9e1rfemV9b+k+S45bXuwp\nfFPL0vTkxDuSMEteqal3bgPjHrK+vL2NhLZSJaMwTU6vqlULNlT21WWvknQ56WZPY+SBSJ748wli\nhsdQoVgFG/xXy5kkdNaR56MQQngOSeiEEG5r2cFlvLjkRWKGxwBGIZclS2DjRmO/UsbLy+vWz9nf\nF3TfibQ4PrrYkQlV91Har8wdSZg1Lx+f25O3nKSmGpVG4+Nzfp07Z8x/zC3hK1fu7kNsM+cCrnx8\nJaULl6blVy35ue/PdAnsYuP/ereThM468nwUQgjPIQmdEMJtpet0gj4O4rdHfqNFlRYObz8lLYV+\nM/vRMaAjY0LGOLz9nCQnw6FDuSd8aWlQs2buCV/x4vCf6P+wNnEtl65fooN/B97u8rbd45aEzjry\nfBRCCM8hCZ0Qwq2NixrH6aunHbq4fNLlJL7e8jVfbPmC+uXrM3/AfIr4FnFY+wVx/nzuyd6hQ8Za\ngjVqJbO7a21KptWi+/Hl6DQf0tONZDAtjZs/W7rNknOOHJGEzhryfBRCCM8hCZ0Qwq0dPn+YVl+1\nIvH5RAr72K8iitaa9YnrmbxpMgv2L6B/w/6MaDOCZpWb2a1NR9MaTpwwkruVcTsonFKZUr4V8PY2\nhoB6eXHHz5Zuy2t/zZqS0FlDno9CCOE5JKETQri97j90Z1iLYTza+FGbXzs5JZkZO2cwedNkLly7\nwIg2I3i8+eNSlMfGZMildeT5KIQQnkMSOiGE25uxcwZTY6ayeNBim13z0PlDTNk0hamxUwmuFszI\nNiO5p/Y9divb7+kkobOOPB+FEMJz5OcZKZ9WhBAupU/9Pmw9vpXD5w8X6DrpOp3Ffy/mgV8eoNVX\nrUhNT2XdE+v4a+Bf9KzTU5I5IYQQQrgE6aETQricUQtGUb5oed4Me9Pqcy9cu8D0bdP5bNNnFPYp\nzMg2IxnYZCDFChWzQ6QiJ9JDZx15PgohhOeQIZdCCI8QczyGB399kIPPHrS4J23XyV18tukzftn5\nC91rdWdkm5GEBoSi7rZQm7ALSeisI89HIYTwHPl5RvrYKxghhLCXFlVaUKZIGVbEryAiKCLX41LT\nU/lz759M3jiZPaf38HTLp9n5z51UK1nNgdEKIYQQQtiPJHRCCJc0rPkwpsZOzTGhO3nlJN9s/YYp\nm6dQo1QNRrQZQb+G/SjkXciESIUQQggh7EeGXAohXNLZ5LMEfRxE/LPxN5cV2Hh0I5M3Tmbevnn0\na9CPEW1G0KJKC5MjFdnJkEvryPNRCCE8h8yhE0J4lEdnP0pwtWDKFSnH5E2TOXP1DP9q8y+GNh9K\nuaLlzA5P5EISOuvI81EIITyHXRI6pVR14HugEpAOfK21/iTbMaWBqUAtIBkYprXenbHvEHAh49wU\nrXVwLu3IA0sIYZWlB5fS7Ydu3FPrHkYGj6Rn7Z54e3mbHZbIgysmdEqpHsBHGMv9fKu1nphtv9XP\nQaXUm8BTwMmMy7yitV6UQ9vyfBRCCA9hr3XoUoHntdaNgPbACKVU/WzHvALEaK2bAUOArAlfOhCm\ntW6RWzLnzqKioswOwS7c9b7Afe/NHe+ra1BXfgv+jUX/WESvur3cLplzx/9mrkgp5QVMBu4BGgED\nbPgc/FBr3TLjdUcy587c+ffbXe/NXe8L3Pfe5L48Q54JndY6SWsdm/HzZWAPkL1EXENgecYxe4Ga\nSqkKGfuUJe24K3f9hXPX+wL3vTd3va/YDbFmh2A37vrfzAUFA/u11oe11inADKB3tmPy+xx0qZ5K\nW3Ln3293vTd3vS9w33uT+/IMViVaSqmaQHNgQ7Zd24C+GccEAwFA9Yx9GliilNqklHqqIMEKIYQQ\nJqgGJGR5n8idX2zm9zk4UikVq5T6RilVyvahCyGEcHcWJ3RKqeLAbODZjJ66rN4HyiiltgIjgBgg\nLWNfiNa6JXAvxnDN0IKHLYQQQjiV/DwHPweCtNbNgSTgQwfHLIQQwg1YVOVSKeUDzAcWaq0/tuD4\neKBJ9sQvYwL4Ja31HQ8tpZTM+BZCCA/hSkVRlFLtgLe01j0y3o8FdPbCKNnOseo5qJSqAczTWjfN\n4VryfBRCCA9i7TPS0oXFpwK7c0vmMoaJXNVap2QMJ1mptb6slCoKeGX8XAzoDoyzReBCCCGEg2wC\namckXceBR4EBWQ/Iz3NQKVVZa52UcYm+wM6cGpfnoxBCiLvJM6FTSoUAjwE7lFIxGHMBXgFqYHxD\n+RXQAJiulEoHdgFPZJxeCfgt49tFH+AnrfVi29+GEEIIYR9a6zSl1EhgMbeWLdijlBpOwZ6Dk5RS\nzTGqYB4ChjvspoQQQrgNp1lYXAghhBBCCCGEdUxfTkAp1UMpFaeU2qeUesnseGxBKVVdKbVcKbVL\nKbVDKTXa7JhsTSnlpZTaqpT60+xYbEUpVUopNUsptSfjv11bs2OyBaXUyxn3s10p9ZNSqpDZMeWX\nUupbpdQJpdT2LNvKKKUWK6X2KqUiXbFSYC73NSnjdzFWKTVHKVXSzBjzK6d7y7LvBaVUulKqrBmx\nuQJ5Rroed3w+gjwjXYE8I12LLZ+PpiZ0yrLFWl2RJYuxu7pngd1mB2FjHwMLtNYNgGYYay66tIw5\nP08BLTKKLfhgzP9xVdMw/l5kNRZYqrWuh7EO2MsOj6rgcrqvxUCjjAqI+3HN+4Kc7w2lVHWgG3DY\n4RG5CHlGuix3fD6CPCNdgTwjXYvNno9m99BZsliry7FwMXaXlfGLdi/wjdmx2ErGNzsdtdbT/r+9\n+3exq4qiOP5dYOGP1DFICGaE9IpF0EpFCAijpVho9A8QLGxikU7SiIVokUYSMVqEQNJYGKxULAw6\nmBRpLBIjjEgiKQR/Lot7EsfgRHnvJmf2YX1g4L1b7cubeevsO+feDWD7d9tXO5c1h6vAr8A9mp5W\nezfwfd+SFmf7U+DKDYefBo6010eAZ25rUTP4t/Oyfdr2n+3tF/w906yUTT4zgDeBV29zOdUkI4sZ\nMR8hGVlFMrKWOfOxd0P3f4a1lqbNh7FXdu0XbaQbMHcDP0p6t22VOSzprt5FLcv2FeAN4AJwCfjJ\n9um+Vc1uu+11mBaKwPbO9dwKLwEf9S5iLpJWgYu2v+ldyxaXjKxnxHyEZGRlychCFs3H3g3d0HTz\nYewlSXoKWG9XV9V+RnAH8BDwdhsA/DPTNoXSJK0ArzA9lfY+YJuk5/pWdcsNtZCS9Brwm+1jvWuZ\nQ1sEHgAObjzcqZzoaLSMHDgfIRk5kmTkFrVMPvZu6C4Buza839mOldf+dX8ceM/2yd71zOhRYFXS\nt8AHwGOSjnauaQ7fMV0R+bK9P84UXtU9DHxm+7LtP4ATwCOda5rbuqR7YZrrBfzQuZ7ZSNrPtH1r\npAXGA8D9wJqm4ds7gTOSRrxqvKxkZC2j5iMkIytLRtaxcD72buiuD2ttTxV6FhjlqVA3HcZele0D\ntnfZXmH6vD6x/XzvupbVtiNclLSnHXqCMW5qPw/slXSnJDGdV/Ub2W+88n0K2N9evwBUXRz+47wk\n7WPaurVq+5duVc3j+rnZPmt7h+0V27uZFooP2h5mkTGjZGQho+YjJCOLSUbWMks+dm3o2tWQa8Na\nzwEf2q7+h7RxGPvjkr5q+8339a4r/tPLwPuSvmZ6gtfrnetZmu014ChwBlhj+tI43LWoJUg6BnwO\n7JF0QdKLwCHgSUnnmcL4UM8aF7HJeb0FbAM+bt8h73QtckGbnNtGZqytabNJRsYWk4zc4pKRtcyZ\njxksHhERERERUVTvLZcRERERERGxoDR0ERERERERRaWhi4iIiIiIKCoNXURERERERFFp6CIiIiIi\nIopKQxcREREREVFUGrqIiIiIiIii0tBFREREREQU9Rev+TCFPnw4HwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=[15, 7])\n", + "ax1.plot(zip(train_costs, valid_costs))\n", + "ax1.legend(['training cost', 'validation cost'])\n", + "ax2.plot(valid_accs)\n", + "ax2.legend(['validation accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Author: \n", + "Adrian Lisko - https://github.com/AdrianLsk" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/ladder_nets/ladder_net_layers.py b/examples/ladder_nets/ladder_net_layers.py new file mode 100644 index 0000000..5101406 --- /dev/null +++ b/examples/ladder_nets/ladder_net_layers.py @@ -0,0 +1,186 @@ +from lasagne.layers import MergeLayer + +import theano +import theano.tensor as T + +import numpy as np + +def _create_milaUDEM_params(shape, name): + values = np.zeros((6,) + shape, dtype=theano.config.floatX) + + b_lin = theano.shared(values[0], name='bias_lin_{}'.format(name)) + b_sigm = theano.shared(values[1], name='bias_sigm_{}'.format(name)) + + w_u_lin = theano.shared(values[2], name='weight_u_lin_{}'.format(name)) + w_u_sigm = theano.shared(values[3], name='weight_u_sigm_{}'.format(name)) + w_zu_lin = theano.shared(values[4], name='weight_zu_lin_{}'.format(name)) + w_zu_sigm = theano.shared(values[5], name='weight_zu_sigm_{}'.format(name)) + + values = np.ones((3,) + shape, dtype=theano.config.floatX) + w_z_lin = theano.shared(values[0], name='weight_z_lin_{}'.format(name)) + w_z_sigm = theano.shared(values[1], name='weight_z_sigm_{}'.format(name)) + w_sigm = theano.shared(values[2], name='weight_sigm_{}'.format(name)) + + # combinator params used in combinator calculations + return [w_u_lin, w_z_lin, w_zu_lin, w_u_sigm, w_z_sigm, + w_zu_sigm, w_sigm, b_lin, b_sigm] + + +def _create_curiousAI_params(shape, name): + values = np.zeros((8,) + shape, dtype=theano.config.floatX) + + b_mu_sig = theano.shared(values[0], name='b_mu_sig_{}'.format(name)) + b_mu_lin = theano.shared(values[1], name='b_mu_lin_{}'.format(name)) + b_v_sig = theano.shared(values[2], name='b_v_sig_{}'.format(name)) + b_v_lin = theano.shared(values[3], name='b_v_lin_{}'.format(name)) + + w_mu_lin = theano.shared(values[4], name='w_mu_lin_{}'.format(name)) + w_v_lin = theano.shared(values[5], name='w_v_lin_{}'.format(name)) + w_mu = theano.shared(values[6], name='w_mu_{}'.format(name)) + w_v = theano.shared(values[7], name='w_v_{}'.format(name)) + + values = np.ones((2,) + shape, dtype=theano.config.floatX) + w_mu_sig = theano.shared(values[0], name='w_mu_sig_{}'.format(name)) + w_v_sig = theano.shared(values[1], name='w_v_sig_{}'.format(name)) + + # combinator params used in combinator calculations + return [w_mu_lin, w_v_lin, w_mu_sig, w_v_sig, w_mu, w_v, + b_mu_lin, b_v_lin, b_mu_sig, b_v_sig] + + +def _create_combinator_params(combinator_type, shape, name): + if combinator_type == 'milaUDEM': + return _create_milaUDEM_params(shape, name) + elif combinator_type == 'curiousAI': + return _create_curiousAI_params(shape, name) + + +def _combinator_milaUDEM(z, u, combinator_params, bc_pttrn): + w_u_lin, w_z_lin, w_zu_lin, w_u_sigm, w_z_sigm, w_zu_sigm, w_sigm, \ + b_lin, b_sigm = combinator_params + + lin_out = w_z_lin.dimshuffle(*bc_pttrn) * z + \ + w_u_lin.dimshuffle(*bc_pttrn) * u + \ + w_zu_lin.dimshuffle(*bc_pttrn) * z * u + \ + b_lin.dimshuffle(*bc_pttrn) + + sigm_pre = w_z_sigm.dimshuffle(*bc_pttrn) * z + \ + w_u_sigm.dimshuffle(*bc_pttrn) * u + \ + w_zu_sigm.dimshuffle(*bc_pttrn) * z * u + \ + b_sigm.dimshuffle(*bc_pttrn) + + sigm_out = T.nnet.sigmoid(sigm_pre) + + output = w_sigm.dimshuffle(*bc_pttrn) * sigm_out + lin_out + + return output + + +def _combinator_curiousAI(z, u, combinator_params, bc_pttrn): + w_mu_lin, w_v_lin, w_mu_sig, w_v_sig, w_mu, w_v, \ + b_mu_lin, b_v_lin, b_mu_sig, b_v_sig = combinator_params + + mu_sig_pre = w_mu_sig.dimshuffle(*bc_pttrn) * u + \ + b_mu_sig.dimshuffle(*bc_pttrn) + + mu_lin_out = w_mu_lin.dimshuffle(*bc_pttrn) * u + \ + b_mu_lin.dimshuffle(*bc_pttrn) + + mu_u = w_mu.dimshuffle(*bc_pttrn) * T.nnet.sigmoid(mu_sig_pre) + \ + mu_lin_out + + v_sig_pre = w_v_sig.dimshuffle(*bc_pttrn) * u + \ + b_v_sig.dimshuffle(*bc_pttrn) + + v_lin_out = w_v_lin.dimshuffle(*bc_pttrn) * u + \ + b_v_lin.dimshuffle(*bc_pttrn) + + v_u = w_v * T.nnet.sigmoid(v_sig_pre) + v_lin_out + + output = (z - mu_u) * v_u + mu_u + + return output + + +def _combinator(z, u, combinator_type, combinator_params): + if u.ndim == 2: + bc_pttrn = ('x', 0) + elif u.ndim == 4: + bc_pttrn = ('x', 0, 1, 2) + + if combinator_type == 'milaUDEM': + return _combinator_milaUDEM(z, u, combinator_params, bc_pttrn) + elif combinator_type == 'curiousAI': + return _combinator_curiousAI(z, u, combinator_params, bc_pttrn) + + +class CombinatorLayer(MergeLayer): + """ + A layer that combines the terms from dirty and clean encoders, + and outputs denoised variable: + $$ \hat{z} = g(\tilde{z}, u)$$ + """ + def __init__(self, incoming_z, incoming_u, combinator_type, **kwargs): + super(CombinatorLayer, self).__init__( + [incoming_z, incoming_u], **kwargs) + self.combinator_type = combinator_type + z_shp, u_shp = self.input_shapes + + if z_shp != u_shp: + raise ValueError("Mismatch: input shapes must be the same. " + "Got dirty z ({0}) of shape {1} and clean u ({" + "2}) of shape {3}".format(incoming_z.name, z_shp, + incoming_u.name, u_shp)) + + self.combinator_params = _create_combinator_params(combinator_type, + u_shp[1:], + self.name) + + def get_output_shape_for(self, input_shapes): + return input_shapes[0] + + def get_output_for(self, inputs, **kwargs): + z, u = inputs + assert z.ndim == u.ndim + return _combinator(z, u, self.combinator_type, self.combinator_params) + + +class SharedNormLayer(MergeLayer): + """ + A layer that combines the terms from dirty and clean encoders, + and outputs denoised variable: + $$ \hat{z} = g(\tilde{z}, u)$$ + """ + def __init__(self, incoming2stats, incoming2norm, axes='auto', epsilon=1e-4, + **kwargs): + super(SharedNormLayer, self).__init__( + [incoming2stats, incoming2norm], **kwargs) + stats_shp, norm_shp = self.input_shapes + + if stats_shp != norm_shp: + raise ValueError("Mismatch: input shapes must be the same. " + "Got dirty z ({0}) of shape {1} and clean u ({" + "2}) of shape {3}" + .format(incoming2stats.name, stats_shp, + incoming2norm.name, norm_shp)) + + if axes == 'auto': + # default: normalize over all but the second axis + axes = (0,) + tuple(range(2, len(stats_shp))) + elif isinstance(axes, int): + axes = (axes,) + self.axes = axes + self.epsilon = epsilon + + def get_output_shape_for(self, input_shapes): + return input_shapes[0] + + def get_output_for(self, inputs, **kwargs): + to_stats, to_norm = inputs + assert to_stats.ndim == to_norm.ndim + + mean = to_stats.mean(self.axes, keepdims=True) + inv_std = T.inv(T.sqrt(to_stats.var(self.axes, + keepdims=True) + self.epsilon)) + + return (to_norm - mean) * inv_std \ No newline at end of file diff --git a/examples/ladder_nets/ladder_nets.py b/examples/ladder_nets/ladder_nets.py new file mode 100644 index 0000000..18985b1 --- /dev/null +++ b/examples/ladder_nets/ladder_nets.py @@ -0,0 +1,347 @@ +from lasagne.layers import InputLayer, DenseLayer +from lasagne.layers import Conv2DLayer as conv +from lasagne.layers import Deconv2DLayer as deconv +from lasagne.layers import MaxPool2DLayer as pool +from lasagne.layers.special import InverseLayer as unpool +from lasagne.layers.special import BiasLayer, ScaleLayer, NonlinearityLayer +from lasagne.layers.noise import GaussianNoiseLayer +from lasagne.layers.normalization import BatchNormLayer, batch_norm +from lasagne.nonlinearities import rectify, linear +import lasagne + +from ladder_net_layers import CombinatorLayer, SharedNormLayer +from utils import softmax, unzip + +import theano.tensor as T +from collections import OrderedDict + +get_items = lambda zipped: unzip(zipped) +xe = T.nnet.categorical_crossentropy + +def build_encoder(net, encoder_specs, activation, name, p_drop_hidden, + shared_net): + # encoder specs is a tuple of string and tuple of integers + for i, (transform, specs) in enumerate(encoder_specs): + if transform == 'unpool': + specs = net.get(specs) + # if specs have already the name of the corresponding pool layer + update = build_enc_layer( + net.values()[-1], name, transform, specs, activation, i, + p_drop_hidden, shared_net + ) + net.update(update) + # apply activation + if i < len(encoder_specs) - 1: + act_name = 'enc_activation_{}'.format(i) + net[act_name] = NonlinearityLayer( + net.values()[-1], nonlinearity=activation, + name='{}_{}'.format(name, act_name) + ) + + # classfication layer activation -> softmax + net['enc_softmax'] = NonlinearityLayer( + net.values()[-1], nonlinearity=softmax, name=name+'_enc_softmax' + ) + + return net['enc_softmax'], net + + +def build_enc_layer(incoming, name, transform, specs, activation, i, + p_drop_hidden, shared_net): + net = OrderedDict() + lname = 'enc_{}_{}'.format(i, transform if 'pool' in transform else 'affine') + nbatchn_lname = 'enc_batchn_{}_norm'.format(i) + noise_lname = 'enc_noise_{}'.format(i) + lbatchn_lname = 'enc_batchn_{}_learn'.format(i) + + if shared_net is None: + # affine pars + W = lasagne.init.GlorotUniform() + # batchnorm pars + beta = lasagne.init.Constant(0) + gamma = None if activation == rectify else lasagne.init.Constant(1) + else: + # batchnorm pars + beta = shared_net[lbatchn_lname + '_beta'].get_params()[0] + gamma = None if activation == rectify else \ + shared_net[lbatchn_lname + '_gamma'].get_params()[0] + if not isinstance(shared_net[lname], (pool, unpool)): + # affine weights + W = shared_net[lname].get_params()[0] + else: + W = None + + # affine (conv/dense/deconv) or (un)pooling transformation: $W \hat{h}$ + net[lname] = get_transform_layer( + incoming, name+'_'+lname, transform, specs, W + ) + + # 1. batchnormalize without learning -> goes to combinator layer + layer2bn = net.values()[-1] + l_name = '{}_{}'.format(name, nbatchn_lname) + bn_broadcast_cond = layer2bn.output_shape[1] == 1 + if len(layer2bn.output_shape) == 4 and bn_broadcast_cond: + ax = (0, 1, 2, 3) + elif len(layer2bn.output_shape) == 2 and bn_broadcast_cond: + ax = (0, 1) + else: + ax = 'auto' + net[nbatchn_lname] = BatchNormLayer( + layer2bn, axes=ax, alpha=0.1, beta=None, gamma=None, name=l_name + ) + if shared_net is None: + # for dirty encoder -> add noise + net[noise_lname] = GaussianNoiseLayer( + net.values()[-1], sigma=p_drop_hidden, + name='{}_{}'.format(name, noise_lname) + ) + + # 2. scaling & offsetting batchnormalization + noise + l_name = '{}_{}'.format(name, lbatchn_lname) + # offset by beta + net[lbatchn_lname + '_beta'] = BiasLayer( + net.values()[-1], b=beta, name=l_name+'_beta' + ) + if gamma is not None: + # if not rectify, scale by gamma + net[lbatchn_lname + '_gamma'] = ScaleLayer( + net.values()[-1], scales=gamma, name=l_name+'_gamma' + ) + + return net + + +def get_transform_layer(incoming, name, transform, specs, W): + if transform == 'conv': + layer = conv( + incoming, num_filters=specs[0], filter_size=specs[1], + stride=specs[2], pad=specs[3], nonlinearity=linear, W=W, b=None, + name=name+'_conv' + ) + elif transform == 'dense': + layer = DenseLayer( + incoming, num_units=specs, nonlinearity=linear, W=W, b=None, + name=name+'_dense' + ) + elif transform == 'pool': + if len(specs) == 4: + psize, pstride = specs[1:3] + else: + psize, pstride = specs + layer = pool( + incoming, pool_size=psize, stride=pstride, name=name + ) + elif transform == 'deconv': + layer = deconv( + incoming, num_filters=specs[0], filter_size=specs[1], + stride=specs[2], crop=specs[3], nonlinearity=linear, W=W, b=None, + name=name+'_deconv' + ) + elif transform == 'unpool': + pl = specs + # print(pl.name, pl.output_shape) + layer = unpool(incoming, pl, name=name) + + return layer + + +def build_dec_layer(incoming, z_l, name, transform, specs, l, + combinator_type, layer2stats=None, last=False): + dirty_net = OrderedDict() + + if l > 0: + # transformation layer: dense, deconv, unpool + lname = 'dec_{}_{}'.format(l, transform if 'pool' in transform + else 'affine') + if transform in ['pool', 'unpool']: + W = None + else: + W = lasagne.init.GlorotUniform() + dirty_net[lname] = get_transform_layer(incoming, name+'_'+lname, + transform, specs, W) + layer2bn = dirty_net.values()[-1] + else: + layer2bn = incoming + + # batchnormalization ... u_l + ul_name = 'dec_batchn_u_{}'.format(l) + bn_broadcast_cond = layer2bn.output_shape[1] == 1 + if len(layer2bn.output_shape) == 4 and bn_broadcast_cond: + ax = (0, 1, 2, 3) + elif len(layer2bn.output_shape) == 2 and bn_broadcast_cond: + ax = (0, 1) + else: + ax = 'auto' + dirty_net[ul_name] = BatchNormLayer( + layer2bn, axes=ax, alpha=1., beta=None, gamma=None, + name=name+'_'+ul_name + ) + + # denoised latent \hat{z}_L-i + comb_name = 'dec_combinator_{}'.format(l) + dirty_net[comb_name] = CombinatorLayer( + z_l, dirty_net.values()[-1], combinator_type=combinator_type, + name=name+'_'+comb_name + ) + + if not last: + # batchnormalized latent \hat{z}_L-i^{BN} + layer2norm = dirty_net[comb_name] + bname = 'dec_batchn_z_{}'.format(l) + dirty_net[bname] = SharedNormLayer( + layer2stats, layer2norm, name=name+'_'+bname + ) + + return dirty_net + + +def build_decoder(dirty_net, clean_net, name, decoder_specs, combinator_type): + L = len(decoder_specs) - 1 + net = OrderedDict() + + # dirty_enc_affine_1 ... z_L + z_L = dirty_net['enc_noise_{}'.format(L)] + + # batchnormalize denoised latent using clean encoder's bn mean/inv_std + # without learning + enc_bname = 'enc_batchn_{}_norm'.format(L) + layer2stats = clean_net[enc_bname] + + # batchnorm and combinator + update = build_dec_layer( + dirty_net.values()[-1], z_L, name, 'N/A', None, 0, combinator_type, + layer2stats + ) + net.update(update) + + for i, (transform, specs) in enumerate(decoder_specs[:-1]): + # dirty_enc_affine_L-i ... z_l + z_l = dirty_net['enc_noise_{}'.format(L-i-1)] + enc_bname = 'enc_batchn_{}_norm'.format(L-i-1) + layer2stats = clean_net[enc_bname] + + if transform == 'unpool': + # print(dirty_net.keys(), specs) + specs = dirty_net.get(specs) + update = build_dec_layer( + net.values()[-1], z_l, name, transform, specs, i+1, + combinator_type, layer2stats + ) + net.update(update) + + # corrupted input ... z_0 + z_0 = dirty_net['input_corr'] + transform, specs = decoder_specs[-1] + + if transform == 'unpool': + specs = dirty_net.get(specs) + update = build_dec_layer( + net.values()[-1], z_0, name, transform, specs, i+2, + combinator_type, None, True + ) + net.update(update) + + return net + + +def build_model(encoder_specs, decoder_specs, p_drop_input, p_drop_hidden, + input_shape, batch_size=None, activation=rectify, + combinator_type='MILAudem'): + net = OrderedDict() + net['input'] = InputLayer( + (batch_size, ) + tuple(input_shape), name='input' + ) + # corrupted input + net['input_corr'] = GaussianNoiseLayer( + net['input'], sigma=p_drop_input, name='input_corr' + ) + + # dirty encoder + train_output_l, dirty_encoder = build_encoder( + net, encoder_specs, activation, 'dirty', p_drop_hidden, None + ) + + # clean encoder + clean_encoder = OrderedDict(net.items()[:1]) + eval_output_l, clean_net = build_encoder( + clean_encoder, encoder_specs, activation, 'clean', 0., dirty_encoder + ) + + # dirty decoder + dirty_decoder = build_decoder( + dirty_encoder, clean_net, 'dirty', decoder_specs, combinator_type + ) + + return (train_output_l, eval_output_l, dirty_encoder, dirty_decoder, + clean_encoder) + + +def get_mu_sigma_costs(hid): + shp = hid.shape + mu = hid.mean(0) + sigma = T.dot(hid.T, hid) / shp[0] + + C_mu = T.sum(mu ** 2) + C_sigma = T.diagonal(sigma - T.log(T.clip(sigma, 1e-15, 1))) + C_sigma -= - T.ones_like(C_sigma) + return C_mu, C_sigma.sum() # trace(C_sigma) + + +def build_costNstats(y_onehot, output_train, output_eval, num_labeled=None, + pseudo_labels=None): + pred = T.clip(output_train, 1e-15, 1) + N = num_labeled if num_labeled else pred.shape[0] + class_cost = xe(pred[:N], y_onehot[:N]).mean() + + if pseudo_labels == 'soft': + n = 0 if num_labeled else N + class_cost += xe(pred[n:], pred[n:]).mean() + elif pseudo_labels == 'hard': + M = y.shape[1] + n = 0 if num_labeled else N + pseudo_target = T.eye(M)[pred[n:].argmax(axis=1)] + class_cost += xe(pred[n:], pseudo_target).mean() + + pred = T.argmax(output_eval[:N], axis=1) + y = T.argmax(y_onehot[:N], axis=1) + accuracy = T.mean(T.eq(pred, y), dtype='float32') + + return class_cost, [accuracy] + + +def build_rec_costs(X, clean_net, dirty_net, decoder_specs, lambdas, + alphas=None, betas=None, use_extra_costs=False): + L = len(decoder_specs) + + # get clean and corresponding dirty latent layer output + z_clean_l = clean_net['input'] + z_dirty_l = dirty_net['dec_combinator_{}'.format(L)] + + z_clean = lasagne.layers.get_output(z_clean_l, X, deterministic=False) + z_dirty = lasagne.layers.get_output(z_dirty_l, X, deterministic=False) + + # squared error + cost = lambdas[L] * T.sqr(z_clean - z_dirty).mean() + if use_extra_costs: + C_mu, C_sigma = get_mu_sigma_costs(z_clean) + cost += alphas[L] * C_mu + betas[L] * C_sigma + + rec_costs = [cost] + + dec_batchns = [x for x in dirty_net.keys() if 'dec_batchn_z' in x][::-1] + + for l, name in enumerate(dec_batchns): + z_clean_l = clean_net['enc_batchn_{}_norm'.format(l)] + z_dirty_l = dirty_net[name] + + z_clean = lasagne.layers.get_output(z_clean_l, X, deterministic=False) + z_dirty = lasagne.layers.get_output(z_dirty_l, X, deterministic=False) + + cost = lambdas[l] * T.sqr(z_clean - z_dirty).mean() + if use_extra_costs: + C_mu, C_sigma = get_mu_sigma_costs(z_clean) + cost += alphas[l] * C_mu + betas[l] * C_sigma + + rec_costs.append(cost) + + return rec_costs diff --git a/examples/ladder_nets/train_ladder_nets.py b/examples/ladder_nets/train_ladder_nets.py new file mode 100644 index 0000000..04b3bd4 --- /dev/null +++ b/examples/ladder_nets/train_ladder_nets.py @@ -0,0 +1,193 @@ +from __future__ import print_function +from utils import load_data +from ladder_nets import * +import time +import lasagne +import cPickle +import numpy as np + +LEARNING_RATE = 0.1 +LR_DECREASE = 1. +BATCH_SIZE = 100 +INPUT_SHAPE = [1, 28, 28] +NUM_EPOCHS = 15 +COMBINATOR_TYPE = 'milaUDEM' # or 'curiousAI' +DROPOUT = 0.3 +EXTRA_COST = False # True +ALPHAS = None # [0.1]*3 +BETAS = None # [0.1]*3 +NUM_LABELED = None +PSEUDO_LABELS = None +CONV = True # False +POOL = True # False + +print ("Loading data...") +dataset = load_data() + +def get_encoder_settings(convolution, pooling): + if convolution and pooling: + settings = [('conv', (40, 8, 1, 0)), ('pool', (0, 2, 2, 0)), + ('conv', (10, 8, 1, 0)), ('pool', (0, 2, 2, 0))] + elif convolution: + settings = [('conv', (40, 15, 1, 0)), ('conv', (10, 14, 1, 0))] + else: + settings = [('dense', 500), ('dense', 10)] + + return settings + +def get_decoder_settings(convolution, pooling): + if convolution and pooling: + settings = [('unpool', 'enc_3_pool'), ('deconv', (40, 8, 1, 0)), + ('unpool', 'enc_1_pool'), ('deconv', (1, 8, 1, 0))] + elif convolution: + settings = [('deconv', (40, 14, 1, 0)), ('deconv', (1, 15, 1, 0))] + else: + settings = [('dense', 10), ('dense', 784)] + + return settings + +# build model +encoder_specs = get_encoder_settings(convolution=CONV, pooling=POOL) +decoder_specs = get_decoder_settings(convolution=CONV, pooling=POOL) +LAMBDAS = [1] * (len(decoder_specs) + 1) +input_shape = INPUT_SHAPE if CONV else np.prod(INPUT_SHAPE) + +print ("Building model ...") +train_output_l, eval_output_l, dirty_encoder, dirty_decoder, clean_encoder = \ + build_model(encoder_specs, decoder_specs, DROPOUT, DROPOUT, + input_shape=input_shape, combinator_type=COMBINATOR_TYPE) + +print (map(lambda x: (x.name, x.output_shape), dirty_encoder.values())) +print (map(lambda x: (x.name, x.output_shape), dirty_decoder.values())) + +# set up input/output variables +X = T.ftensor4('x') if CONV else T.fmatrix('X') +y = T.ivector('y') +y_onehot = lasagne.utils.one_hot(y, 10) + +# training output +output_train = lasagne.layers.get_output(train_output_l, X, + deterministic=False).flatten(2) + +# evaluation output. Also includes output of transform for plotting +output_eval = lasagne.layers.get_output(eval_output_l, X, + deterministic=True).flatten(2) + +# set up (possibly amortizable) lr, cost and updates +sh_lr = theano.shared(lasagne.utils.floatX(LEARNING_RATE)) + +print ("Building costs and updates ...") +class_cost, stats = build_costNstats(y_onehot, output_train, output_eval, + NUM_LABELED, PSEUDO_LABELS) + +rec_costs = build_rec_costs(X, clean_encoder, dirty_decoder, decoder_specs, + lambdas=LAMBDAS, alphas=ALPHAS, betas=BETAS, + use_extra_costs=EXTRA_COST) + +cost = class_cost + T.sum(rec_costs) +net_params = lasagne.layers.get_all_params(train_output_l, trainable=True) +updates = lasagne.updates.adam(cost, net_params, learning_rate=sh_lr) + +# get training and evaluation functions, cost = class_cost + T.sum(rec_costs) +batch_index = T.iscalar('batch_index') +batch_slice = slice(batch_index * BATCH_SIZE, (batch_index + 1) * BATCH_SIZE) + +print ("Compiling functions...") +train = theano.function([batch_index], [cost] + rec_costs, + updates=updates, givens={ + X: dataset['X_train'][batch_slice].reshape( + (-1,) + tuple(input_shape) + ), + y: dataset['y_train'][batch_slice], + }) + +eval = theano.function([batch_index], [cost] + stats, givens={ + X: dataset['X_valid'][batch_slice].reshape( + (-1,) + tuple(input_shape) + ), + y: dataset['y_valid'][batch_slice], + }) + +network_dump = {'train_output_layer': train_output_l, + 'eval_output_layer': eval_output_l, + 'dirty_net': dirty_decoder, + 'clean_net': clean_encoder, + 'x': X, + 'y': y, + 'output_eval': output_eval + } + +def save_dump(filename,param_values): + f = file(filename, 'wb') + cPickle.dump(param_values,f,protocol=cPickle.HIGHEST_PROTOCOL) + f.close() + + +def train_epoch(): + costs = [] + rec_costs = [] + for b in range(num_batches_train): + train_out = train(b) + train_cost = train_out[0] + rec_cost = train_out[1:] + + costs.append(train_cost) + rec_costs.append(rec_cost) + + return (np.mean(costs), np.mean(rec_costs, axis=0)) + + +def eval_epoch(): + costs = [] + accs = [] + for b in range(num_batches_valid): + eval_cost, eval_acc = eval(b) + costs.append(eval_cost) + accs.append(eval_acc) + + return np.mean(eval_cost), np.mean(eval_acc) + + +num_batches_train = dataset['num_examples_train'] // BATCH_SIZE +num_batches_valid = dataset['num_examples_valid'] // BATCH_SIZE + +train_costs, valid_costs, valid_accs = [], [], [] + +print ("Starting training...") +now = time.time() + +try: + for n in range(NUM_EPOCHS): + train_cost, rec_costs = train_epoch() + eval_cost, acc = eval_epoch() + + train_costs.append(train_cost) + valid_costs.append(eval_cost) + valid_accs.append(acc) + + print ("Epoch %d took %.3f s" % (n + 1, time.time() - now)) + now = time.time() + print ("Train cost {}, val cost {}, val acc {}".format(train_costs[-1], + valid_costs[-1], + valid_accs[-1])) + print ('\n'.join(['Layer #{} rec cost: {}'.format(i, c) for i, c + in enumerate(rec_costs)])) + + if (n+1) % 10 == 0: + new_lr = sh_lr.get_value() * LR_DECREASE + print ("New LR:", new_lr) + sh_lr.set_value(lasagne.utils.floatX(new_lr)) +except KeyboardInterrupt: + pass + +# uncomment if to save the learning curve +# save_dump('final_epoch_{}_accs_ladder_net_mnist.pkl'.format(n), +# zip(train_cost, valid_cost)) + +# uncomment if to save the params only +# save_dump('final_epoch_{}_ladder_net_mnist.pkl'.format(n), +# lasagne.layers.get_all_param_values(output_layer)) + +# uncomment if to save the whole network +# save_dump('final_epoch_{}_ladder_net_mnist.pkl'.format(n), +# network_dump) \ No newline at end of file diff --git a/examples/ladder_nets/utils.py b/examples/ladder_nets/utils.py new file mode 100644 index 0000000..8e039a3 --- /dev/null +++ b/examples/ladder_nets/utils.py @@ -0,0 +1,60 @@ +import numpy as np +import gzip +import cPickle as pickle + +import lasagne +import theano.misc.pkl_utils +import theano.tensor as T + +def pickle_load(f, encoding): + return pickle.load(f) + + +def load_data(shared_var=True): + """Get data with labels, split into training, validation and test set.""" + with gzip.open('./mnist.pkl.gz', 'rb') as f: + data = pickle_load(f, encoding='latin-1') + X_train, y_train = data[0] + X_valid, y_valid = data[1] + X_test, y_test = data[2] + + if shared_var: + return dict( + X_train=theano.shared(lasagne.utils.floatX(X_train)), + y_train=T.cast(theano.shared(y_train), 'int32'), + X_valid=theano.shared(lasagne.utils.floatX(X_valid)), + y_valid=T.cast(theano.shared(y_valid), 'int32'), + X_test=theano.shared(lasagne.utils.floatX(X_test)), + y_test=T.cast(theano.shared(y_test), 'int32'), + num_examples_train=X_train.shape[0], + num_examples_valid=X_valid.shape[0], + num_examples_test=X_test.shape[0], + input_dm=X_train.shape[1], + output_dim=10, + ) + else: + return dict( + X_train=np.float32(X_train), + y_train=np.int32(y_train), + X_valid=np.float32(X_valid), + y_valid=np.int32(y_valid), + X_test=np.float32(X_test), + y_test=np.int32(y_test), + num_examples_train=X_train.shape[0], + num_examples_valid=X_valid.shape[0], + num_examples_test=X_test.shape[0], + input_dm=X_train.shape[1], + output_dim=10, + ) + + +def softmax(vec, axis=1): + """ + The ND implementation of softmax nonlinearity applied over a specified + axis, which is by default the second dimension. + """ + xdev = vec - vec.max(axis, keepdims=True) + rval = T.exp(xdev)/(T.exp(xdev).sum(axis, keepdims=True)) + return rval + +unzip = lambda zipped: zip(*zipped) \ No newline at end of file