From 6fe6ba97481bd40126a6fa25d9bdc0ba9bc14b60 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 00:43:26 +0900
Subject: [PATCH 001/638] Update _config.yml

---
 _config.yml | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/_config.yml b/_config.yml
index 321142f30..e7415a957 100644
--- a/_config.yml
+++ b/_config.yml
@@ -3,16 +3,16 @@
 #
 
 # Name of your site (displayed in the header)
-name: Reverie
+name: 50+ days of AI
 
 # Name of the author
-author: Amit Merchant
+author: Ivan Ivanov
 
 # Short bio or description (displayed in the header)
-description: An elegant Jekyll theme
+description: Self-study and blog every day
 
 # URL of your avatar or profile pic (you could use your GitHub profile pic)
-avatar: /images/reverie.png
+avatar: https://media.licdn.com/dms/image/D5603AQHPdBBW-h7KqA/profile-displayphoto-shrink_800_800/0/1683545920956?e=1718841600&v=beta&t=NTLcQl7OtyyZIoQ1--64i_BfBcnUjT2pbe9GghPSYK8
 
 #
 # Flags below are optional
@@ -21,17 +21,17 @@ avatar: /images/reverie.png
 # Includes an icon in the footer for each username you enter
 footer-links:
   dribbble:
-  email: amit.d.merchant@gmail.com
+  email: isivanov98@outlook.com
   facebook:
   flickr:
-  github: amitmerchant1990/reverie
+  github: divakaivan
   instagram:
-  linkedin: 
+  linkedin: ivansivanov
   pinterest:
-  rss: rss
-  twitter: amit_merchant
+  rss: 
+  twitter: hi_iam_ivan
   stackoverflow: 
-  youtube: # channel/<your_long_string> or user/<user-name>
+  youtube: user/ivaninkorea
   googleplus: # anything in your profile username that comes after plus.google.com/
   playconsole:
   mastodon: # <your-mastodon-instance>/@<your-user-name> eg. fosstodon.org/@username
@@ -48,14 +48,14 @@ google_analytics: UA-43339302-11
 
 # Your website URL (e.g. http://amitmerchant1990.github.io or http://www.amitmerchant.com)
 # Used for Sitemap.xml and your RSS feed
-url: http://www.amitmerchant.com/reverie
-enforce_ssl: https://www.amitmerchant.com/reverie
+url: http://ivanstudyblog.github.io
+enforce_ssl: https://ivanstudyblog.github.io
 
 # If you're hosting your site at a Project repository on GitHub pages
 # (http://yourusername.github.io/repository-name)
 # and NOT your User repository (http://yourusername.github.io)
 # then add in the baseurl here, like this: "/repository-name"
-baseurl: "/reverie"
+baseurl: 
 
 #
 # !! You don't need to change any of the configuration flags below !!
@@ -99,4 +99,3 @@ exclude:
   - LICENSE
   - README.md
   - CNAME
-

From 50e628926b3851591e2d85ae742839adbea473a0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 00:47:04 +0900
Subject: [PATCH 002/638] Update _config.yml

---
 _config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_config.yml b/_config.yml
index e7415a957..a72bdce81 100644
--- a/_config.yml
+++ b/_config.yml
@@ -12,7 +12,7 @@ author: Ivan Ivanov
 description: Self-study and blog every day
 
 # URL of your avatar or profile pic (you could use your GitHub profile pic)
-avatar: https://media.licdn.com/dms/image/D5603AQHPdBBW-h7KqA/profile-displayphoto-shrink_800_800/0/1683545920956?e=1718841600&v=beta&t=NTLcQl7OtyyZIoQ1--64i_BfBcnUjT2pbe9GghPSYK8
+avatar:
 
 #
 # Flags below are optional

From 3d2cf5e9e64547853026dd133c31b157145cd75e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 00:49:09 +0900
Subject: [PATCH 003/638] add avatar

---
 avatar.jpeg | Bin 0 -> 39775 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 avatar.jpeg

diff --git a/avatar.jpeg b/avatar.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..04ed19a0650af4e76e9d0bc0de3ae4c57ff9a1dd
GIT binary patch
literal 39775
zcmb4pRahNCx9kQ2gy8NFoZ#*f+}%C6yL%wP-QC?`;~rc$ZX0)ZcP{@q=boqga%<-6
zdFXGnr&h17S|3Xvn*bDPaVc>C1Oxy8@wov$RsbRZC`icv?4S7ZxkG=2hW_#e`Wq}P
z%vXeO2ng`s;NcOG&`=POP?6x_QLs=@(J?SFF%gilaj-CO&@eDD{&NussLyl0K*K>p
z!(kx8BVzo2<D(aV`V}G;A{GjQ6aa||0fh?j(GMU5006KM|9Ra13kb;1M}&d>3V{3U
z#)kkvefIu8`ahNd2v86JNE9fP&sUwquRb;;=8A-n%=0Y2XWyNoyJTJFo~arPWRX2a
zcN!Uxt}@^ZKcu~P&fUjiKjARfT$m3#%*2I4G>j47*V7~^%fWkuj?*q3Yw&6dck|cy
zI!0psju}SnjHF8DgGTY|`xVC@dMES&+*kN{=4|`3ItTi{_Pf?Y89?qTz9B7Ppl?>b
zp<LuHxORqeCOS63yk(jpmH}bTlnUjL!C~VE8V8;<q==LO#iquHXOS2-T&oi4PWi4~
zkW6tAd=P}7YT0hbw<Kn?F3B&g%FrsP&aQm-ZW<s0PtJIn?dW@YXztazvB!nS6J{!D
zrb^~$KCTBk^+m*P(cB|Xi@WkDg4x!P67CQW#bLZ6!=IWhWSgprPO1E!xLNIH^_IU7
z`)o2J6k)T?4;XrK|J1NEr9jFeQB&zqHW(L@NmwvKu`^wF8EXF0P$iQ;3hF(QfbM_6
zjGI&kPE%S<OwFbql^(YyCy+)KYR{B9h3H<JfeX#4RbfLwk4SBJBOJBzQTT;EX9nEP
z>|cYz3R2m2^Vx%>$L$EADxO(g+t6mQqudzzZ;1sxBQ|wAFpfBVxh48!`{x*`uvxL<
zC{!l4h1K!)0%@^r5E-!~8?`|FNJb#7#?+&9eT7PA*H$^RrixYfrH+6uaf`Q3DRpe=
zyj{W-(z)EViYr+eK?p~_;~ehfsE%~3ERH4@3+MNv8gCw(gZ^+^SY|}*RJ)es4!2>H
zGL6h*7Fn&FY4@J8ozZSrrh*9mDKOi-UP{upYn8KShe&l{o|~(T$vvC1qzYm6HCFC9
zhr?8pcYLYfJ^7*oe8uqLhK@1P$`nB%qCW&KvN&axY*FblS|L6Nw<EsM^X<k3=0pm#
zis)Ymey37klHq5k-A4&G37lw4$iEKhi;CCgelLq-V%+|!TKk@j$7SKQ18B*AJhX-k
z-wfKtZD*Zvjm%Mw(r4u}RP5NAdVO*Ljwh;~aU37m85^+U9oVdueGLIIXuG&N7f~c4
zWUFl_q8J5;44}y29C01C*#u}31*fXXlqAOe=|iQ?9h4Q%n?}(*e8h$%tjvHr`?mjd
z&E`9kv^MBXhZ(}%VZ~jXHn!Z(DL1dmjQ9e3x@#4Y|8U9al4=b6?qu8=YTA4donkp7
zbr}7xi?L?+HJyLkGng>|k_gwb$?fmFSDx1xvVsr&H)jO~8zgj!%inJofXVGgi7UQu
zSx{`PQ%X_XeZ{jg+&m&yksHYcd5TW!ejxD_*}Ce4fks9J+C7&Mh8RJDs0I4E5}uq~
z{Cmf(30DgAT-QP0+^4Qs^nv}=A{4;oBf#g!<2qJ?rPjXIMoFuPfY?Lo2>i;Ckmj&L
zXetf>FhKnKXO8ze1j;3iNl^F`vw7)a;h%KY&U!-cTlmwU3*^Q;Gg~6tiN?X=gOZ^(
zNwM^LW`gQ|k=}*ySE-}`KE*2^Rh*-8X2C=ur;|mrg{fLQj2dCVp2Iw})H3UHho;s$
zn4y1YDq|ak2T>;i=KqXJFOh?Z9)62<LE$n(BgI!%-sEW9?79n9|3C(v$ZD0HXdAod
zYd?4J9=pOUUu(@5<0e07?fvEak|;s@WnBC>><q4p8eXx~&z>vWLFwCZ1^>!mew~G_
z1+b9&_=t;|>Vo<m-*}uJbdj9&%ybE{f#zWc;S^te^UHc>hX^G6g(uV|Dy~V2i`#_!
zBI#JPV@O5$VtMk5qVxO@1BG6>y10uS4D<&okrNQn_(YL(3CV%$c%pQS2j8F4xWV!=
z>KtBEekc%LtEy4TBCnXAP&FQiZeh~ttRVLOmBYmU%*m*s_R?*Z<btR?`;bU}w#AzR
zg~{<;Q81n{l4MW12qwmKj6ZetyRG$8b|>ZO{hf8n;7#nC`-GOZ*rCsza@L{DMbdoq
z(d3G$v1a`o>A3*fJ|HBXIn<2CC7R8nujxmd0%25K^kt`)A1fjLp=Z&^z!Dkk(Yx}`
z$bbaONXC>CPJK#|f{cG~uFfu)Iq7W@2Le$lfNPh>_25Rv_+L{SVh5cu%$)*uiPOMN
zg?)aNCU@pZPi3#|b_v#F;BBV0Bzm&;(9hF5^|5U6;|<RTw}Vsn9d84s;>+AY>cH~v
zhJC+$nQW=TX$aT;HcxV<mNa600B8)Ry4gf~Jxk!quy<b^-0%h>QgV+fiA8RV>r9Y*
z1PhK;zVGb5SH9j%`S2fZp;;^%NF8vib%whfulO03zLu0CF4&W-HmVhI-{dmIE<=|O
zv%nLe24`GXyMfJ8cc$65u0|mNO8s4^1A>TRC``$`&akCC5T*Yj$`WR8oRO_;`-pQ-
zH5Uc$blxDfX$>jk^}B55yV5uKsTf=$YG7^cL%pFQJFD#(i8iK&a?cw^>=Rz#8Af`t
z(jCF*T8OW8EK*R<VvWDXYnI&Z$u%JBge(%9?{wtYG$2cHW_~$E@?W4|c=ulZ#>k6D
zQ5^}r9>jeMDahW@uAEwFF=uI5ogL65lMt2NZev#Waz4GQ+R^U2m%Z)b{s1Tszej|{
zJ2TvCpZPUPKE>qQuXYg6`F3eYv3Z|MTjl??6tXzC&f)Z;Dz`7?`Qi<Jg<{wDY}!g_
z)*HY20I2UuCGKh_B5oxj03eAVE<Fn109rP-Z~SVab}s$D8oXWjP5%1KV(m6ZGM1=`
zNokS;+vdY*Sr!~-3Rg}#N~%^EGYqlLQgW7fqiL%T^a`2L*3}Pbj6(UPMl8=WuO#YD
zqC!xXfjo+Y&XWt$I`C{;z)4K!U-^a?ZuETz2R}_?gdnY|Spr4@kdVk8Q2!utWA$XJ
z8%ypbR__L?Y+3nnOc2Gg#1;#=A1aM4Mt=Nw*K^kWlY63<>PSnG>D}>niBa1hD-!_-
z)JtPIjAX2=ow``*@ytR=9o$lFoD^--@a<AuV5LXqYaZ{G8}r%S)7+z5-a()u`aaTI
zf28=U(K}cmn0bl|S|y|VFd-+WJ#g;`BA}K*b+k-bVh9k3r2p-fXO9nNo?!I@;5Dgj
zKi*+{VB3DIFTGUXIjBKeZCqSse89gWDQzYuo#1q#aFt@F$}HFw5@=lO>F<8k&IaQt
z%LpsJh5Bb-wc5oBl(V=iqdoQjdP<3$`_-Xm-8GX~S-;gq6YWXAUcevsVC7e$b)Xsr
zpCw^crAw3MC1VZN;cMr$y16M3j%=3!bJVsx8JO3P+FYe5Gd-$9;B(-ajY{phs@^AM
z;`7&_Kkb*WqSx%Too$y*^RaW)H?$}$>_t#loI9#dVe1Pz$tMVFWeiiJsGOo$9W)sw
zPijPzAE&nY!Mu?`rKI6ubc(-e4QSZj&1e2iE%P+hZC~Ikw70KVrVU<6ipW@>TdU{A
zo1Rvzn^5NRtv>gY=p<s0%-kW8?=qIZLUkoZbPX&Av^5RY$=OBZc2Tg?oW2>A#q<5?
z@ty1=+8{O2$6wqFOSLB1b!$+?lKJP)jAL1v=Dbk(B6Z7cXL0tF5>$z{oKb>X0m7(e
z`CEuHBr~)y(!M`@;)NyTY%b5<#D7g$nB%9DuR3LxqTHt2>Ww0eRBc&86=F-!ZbsFy
z%Wi)nzs)`->WW=s+~r>&qCL*G1z=@Pz{vN=0D8=B?C`7WbLQ(6gG#VP_KXA242fQ$
z>l-o){Ht8lC{kB_4=gp+hkq<i8nVz<GL)9+s1K9qNl!sl71;>u0yKS4Ah--w&g!af
zHD7$sQ?l>Y&{ExE{vLVZ`5fUcrRIA!Tu7l*mgFzVna6r$QwYiQ=xo1lZpEm;lt<Xc
zWkg5;DdG3zlSrTcg^4G2YhL@ST~B!^|6@i_s&vtiZNzs~)|Zz|Vmgk!^-OTz&!5fV
zxF(?MhssS3k3HHQl%ftHF2kyy``cBPbmYr-Z?k?UO}hVHQZ?#BDj66%GQ&K>m42Dy
ze~E2oLACorfi;0o8F)Z&=K!>_j15U}Ha&X(HSbv?A;ZalSp;K5Haj=kfj8h6AIT$F
z*qL0{70*hBNJ$m}=lIoGx*YqV^{H{}GvQ}N0YH3#goOB9r$BxlpaB3#z~|Bf8Wjc&
z9R-5~7L)Wl851)W3%L+08wDl%Phru|g}~R(^#KGl<X`X6tH|p$N^fRy(jl^887oqB
zCqmMgy&NkNuoSVdY+5%Q{%($S<f}2n2LO$qFM%1E@>Er`$jovV%NmWvx-fg$I=}N&
zX6;|+fV&(yQlRADZp?WX*Qpc<ZI=ziXO5;Y%Pe#sTsG$I&Fdv^)StUHcJDdV&|$>N
z=)PZ)o=cYoHg0|x&%T~>Q2l`Z=YJ7-dR0Ki<oS}6I;5LSlzNz%k|VDiz={ua*U{0r
zGcmrJ7tajyBsW<wGq+9vKGZY%(mV`yp44~^SJgMSLZ6c=ut`lcnx2EMvQkXJws62;
z$pM`DTAUW<{Fobe`Z!PWxDNpJMw{oCqExn3H!}pyU3{@$Wa#Xf_3IJi=R1_(-4)ya
zz<xTxUjZL@H1DzY8Chy(8{eKk08(q>XS}=JJJAcDSac<L!%k(xYG9T|hYI7p<={3{
z@^9`JRkrb%Wi`%b!sWrmP$SSFAvE?T;pZk=G;M5b>{cfZ_WN`GnS;ofjP1R-HT@4t
zg#|1-vNmk!<WJt)zsI!2V_}mVZWAUt3>8n6N@{IfTavrU6Ou!k%!cnceKjI0y8wRx
zY(Qy?B=+yTz|*F7hOy7KsVUzLmFZOEI_vgDFU57uT`%Q7H~mCfQf{PXq^$P>)^r`4
zEwsfx)T_P>L+)JJGK?8lf!glryHPMx({u3fXc5*a=!sKBSAaT$U|}bUzn08XGl%c&
zw29WH3%hr*ILoJW1sGn6kbpXBXFpYZ%>VhB-9GOFcGp{&f|C~dJ3QPy<P=)#i8DLK
zf9kn(ew`r&zXCu>r=|Y&$N2qRcyQ_oB_?VltS6sYS;NWcZ%{Y0ms_ZbIUL4GNJEZg
z)faCXN9GY#!g|s(xs8M7q0dpYwRKai;LmS19!_lv$eO-GWp?^yoByiu`G%(86f@b=
zv=kV#kl3ncxgXh#_{g5*FeYbs*RFVdPYbNU)~FHVNdGj58>$%&wO$aqxXmu~C8(UP
zt)%N%hi!LsIVAY(9nf#W$UZ-4Gl=>AEz-LFHp&A$J5V^qn_Fm-p4jgOR|c!Vnp*m=
z4P}sN0RT;oh#UJQ+!pe*sa&znPMcWnMng@o(1=<fd#k)5^|_^Tqx$Fbh+JDi%J9m{
zA7HRR3^`+|AgV`a!wrU+3BO*(b=~{aAH9I1)H0pxp{7xD2R|YKUCq?0W+T&t55S`1
z+=7JZbRtO^eaQK(t2`-p2SlVLWlpDO=o}+&BFDIDvO;AVn$rW1iWGs;(0BRT4n~p;
zAA?ywQ_dP?yx-@A6uf6n$h}L+L)O6XpzxdI&K5Rf<QJ}2nrh(>07DVFM?u5{)|Oj}
zw(v$$^6T$W&RSDCXRI-d4?t!W__2Q-!Pj7bI}E>jFaGK8iptVExaqG~$1^Sr)<xKF
z(Q7B(+SzNa*=Y7(S-Vh&S90e@9vyZLmd0EIt5=R2B`v3<@11&hX--0~KZxT~3hL2v
z(4|*&HZ{HF1ZQfaWkaCGW<7qzmpzHW@mwEO+9|mUZ)D{>)ENI`+rN_YBL32HKRwo@
z>p>PDM2$Nu-h1%=1nW;&%I8K0tN}<uBGR(Ev3zqZ{rL+H;GGsb^ipk)4vfa8ivDS$
z?o&1Jo;{MFRz4t)+1S`{9yK=>-~7r)kS)vmY-@k3J+#olEm(YOQyGIcx)l_01E>9?
zq?(ajslMaxUIC8f@U-ovEp>A><eJZl@^T&v*8jedR*XU179u3wbNjf1cEy#-Ofi8i
zxr(U(%;%dXezxxMFWiWc0C29$7|<y6CLbo<3b-H~h#C2NxBLlno!c=-SvNhi7Q|7x
zomT;ZXmwBC2<F=)_#|E^;^wA%p^T8!B*`xPwtAelR$W)VHJhi@xIYM0X7nIdIA*_9
zf8=>=VS1zqdW_%Z#L!z+iu{}>-O9?9Q};uc^=IM!a_?1D&-+{h1XE5#w9aVnl&Spn
zgKv6%Kb3|TP*u^0aEN%AKLAchyTI!JFeha;@sj}Q1>Hu+z_rLeFVdfDPu&0nZv~RC
ziGNQ59cx+?jz0ipbLEfjRXqw0RsL9SaH$!*J%+#snlWYnGwjXM(_JGsUCm6xDExQz
zF6k*(4ZTi~+uLM}!W}Y|`?C2m<!LsDTkmEBYh0N!4}0?L2jEL!<+m-^nTu=I=Y0pY
zdd2-(b%*+#5oNRM?uP*#rI=p9bN`2iXY0PtAG6)^nG&hrKQZFvC$An`RLYm$w{E<q
zoV$~$`C!e{C?6ur)_v&p8qz`icfzXA0iD^SwqFEYdb82>4AJ+Sh5}pmm%b?l^M!B(
zV&@!_FG##Lxtlw+Qui8(!F~GxDA7OLioDSg37&%;%^=+0sZ6ZShOL^}M<kqoG_E||
zM`{bTOC2ZBS-<$C(C)^YR?h5j(!)87sSV2W+cycUE)irqY{6;QYsMhU)4B<4K(!j3
z@X(3L39xiTbKzC;8K5Gv`fu9n3ymILZBF=#jGBx}$s51By}{6bUoBUiOvmj(#x6v*
z%c~DbbfrkYs<u?7b$Pr$&>Hhwh-Ro)F7#H>S^njz@&T}`oA$6gvw!p~VX#5qq$9XS
ze@}14Yu5Py{7Xlz$EMFcQwT@4=44>5R2+6%3EOFK@T|)e^%8>XQXqeTwMQK{E=?Kr
zJ#};9>=QgPS#R|THDU(S&gj?wI2$82A$vm|RTALL+-DxcypwMj4VJE!r+T-O?8S++
z<Fk;*-10axBPnCIS#=)UkDg;)is$Gv63%4)!-|z!#QEdr2LPAc(8sbdqU-}8!(uO2
zQ_js0H?woIYczPZStX!@Z%m8*n3;9Zzf|)B0;*W<NdENJ^xvCoEGF&zwW2@%k34uL
zQ%6C|db)lGH8^`IqsYd<LUXmzO~krAwJc7W@|*-!Ck7G8Xnipixy;gwf%5@-wxiki
z;5*Y1M`Fj_9enilRtyD7$$TvxrBO742e)*Pj>F#N^t_vEW3#Lzq7f@r9qZMExdGco
z|9bKX;%E`GMlGc}3|{=d2XnG`k!cZV1*7Zg7kO_{1C+u3mQRM<LA@8>nvCVtiDY=s
zusqx(bk@6-Dt&Qg3frXmp2F(no>ezDK*W<PtXW<d1E)9rE5>=MUUakWX27$0;o%h_
zV>*SjXnmt4yZ+_GO(US(2&#?^`2P7%>nH%JUiIr4C|K1ez#w@UWZ!G9jhT@I#?r?N
z4!)GJ(tLf~Z}3>&L%-X%>qaH+eKT`ABhNb{Pr$Olg_Z8VVoyPlH)I3g07oRFyO>A(
zSu#I9J#?T(Qu?@UbmeqvKiZI8>sD3P(jH{Ol_xK6!f^;;cZz0A$&Oyx&WUlL)%23)
z()&i@n@!bx^quZHo{iW=lqX>(!Cs(;G}@4~(n$Ozc%bt@|7o$y^#NAR#yhZ4Fv$DC
zz`@nI&$GqgdtA48awq#yGDxjwez&81@_PbtO#A}l83&^JPBQ4|S32-HPNgS-Q^}n@
zWLty9d524%mnbMza}Q_HlUxW_yp=J%!T;`+ASY^Qm_{M6PJ6@%6qzn9<aXc+4Xno8
zw|4Pck~#a^weZ<gs8JJv+%xs-+<i>KuC9yon6RKIACCPJVo3=O=BTacf~6<)*HgYQ
zJmT^}7SxgsB*J0&M`-V{_T4T-Sv;welp$jwGjQ+Ot(nAq{bKEXJGmN8lloc+vcGFC
zHh|M$M=}pJV-DBM+Dv-+A$2F7V<oVel9|%VevT-_omfBNCj>mdFj;J6yD!wNtemSE
z_zohjwx-6hu|2Jdj2zTU@S(KrhJkN67E-<nH7c)f?qjiKkdfFh^vhYhRwSK`<1uu%
zs~p)|aZ*poOLQz(d1>>{>-<des<)0!u#Ark|E6WqkGrI{5`D{#Inww57%8Mta7Itu
z3&i>#Y(44ko!s#<=$kQSRaG55f?vkX<x2eW_vHhkax52!KLCOIX5%^9&c005-Q3@V
z{NR(ihlZl0=$7oamNyFPzHG&Bu~kXbsGCCFCeOXI(^c*L*=Lj#yqRlJWpR{d6%S2k
z0`1ArCtNe-37l@wG(p}KZ|Z9=t-bQTa`U#cm@B;b3RSArXaG6OnO?1>7DumqRtgB)
zT4a#Tv7aA~y5*{#eFH^beHI^YBVPT)Vh2F-e34G`gzd@Dol9?lqK|OshNhtrus;NG
z$Oya{lz10*W@Jxq#_B5H3by99ysqyxI|`IraqGftXN!Gn$n33Cl^;@xpxsvuyWj2_
zK+iuu0R13;q7Q(`$<*F2o4f8uEnK2mSIw}|F3HMJ<Q6ke%H{B3Nv@+dZPQ6s%qw00
z)e+VaRGMp^ZS%7_=iRb~CLmCG?CK%qtZYDVdtC?%@&`s-WKaKi9bbX|-xa-O&U3r7
zYw(GX{NnYQP*&i)LEDQr(EMaTU*0%|a#ZiFBttoIEiki-+v1v<0VBEN?qvDQh)7Pk
zFcApxOX!7j-fw;1=odP(t1?-<Hscw!p4Xj8yYZUVrYCZ%qxteHh)s|Va$;nWBkYPV
zkQ>HgF$B1f^0wRxO?{Ku)|kfBq`gl2_;6(QH^rff&%w-&zx&stH}jC)ih|sj@+e&&
z7P#dzub&uHICJJI!?|YtMJHnouXwDIuP@#4!u+7Sa?WbinNtq{Wt#=R3(NkE96P-G
z0Ib-bNP!ps)o`o|k{r13FJS-K#oLEKzq9G)?Kjfy{yuDe;#=MkmE9*5RzL70`s(N7
zC)}mne03%h9&Kpzh157CQd*mY&(o`JzMcV$kinJVH2)oIz-WgiCMayVi??@tn7)w?
z6*&vz7bESNtuE5$POj<gd7nz%8`@Pub>whR7RwlCYt0Y~NoZ&CbEyk4<2Xxee$URv
zk9gc%06wbKs~U@%Q_IbOR<9GSCO)5kqRK9_KXI&v<4-T2+k2X^<n9Of;D4?r5#uWD
zVjSN)zzCW_(sj!be(%hd3~+*T2!GyEM&{G?$A>OSCuYEpEAbG^+lp>w;msFyD+bhS
z1q3M^z1!*vCpaufrU{d~v^#8Cc8myjHjc`s66h3^!I`BFjx8pR^aRLiRhY}l%G7it
zpEAQ8bCyOuW{bh2->j)`@n}_8X5q+{TeAXH<UN?GKSyB{3Kv!rkBlO;*70$3=T1^5
zO4XH<uu{cWd=T*~dBQH@b4R)^?O&M@B99K0zZOsjVLQ)?B@T{Mg8XG)+@J2{T{A~_
z8A-8a%k5~{YPiAtgsg^U?omOb%y`#fLUGeV*lx{y?X)gDCBAeLB%Oa!;O@kWiqu!J
z4ew?atCN#(GGi#L!9wG7kzlN$&P@8w;<teTxq>m*S6ctB1|G0eCHg_Fo9M}8#K7{H
z_D+s8SrsGWSPaol+|M>kF>L358graw>}99FnsUC<Enc3|dlM<*iWEW=Deh5x04_)1
zab6f#e7tW9J^&@w+ucu?X=j?dlk$%H5N#gEy1*2Y9-?pi3xf3a*bMl`W#I$n)MmyC
z&wPQ|ie?3TvfS3}ok=Hp)yAg{N+w@n`W*t2sH`e*6Qc)tsvdokfk``Iz&kZ~GBbi^
z%mH)qA)|65BLlT_f>v=tdOAKLn<%{O<<j1N(RG@F&9sRX3n6hsZ1<5Izd34uD;syN
zB?Dq#KLFQ=9;e@vFC#-d{#_~cu{|DN4|An=))86FKcR?_7il0!FOZ8fjN+qdB*5LV
zk_Z{L7Bh0>%*u=&?a%ZG8m6W7u^TMwJp=;-_aC*}!q-T{b$=0B8OdJleG`sGf85!b
zMQ`kucYf(8(Z1bQr%@f1yMu?-Q@GPX`KC~D32eSITdj)Q1M|g6dG$|1JQ*2IJ8)j_
zgf~w{X#FN6^{Z%SJToVCuV;IvszuP1UA7`t-g-+4f4#h0sefyeFb>pILF>KAEXFnT
zD>P=ICWjM=*cQ0(@DN}#3L#n=!;%;y(g0yxa;E!aNqlSZ9A-{EKHv}o4Gik<J7Ppx
zMCBG?Hy4j|atl4R!gm*4yv7YQC|MR`N4+vcvL8jJ?7Y8c5ue|wMAyr1V7-h+b>qdO
z(Zr3`c^u#T!wIjhn2XMTjw7~FV%gb9Wb#8ZWI15q;H@FKUPVwd<%(vMRUjQUM6l}l
znR&ZN5|6)N<V4V!S6f#XpawD}`J6Z#9F5I(Ycc4@J^=QJnJ>XI?-lqmo*w|Qooj8`
zQKJh!-^7fmb770L#jwOUOJP5_WLWllDchSD7CHsW`Xzz9i2Q^35y#hmX1W4IRA3_m
zxeaOV{*~<J1jwH+JN{UvzXJ_Wxuz;rWTEKObVNC>F2TPmGU$BoyEqE3G=fi?3mOdS
zV`0MwB*`;t6g}%2EBLCHN?bg;Dx#I{pT5e}IUeil4wZEk#ND5c8efT^^R_hiARGNM
z)7EwoY2>=|VtJz7A+<d7jgdM=pmuo6I5nxzXqWX#6MdrnwPp27ZLig_w_3uJEIYSz
zbi8~FV~;=dR~!&COCq*f=3MRNtrw1{$Zsfl2&BHp6Ce+XMcFJ$4UT7y11=4w&r8RR
z$S0vT>W4m-WRIJ0(nZE?+ae@?cSGyx*(ywXW6aS;K8lMI!kc-#_V+84?z`6n^MC>~
z`jSYOK|XpVjIZ(&`AKOq`Q8``APqTV>>X=VoQBmYoaJ$foqh-jV@9~7<9Rcq0)&5z
z>guh8TD|k6Z!3x7YkvHr3>!n&0xIr=>U?1<(fCB}9sL|9tUXb{NnFu)b__R4WJ51g
z>7eo^M<>juFE~((TzX8|AynyZ?qZ}D^UJoJt%h>H!rYfRRIRfrh7cNT9_iDQ|Mp)g
z0e4Q%SnjbjR!$k4CdST0`<Gh1-l|nKohj~J?-hdk8G$wRz!$EP-unE(MwN(tp4ScJ
z(2!W?pA&OsaoO%e8owg=rWNzvg-oPwIOJ7gL$CON+Dbduu_k%r@ASQv?NHr^dsMU0
z{%hd2-^W<-#;z*|KxZYSu@`OmF^HC%r))l48C3CU2{L99UxE9x%d?{H5c@IU`HH!j
z8++4iY7O<-9`r8Ky5#)St2ObNe}H_+mV`Q;2Aaj>@*@=c*)`X&WAbUqz{=NGC(jjh
zN8|F{LwQx;Q(l20^X<m0wRN(n^Hmw$?>ua$_c$3mve3ACmiJYOtvuH^FT=Gz)M7;&
zGQeeXy)Ja#Y6ZIax-Bn}kX7l<xro(tTFQ>y&K^dGlZj*Q=Zu`>Dt<n{drCVok8epf
znX!>+z%O9Po2j$`7u<r;-lcc(g9hm4q5kG)XMZkM`;plQQZrH7XC=zJ<$nJMPT!hS
z_m%mqG}dC@YUOp=pZ+>JddZ6>>(Lo6L)a;pLLUGkpv?yWX0|<)B~)ip6DPQiXTP`C
zn99SV|K8N>cs)Z$M@uq$kh;5BK4K5e70+b7<?Cx&pnvwW8{2jK^{b}<U&QEGte}s4
zLsRq7rveDJMK;#tHaJr&Nt7$F8-)CHJAcwK;X{W5E%YS}cRwR7Gnc3I0z(j^IaJkq
zu=a|kIb74xY`$8-wd~K!QGuKv3o!Q9pW-Q5g1>);zi(YlC)mKJA&1-EHlrnj-8J!v
zHreQ2nbDB+#V6wInz4DfYt{N}uxnZ7tgxVrPIP+oudK$WT1|`kX(d5Geu0Mm-x3uF
zNK_~^k}oLVnb9#wg_NL~Sd<MNF^&F9rT#2s5ri;+y70-jq)wEK*P~C?Dg}<w0jG^D
zy8SG}C7hyW8J?O9<kxqMkAP~A=@v6;-$*U-CU#yhMe?7^l5CRFL)v&rHykfou|;Z}
zUvY*&45}6co{DuF&J0|=DAh`S`LZ#TrA97;@uzZ}E$s-#XiL#7N{?(U4`j5QoWNEx
zfydmZBk}Y-H?H`Q5yE*ixgq0~B+bnM;)J?)MEpE|FH)0@V1o-aWMe%$ZJQAatl{=>
zpR8~L%9=osexF5kqWf0SJq2MCy5+e?b|>R%VCf>=xs;8`2nB}2Y=XS(m7i7uUuUG$
zp6_#9#C*6T+t|*j^f$es-k$c|7RwPm0trOQu*>wBRS#O?5G(;C{qbw(_IfNqL|5ZK
zY0(VOZ<Pk=B=Kvw8SyWgJ7r-Dt8vuDGHCY{*hFtW0^#nfLAbQ5+vG)p3gE>en;-3U
zrY0=Klg}POce-fj5}^ZWURu=N)h&AtEc>3Aq{>cqVKVFwOOZu-1Kb>K34D4-49dw_
zBVDLxrHY%MQW6)lhmMRfk>uuC;th`3eexUYaIUaYqLe&}xRf-ji*eF}cmR3O%zV0L
zJNm0c%2Cbk{O!gO!YXG#G2@K+P_KB05Ci0W+#foYp{XY~s;ut*b`xHwX`;v<(w34t
z%+zc<GxHI4+R#uijOZb=ZvuJImtwCN4%|KnQXc^MeM{xM0wQkEoKjI$(OecUi+Qr-
zk-zbX!>4YwTkBWI)cY+zY$sSunJ)(UIa>!Si@gq0I&u_Tq&@@<jqoE=H7~(+qm1_D
zF#DQMv`Hd0lBqCC{oz3ObO2}9WjuA2SU*95EsE^6z>Qj<-<E#3^9f9sC_$QHG?2|%
z7i~>ihRej>;gxV0VDccVv=m9}q;k*f>{OL{+dYzcgr3aL+q^o{2*(3XeQb_W7p^<d
zK=a{CF)ZJ&t%;Y;UfpS?8Y3&z3bWBNGd<USAY(s`vc6+yoRephSE?T5Yiz^D+3pl|
zGMZ2bev`l^+I>tLt7lZ-5mvXbNM60+Z>3M6r8rilunP)fbi&EhOQX<_<1C6+Hsh=l
z9Dt`l*#U|zx=66G&<|<Uw%c%}E|%fB$%oxco=SMSkde;vGMg+Mhh&v3U44>W*xj4R
zc$+HCzpP@q^s4C6W9H4%qScLUY9$_Pe17Foz(=AkoV}aoAf6|=m%3;)P7h(8WYk94
zBF-B*b30?B%!PzJ>JwqkBTvZ1y0@J?twU#Q{{KsGs4oyu|I6_ICOGpa!-YN>{@(;A
z`zi8Y1c&?&!LbR`O4~^XDZMZA!`Z?xlw^xls7<}jcZ|c@&PHk4Nk}_zyYo{!3hbRg
zs+3Dhr_<h<_(c7TSf?5X=8)oXHWSiVD~MX1+5L<#FItuIFIrWkXZP_-qCu_LMtF^N
zSW8)~OMkVz<FimxnkcjT^KG;xozL5k&Dy~k@P<;~M7tJC+u}~erV7JkE_uB%pU{7>
z$hTbSKIsUrP33tb(bJe1?Ws$t{-yCrcUzhjR<*nrACXy1>0Sz!_Kq(h3_+~&B0rZE
z|B|sf^!3<W<!gDk0P$ro#pxLJP7LuQ87C6-oOG#94moNFOGJZ6#mpnwobJ06*$LSh
zvBtMeP{OXCs>WIO_J{~L#_+y|O%Rd7bM%El@&nMhVZPZR5N&+uR1LS4NhpQrc{C@^
z_JDy_B^->m<JgOMDMNr3r~gCi%66GnC0|f&B5wCn!_9q;?52_DO;7=S0LD|~l4xgl
z{uF1sH-Fiomdi)YOp8F`Ye`6<QVL{NFZTJ9;HQ`zCx!5*44zoO8`|{)utmqO04lpS
z9GM@z#cNxBr#@y@!05UI)4D_omWT_wNtjYsMc!4VNPu`|Qr8}`$BYST8dH`TrQBYu
z*l~e}po&)>gq-_$y5lRK8|zo16+$F_Ep|cSxOm4%zkBH#dwx9k8`<YiUv24m{Ikin
z+;syZ#1EcREgZZL&Fyun%eb2hHE7lNP5|vG&4gKE1QhO@F4a!sX2piS)o(ei<$Nhh
z11AurdMe((W$A&-BQ1C1g=gaU%Irogog2&z%w1s_P_Lw%#NnLe3}Aw#5i$1)Q%H9j
z!sy45MQ4piI@7$O3~XdIE8JYAqk=@y5>?~U*qS8jz3CME;Cdekth47gbhxoC3A&cE
zlxBw1MgCMIjm8xZz_>Vp{~c>A@pArnmmBp@kb4oJ{S36;gK)@Z43o}ahBe66CMPL+
zg+`Q5FJ&$|PoI*P=-^p>k2Lg}i)1rG2Z@a~UpI~zrU1;~?U!|$F^%<#8*M;|T24K~
zF8*h`(M@AbqQ9GJu$#ZeFpirrqWh>r^4Q#7nyyY@iRD$f%wcFU=&@fsHQgVQI7BL`
zm!9FZ>&f%d^syf}?2n2JJez&F@OSs=cw`Bup0Pyz<fFu=Jq!g21^opU<_k0=^rt=i
zSpo`$O7aDb8HJQZND1B0G49hKW))TrfW|O#%IjfMiLd)<JiT>E4rCWe$iFgi4*WCo
zKbA3!AY|E;Zp9*N4EeDIUHwQ2<(=4&N>hk(xmt=BnQ&Rz?YI2CcfF+DAW&3qM*I$K
z*`+T1wR%+eRSf(2WcsFu0nRg{hkudx{hRQ^IBisywbDVoSpGDzky2D~;T4y^3GpSX
zSBa=M6PR2O6-$lh8A`EC@AziF6th8%hpgQh<X(LzFEikhAJRkQ2hX6~CRnQC3<RlK
zu+lDca(m;SZ3|eh^MbTB7(E(c4;#$mH;o?TG$KAHuC>UR@^;-WMv8u)IEZ3*SJ%L_
z2{vR0V{NuuyXdE{c>9FHxukDuS^Cd(n-_?R4nk|()uwh!d3s8FV$iyH)DifdN{T^x
z$xVn~W36yIU6~Nsima3lJ23e}_p1~zwdrlooTSWs7!<c`WR0pzRE`dPut#^jt1~^M
zZLO1(%Fk45rtQ-opbgb<xvh(@turO$=09h&!4`iGv1E<?13AckKLzvk>WloXYn|$6
znit)3(D6g5N~XO|adtbKHa%v!8e(#V(ye?|uug?N`{xy@%6XUXL<K>pxyPz@t_sqw
za%Z1*cfal1g0=@~$!iFB{@F>5>J>^8WL6k`elAZYeq8&SC8EGC%+hKPwIQmz%yU9(
zg$W(l{#!6iYu+ih%Ij9yB@a{YT$RToPo;An-H4~a#r`LT(`tIUM!Y3M?uBr)y`EZi
z3Bh&Jq*;YpKURUgaJv*OIN~u;@0o^%Gp8QWSSHgGx;>!(Q?OL}R92`y0FqUwSR`G{
zN%(91;I9nsqN{u&O#A2Mc*hI$JE|}$@_R-i-EeZHj&}B8d_-;ZGcDC+sjlkYKJ(Tp
z@)g1jBo1>Q0M~ZAN?Kr~Mj8!AxaMk7$LhqZWM+QiVJVCOH0K`8dhY?9?Y0xZPJD%l
zB=NDDC+Ma8?{O4LQ-xUm>~yb`_J!#!>4X%c=a2KlIU#rWpSrJVxNVr~wwg#mW>Aic
zr8+6#GT}WFcc@0O=yp#`b)L0v`_<k9)AV1Z`nuHgtgX`7%W3RK)d({VTe3GWEwOQQ
zg&gZajze#fY=CVqjA1fMH=UL{>Ne@-)^Bqdnuia_?9vAL8;zda*eNUdb!DnBC3c#5
z@#ZeZRslYAz0I$Se!~PB%fymt?-l+{83QMX4;Jl|X4Z`tw`4(W!mHi-7Eb%w#~MSX
z3rBULH)1iHg86wm3m1BdLSI*XjXk7cUu>+qLjU&Kcd1bel(i};zbki@+ThbkL++PL
zY0{Fx2(8~*Q%r-L=PG*-I?dvG4)%;d+K=U28a)uDd<!;>qM;d^>4%qrIR07mA=B)S
zza5p4v^hsi0x3n;BQb87F3x|RfkOKv0gJrfe7X*~rql>^D}MlQ>8PhP2FypKZ?*LL
z*S^&@{R^zHQ86_vnSj*`DvtnyuF^4Rj*F+c1$&4wIM|B0BG*RiK1(O$GcO0}1hL7N
zGUQfrFtzCxF0~KKfrK+yTI@gZkiu#Wx7T33RC-b3)!#1AZv=%l`MC(!g#*?!k&l|T
z%M%O@W8G^>VF>yR_IN@C;Un2#1I<c^+wVaqaOKVvmWGF0VzypYQV^=}l_1D&w5jib
z9;babt~$MyuAy3%ci>yC7%X4x`!+n%Y~F-!7AsGew69Yn-i_bF3&ga+CGtgP92Z)t
zYo66~J^T~Tv_p19_z?2piR0dL9yJa1Vj+)4Q>`4-#;Aj9cpJZFr%%56L-PmgiXtTX
zeUGTEal?J%O3ylO_vrGduGv)^fKc7?oVs=z-bxzXtU_S}^6f>)U$)OG2}Xz&DQcG=
z&%M4`Vb0nFn3L1iH~~oT?JiCO7-&6KIyL8<^9!;};~MF&w0>5^kdNCh(^W15(Jvog
ze(;70mWQ;&J-@IVkszcz#VwTxkUR`aa5%=~pB7cOm*iWx<4g~$u3EMDl`8I?lW<6@
zwc$~r`D(RGw-4}?wCmmEPtnk>5rvD#+H9x7+k_T{u*;Osly(cdXjF8l!KSY}<gewz
z4TdG>5(Jtr<1DBfdn{dqbc44AX}tZk1nsT7C9h>h)vfCH&T2ms3-ebXc3Yizh3mpT
zFs?7Zh_2B7Da0)NwnX)0wyj+5BDDB%ns-1wz|yX)ReVRdh8&=V*v(Uo>43da2iamV
zv7^v`Vae)MTPefab3aF#drdw0yHNQiV(Z7WbRYg;ZbeD^VX5?}YE$(yTtzvzxas#I
z&e#K)0@M73Jr(*Sx*CQ2xpGV_JOLm1LktJu)w)4y*ai|K(pTc855Vs+TFdWSu~t}8
zL7E;Wc!Wt`F}YqXDDxYX-%Y84|Do<ecQST#cD1AW!(89bjD#nOAnL9T_-`|Rnws6y
z$_Rc$B2xnIYf}?F<aWDCE=XvGlW;tbG}8@Nx+GG1RuSu)3sF+4fn@Xoq(HHd)4i&V
z)lO&D<~#XhN^G_bQ6b}iOI-?V=iP>7H=&S6YYmU@Q`FEQjd!y{$OjH^<9)+mL+tTc
zBmW+(7FA3ok4gU2)J#i|?+T(pThjuc@1ZGSl4q+rRbX@V#h1a6U+a`ahKA&(%NHrB
zT7|eIdpK`%kAWL%8MKhR0p7Lc#`f|2iE@Zfp&FPMG+Ik5mr!T?-Y-&5!V`@5@;1u8
z6U52%KGV+Y2Q4>_HIKrJ-vZm}<HR-P5F$@OsEq6MBJtbsewxevsIvXv&}L;phh39s
z?Ofrc?J(XWbj#E`2QLL84yB-jTlj^x?!OjQ#lu6a{^k!p?1WJsAAngxmA$0BYp%K0
z)i;{0D^yuQsEYS5b<dc5^4n0X(pev$<z}eV{k(;a7ATe=e5Bvg+!K9bp?O6nD$(Pj
zgr#PGvQ&0`E$|bGWSSlCl(ZU;)0a98_v<BJyMsd6b0IccZLBTJ@=Jg@*pzV*NF}k#
z+mvpW<m^=D7Kh|E=G*km1!>WA{6aqwl+yleWq88uTpl3#c|4@I%hH`@b^m?w5#38$
zL$`u-wB*)b-s7vTA0EGz^CScTsM@&J=2leotW-p|&_km(?VwNGv+b1|mZozLn(z#~
z!COo}Pqg&jLe3*6@3<WIFTN<O{lj&(HyE$1Ng+8FKZN_ejWhKI_qCqVh=+!|sHA%u
zO^9~OV}Dk3scY8UV&j0-pj6m~tty=r)HQ!4Ni!6Vai5-`Br;zHvee5lTVT_gQTW9N
z`B^x}Y+E~qGxl6A&|f^E)j~kOSv~Z`fc^Ac(%n~hjlq7Jy(;!GNrb(%^yP=UdyM(*
zy>G+ZDQQ$L+p8ESYY=QNo2r9jb8q@;v3i`*nuQ8y*0T5yY3lvBrg|$xbdvsGk*8e`
zYz9GbGZH$3`_tpMe_OO7?e@8eF_wypiLtM<2)E8m1D5%V%BH(rx4!YxwpCt_>F=I~
z^cOeaTSe?8fzmUo5bIR688XQE`c&16Pr94yktX&-`Vqr(liqQYWGC$BiM6FGWAN|S
zJ0V9`8r3cZ)PaSHhn>cR!DsjqcE`;QX96UOTRUgSZV{^io6XLg$DO*l@4q5UV&rYx
z7LHPNy^q;U4j%4XB0Gd7R6xo_Fl_NEH+XFm<Rt>(l(Wgp$mLhc-(G28+gj~7b@U~h
zkzq|A4?X}--*7X3P{9vLAo@mfHAVC~wf{S<Q=7-NZi5co%V@Qa`o~R28!g+gYu8Lh
z)#(lFoYp@Y_G+ez?G;}Cy;zE^7RtSohb{NdH&+I~fZziV0tSUAPRBKM@sBloKwMwM
z^3L~Gp{HNM6Hp6DARH&}A=`Rk^hleksd5mGxi(yhoL{vwP8Rq~x$&Zyx{qLnq>qIJ
z#UX9Ec1(FMFv4C+!jL3gyPut!1S)Cn+9?<H$=P%;JOxg}(@6BQmxxJF@v2`=+o{ZV
zMf|zbB`%+Os>b%rr{0pk({9`Ik=fR7tR}ZOH$<+lv!8%%P5b~X`-Wv;QYwuUgssDS
z&l|4ve0^KMQnwEg$K)lgo|;})WZ3Zci-4zo+#&mv?z57i*=q^`H*RG`<1cf6wo?=+
z<N&*vAar0o1$U?3-a)6~)us0mvB;C2{e_sXZxr3C4FAdpz~r#_j?tcqG6Zi?ne8jg
z-ELZ{Dymfy834ntJZPt3ih-m0KO1F%zSSJ|{vTUsvCF4weXFWmvMfcL?tYeui;LTO
z{+_7g9nr%}MDz~-{q~Tfc^(_t=%lq5(yBh9eg8@JVwKHGxQl6Jnu+sNw}UqqO?NH?
z1SO~oBy+q6<U2J+t}sEuz@TZ7B_w<#ufO9415C|MxgfF-c}X*O^VK;E2DePsmCOlv
zrw3{<>zhhd=NQ-|tHEI_uE|-+1(|CYmctP_|FK=$KCsMog+vU71PE0+Htl?){a&H6
z-x8DlYur@<H@b8qO9NF`%n=jg_i{6#uuUYpb&y0J$EP9vb&XJOghyAjI&P?J))uDR
zgQ7?3#2_tT&T5R3B$l&~e*EHC@UZ+oQ>M!@+W%e^0H}5|GA`2XS>`<Y0DS8=M5>ii
zdeVEpQ`cLuNn?|PK4W9x3p}IjzmFjZ3JkW-%6(y75uUk7)hrMMr$(;6;C9dR89CJS
zEsh<k8gH@-5Jm8*bf^rVEKg!`!0n%{o9DrT&o(X2biL>A18X0<vzjz`amANT^rgUA
zigH!OllWAIQE5Mz-p@Wq{<v_~w6)s&INXH+<Y&8E2nu8woYh}yW>y@buh%QMyuL4r
zX9hAIFMg|ewtwbaPZQJGJ8WD-b@zg46v<vFmfr{%iRU<TC|%M$0nn(Eq3K)@Y7>dX
zAFT$c{=D~%@Uv`IU4jc`3<9Ta<yoqMe@f&94-J^3Ao`=bC~o8Sk%<h=yQnUysn+ra
zizg=``Fj(Wd%Vik-9K~t$OyOQo~%wYb!XMg6NO4>6X~5u(0@b91A>JAqG_<HV;<EL
z-JgI_4@|vc@^{ZK({n;c5%T7c_>#c*9nyIPx?HI|G!a4X6*>31B1cU|Ft=0<AAlTD
zhusUqY^_^P`0h@`#FdD?SM-1|^XTZej?dyL#3Zeg<E)7`j00?hO!ebG4bX-cSH%zI
zJi`I5M!$DYY_jU@c~KB-%+H%8?ZYX??Zb1GmWx&fu&|Q_i<B~${hafjRAKaFY`4Y?
z-_Dz3(wmXvzo^jo*kc;6B$iz&%qSXvYHM~mUF+s8q|k!2c1q(c4>tc4uEZ!Gc3or6
z_Re0^Gb=feToW~Is=e1)J-$5En?(gj({*B_$4r|*&B=7lNiakJiIKu;_>zerR7f-<
zQ+$y1QU<{LKYGCBgd(N{(+<bwT@!2Ly+a=Whf1e6%w5lC`gI~x@ji1?xSNiMn;$by
zyS4G?8iTx?zL<eW&onJ-_y*OJ&@ckbpn6a14*>71iRsc-D$!ZS1i8zso#(y@{qc<$
z$4}D<c<<6$Ae@e11*SK5j%`5LU#5QdWjPF!)}rb^-(5W`MftM{v+{~HpB^eFO!2qd
z;h_lU2ihw8;>3^QTetFQFX+_G*{2jb)uINZXtjx3rc^t#$T6#gZS1ij#Jv;12n`X)
z$2}MOQKq0(HIv!=(H`+)x8DaA`QzuPx!{N%UXiYijhC~-YM=RrlUf<ML&9f#|NLN1
z-UT7em|2BcQ>onJP`B?2j3by+*=PcjeS*Q@LWko_Gs}Md-e9R{a~f;(7NXXXAb$db
z7P)@ILNsxI1#7pww#xoP{l0n0HJvpp8;iJ{^Y-a55Py|T0k^%VufbunrcZJNI&W62
z@DN@Ow2I?;)dyg3>O3riEaxXB#5&AI8!u<p8?8)v(yzikf93{ST%_;G>bFJmh9-wh
zk48tZ158DzXIBa%*Bm_}=q&_iH)|l6Mv(Q-T&VaRxGCBF<ZNHms3BfI>}c!8zIK-@
zaSz>28K68@o?F3+j_5(Jj3&NL^;9@_pND&S@>KJkrrSRn$rcHU44v=W7Op?BRWr`Y
z`~D3n#mjZ@OxaYAPjYKU=;XK6mM1!S3W5#mh|yD7J#OCIEb956w7E0LmW^>9Uh6I}
zbbj~jzdGfBzn1j7$;LNDj~2@gpo0UCwV3+fTw&u^xuxbut5dUgk)0eZzyBpm-M8wx
z^~ubjV{~!ilwe!qKyH)>CMkINOA#nOhTq^UCjoq?Uzd4oe)_&>Oisex*-_&zUHO+g
zx1C<M&^e+VTazUDpLQv1zPwFUr?9PQ3do*E*q>!V;aC(_5_aFf+)ZqdZ?f80XT|U<
zN$8+?#OYuLBCpEY+IkM5v#BE{KAWOWlIp1sC@2p+Xk03Iq1K9!(0v)sc_(TJCc^PB
zqg^yAN*EY@A@i8S#37$4)kxLg!9Y|KlI0&6)|#skz317<2O$oP)eE4xi*whj=j7Y?
zTMRk?{h6E1Ur6ZCfEJVeeFmm#sWVw#^!qAh=KTd+p1|Y_Q+M*0_;btV{IGb0tHi$1
z4C!>70Y`+pIFDNelSn#m8|^J&ls|Cr#YwAyCLaK+(N!Vrmv%0{lJtHVLR?DyQLLEM
zqup{h_5uyO??23fGQL-*mAN`J$QM|tKNi_>K<0bc#_&+St4erPA*$J{VecvM{nHB=
z1Me4yO}{O3Laf#VMsd#?{zW2SkuYkfd#ga~YjR7K71?s%qL#Q%#2vaZs=L;zO-+`0
zba8n0`qUmcLHgoPWxO^u(J6Tf##V=anrTO1xo=@z@Awk`)xylPu^wYvEo&~#i$6#y
z2R}+IGWlwBSuq+`Sp7{M$2X1j`IVeA?WW1c=i%lDQ9VwH_Lk`AmUuz+*7R?Rwj+Hq
z^a@kCjnjB9?H_x7gMo+J&X0C%Az(OPU29JxCWOf;p~9DAVP~HPgI;2aSK4Ittu@j8
zOx{v8-J!<q6V~O^33cmZ;8uZin(B~mE+KxIsVCv>kDINcGb6j!#wB9>HT=P7%Y1}^
zfas8l$`1e=KEYe!XGEjXAr^Vea4pB|Z)q+S-FF^R4&}jc(rW~Eg%<{-Qc0iuX)B0o
z9=A%vp(*<duV%0H)6e=D=i2X=pj>AYgPHgsJT{@mm978om#PH?#y=Y6|D+|hzmt;o
zm|VDo*YukZ(_+2O@D7YrpVXrgk5Tvxz1Z-df}O<%YGW-GGc>d+I5bdHj;y#1b2j8s
z7u;#f>!M2nH=;mS_)wx79wAT8dhDs`n`W;*0v7zy$d*%$UyJN3uxS}I&=<-zAxbUl
z<W23w5(<)&r;0f;WjqMi(8s{XBImmljIV_i0UPV>92&F@va6uvTlq@J;D!7^!KB|_
z#1_1-2eybu^>4|_PC3ab2FL0J#svaoX6#wovv2nj>!8G9Wp2t~@M1Yakr=ze_|yqa
z*Y^W667S#wa5FVOhTY*?YLlzk@JU!-Uy2JX#|hb$hgQ060z(;aYiT&u6_M{VOrKVx
zqdg9dR47cFM+Kx+T5w33JXU|>1$$4E?>o4%$tUm=qn3`xnEiwAv9#>OUNehfca@8Z
zHMMpTstzoq{6AE^b8Kec8})tHZtvP|r?zcOZQC}dw(Y5H+is_t+O}=Z)9>$n@+5C^
z{yxb$$zFTs`mD7!Q@sfq)o5q23UDqXrj-vn$3ElW((zhHzhF;nQ(d)Wq_bp<oNg*3
z!%tOGJ3%@sa|y$V<`*^1&|cOqG6LTCNluCVPuNAP+>M<kCB$lpVN{oKzDd?aQ?IRK
zxNCZ}WeI$8JWb?r7Db$xqjLNiA`@Tl76|U9sqyk%ti3&l;;CdO*7xjHYE^2`&SlJ#
z-{&`HIS8!sL(6VI6yVN$pa>4!I4{~%4o?xQ8)BLN&T|a(1`zz5WQ0^#s1CMI{6grF
zY{BYedQpoN#ioov%s+}UCV$Y*TZG*Ed*04{{!LqoQ3+({MFblJh#b3_3nd!1Lsa8z
zJJ^(YoPAYOYAgj(R2<z&8EtW{h`4^#7Y<e621Xe-BJ^qMI+VDG1fOJgT<k98DV?1m
z03`StR!R=F^VZuLO^cIWHO<<Y@s{~R$sQs`AJ&$9+&=kedzlA#KKkokD7)D8PG)AM
z#gkmXnv{?`O)LDPDjAo$F&J1R!2|2At$%J3%}T-B<=czC4ee?<iEU<<>qT*xoSB{0
z54?u!TN;^<%g|K{!d)y63chA!Kgs@fV&Qa>qw7dF<CT)j&g*5&eD)q>;RsT$to>za
zL(vhSFk;bdnP)MHY)s9NTE=R@ht!1h4<JwRS!(HY()>Upg-|DgUX-v~U8|x=G)8OJ
z0o7WMuIhDF&7PHZ3-=Q8A>2b$!;HUExsLOo_c^i6)$cVZ<_Wv+s@~=F9IJOb5i`L^
zmh-T1!svn7LE&h2R1j<k&c&dz5*24&0K)GkI`DQcK^c>GH=cgc$5r6(pCwn!PNhNi
zjRSb)Y848+Y%W&4mq^^%S0}W7y$M(l-jz2t;>l^5InCw7wCvysWNlw%(CDvrN<0TI
zwT2V~o;{1ys=n7mJ?VO-)U<n43u@Kq!6`JOowWt-5x|AhGs(v2y?^UAGr!O5i!%A;
zyC8#1*>ajwLZG_VlhqGe)&`C;D`p-fB7=ollRwT6`h#N!!SQ`>4<+UGpWCF;Asd!r
z7zF$nu)jE;M!(NuZ(z!wDXrPFnA}&?2M{$AI2U;|5lYRg!H$PC@@IjcORZ{~Z`Hmj
zx?U~5=d5;2S!=;U&iBhv+|RXN#n<{r-TNQQxLx1RQt6bH_cHRm;Iinb_vEVHVvpsN
zAreKrrY<QMbVpe_-^i$YB0X#AIAnPO>yuGRo?NuQkOt;mHTzTP>8Ve?w%`2NKY*o#
z>A?Joq2GJ;*U3xGHM$SGFMadstonoeEtFj*>jUd5$OqMU#<o0q<@BP3_iP1dRs!!^
z<rxMlkj;2rcY(H0&MHXg)R+XP2eZl3ONP|b&HtjOvRhjpU4GNr9=+PPvbS1gFq`d3
zJ1=>uS78q|2ord(BV-jbqh5h%(cx8|Uoi+7*^w-0bwA|8P0^bFAjoZ{oNgmjT8P%}
zY*Jo2t6rxT;FWTf)&b{F4{U-SxoDS~@vPCa-qmWhS{zy&SscN{x>L1jmv_*fN9jI=
z`vzB(z;96XzgXk93FLoOCm;Z1VkRL)Lq}jhLVkV!EYZ#FF5~|tQy>DwT9<SBu`N3N
z1Ta0rq8k0)e+4)|iqmC(4zDKL6k8Q%@2x!`<PG!fIDk1T>X8(xuwm0f)<*cVaUhmc
z28%Kn5+)Dpd5L~eso-ujKzomaMl7-0AAuh7Nk=+$QcXM+`Mf~)zlIg@jwJIYF3zA$
z)3eOKAY<xlZjDyHq&Q{Jw>|WVMKft}x#RmASX7)(j`TPZYj&u3v+8sS{$&|5&K;*&
zB)DmFlVC_#k#;~xj@+Xyu3DJthXOKs6*L8<ivp)b!6WmBHlJ3(?dHQxj1weRJ2-sy
zkxe*dykZDQyau}Bb=cWMY=hc$cJ3j(8bP+eikf>BqoN?1F~XBi89Sova)CJgaUI}?
z_VI@9%h*dlho8olo-mYiLz#oGE!f5H%iU#&=a2XjD&OT$LGqYqZ$F4RSi&??#;B|+
zwSnO!_z43ap_m-_Em&}{otN-FhCjQgT2>ttTVP4jwc1k0Z@DtS#B-cMD}0q;Qt4r?
zfo*KPaB(e3nL~f4Ud-%Hc7)r!62A0w#12y{_N7d1Mj<g8WD;S@sQb<*$tThnX5B97
zb(G@=TvRsT&`wXL8V|&M1ri2T(x{2T*atyJDd5VJ;D#Dwj>KW+nq>qFoxMbstHwSc
zF`e*lhFd}uWf|S@23{lKAw{YPx;zNMo8+ynU&Df$Piu|n`Or?O=9IgN8Y^kf5i`2~
z-X<ZBs#t2GR)8eF%DBZ%s<*7>-2DRtj2nLN5Q|2+s{R8QxTk7HKD%nvgsoBQbymy6
zPcC)T%~pD3AS+!tT*EYEYu_WQK_W^DvfFdVz={qRp{Z9CCTfft!z+^TAs=39AevJ;
zEnS=(nWcIub?ft!&}l^nnnT=4{}KoU`mu7$IC2ks9J8V<=?&|&rTMU*Gc0M>-X8qI
zGCB$Bt5D;2XA98^rupl_7&2Hrj}hBij}6Pnb(5eeKAS$F^G05AfGl@|Nc1$PJ;HES
z9@j<9$$iT*Zr`+Br{I3QWTT^@gM4PEx@{Sbq3kW*{Uwz)q!8A%TkZz0h3msfb{T$4
zqZL%K(0h}EE%}`)lBAH&P|VAbWrCT_e<(cQ$oIcw<zs_NT`eol7I^78D-W}VYJAHr
zVfzO#-(&h5T~l(5P+Hv}qM`qu;3}v>fiIaMxuDzBaj7uTpJ!v;EcY#6;}Sz)XYmyj
z!`{Vj{aR;<Mf~fG!9l6GM$vkzl?fVaMW!hKgIL?k)%;4~qufXgbEU^c+JS1^Hf%zr
zNY=5*5d5Te7;@EX04nd-Q{sG`3v@(z)x$ac#zni)t1CO!lpx-rONvKlL_KeWM(JCe
z;yvm)?|g-6R`XH`0MVXAI+}Ju;cys2M%Ng9Ga4E?pezgHYNHu-LfagfVZF{p%2X4l
zd#4X`ol^@jYlPuk@F+G7I`nTR&k)qa!ZiS_)~0-of1d2FZ->_rZ5Zf0)t_8ShGZU{
z<^e4YDS#$-c%8sBZ4yUwC6xYBP?3|i6+?RlpsfDgfLVS?a+9x-e>yn=MQVeWCd?8}
z{d@bz`=HISNEghhF17xUJ4>K)Wm(?2oHWDiz4A?q2KD`G)GbEKvua#F;*hB3bXp~1
zvo?VzzZs`f>-DGMD*Y9lEL4PFm|o%{rb-iDv<~1q3qkK1Wa<(=88@Pridb4q7z<|L
zZga{YC017OD;?HfS87k)FGKWx79PNhCfF$xS@#)k8G!^h2YsI@eG0|Dr|+2O8R~@F
z*=XOyccd9^v8%|eS=4~H4GstSH5LLzIfh1inePzloaYbqPAd4`{J}oC8ad&O3AYy^
zsugGH_|EVhVlr?o*rzqS{vFpP6`*}tmfkC8E9gRw;cG<Wkc0r%kJg4IqLlNJO^OKa
z1Nu^CSvEC}%s{HE7<qaPx}0dDbq<?Cm$Ky|<*ijJ)OpEJVesBFza`KmfVXew3ge1T
zs%BUoTQXn}xQMGIE|tYwYg2@VNt)r-V7T>$9!RR7hPqKnlVQf*(3@Vc!=?8potx&+
z0Fn^H@3c)JFK;J}VZMqd2j(H{%ujn4Uo2Ds@wn6?Lg*k@yu?T-!T0#g#=dvx%Beg~
zAAUo<0wF2FCAQ&=s0V5nmkb4JC-e|P9~#sfTV%Jz6{TfuIJkk&7&cS7Gkjguqoq9V
zawP>UeGXZrYn|YpA(%bqB3-PbBdH@EiyYZy9l1n<^Y%luYND4iF*N9iDvdCfh-g%u
zu|;eFMj2u*MbeUr>0-{q=MBi>3K~c!F=GzjIBPHUnX!j7NElvtLh`BkgzH;pwj}YD
z(695twy_Fd99vwrQ>yfsibT}pyWb(9+gGoP0)r=60KdNyyCCcg<*JJ5he@psK`-wH
zrAIznYw0cxC1<?8cXBaVqT$FQh&zGv!kup;+;I|s9`jikT*x-0iQFYx7UEzbZADpA
zH2l|J@JmQ+vJZ0Cz{Z#7y;FKfqdkqDRxc~mPfjIwQ?>BaZ))T)q8oF1dCOiD9&9_U
z38Rgfu_n5T!eHYm7}@}lsA1<OG)?%Y+5UItZH>aXkl_==nImvvkhbs|HHNPQvZDP5
z`Ej}jYt=onrXd)~h8Azm2$&ldOsIjn33uStTKJsZu42)yXROhC`lfulrUn@`r)AUG
z^~oM%Pc*H;5Oc=iZN@g(8Bxj!w$mphQ1DHKB^^!g({z0qbCF9-f=|0C^+Oq0;bSmw
zb<iG_vV|z=t{l+^B;ub9KST#n#z<CFO~Y=b9qpp5jV7m49CWk}S$Ed@y*^8KTHN)V
zfyeu3MnUeH!NEzAo!oBenT@;C?~5c(tF-qIfZ&54F7PwE`Poq*aHl_w9==z0_y-#o
zB-Xm+!!H)$GuH7z#02xUnRB26Dt@s$0~uS7ri<LWDDO;VBn1nC=t?}cSQYt(7@VoL
z?jYZ+@Gsa)`bA}U=GK^TaqyFj09dSj8Azb}+|xjtWrwUdnofVz*-*P7-%TpU_Q~s)
zJ3TxW^C#TH=)?E2b8d3(*T!>mLWVM-u%?Z>DuLE~pX@?;ml5thOGC0R7F<Uau8*fY
z@!0_%`T0|>@wZ|dR<rQr4c{~7OySw!!XCV|rq9&FVdEu@J}Uw=uYUgb`wdSBe`11p
zm|r2=A_-ol>PMh$cSEQU7=mT4osfPZND*3wC<U+X@D2pV)sRZHW0cL9Ycw`oD=V%M
zA^$aOmhn!2P5!gCX>phw=z>QOgvpe0st`=BJfI)=bvSzGb%6a4eo6BW0G9>{kHsI9
zA00ilDHl6Ap-@It>;Sn}H4<vN_(J{=t6SFtliot(f+Z8$x2QWcK;PUUnEtG;JF^{B
zu;5zJAsaR#oCo$>&=k>kB@-n;2KyOlk8*_h@Qpu4cptQ{b?tukNFQ{G`+574BUCz6
zzd|am@Z7J@D2-kR2einXO8+j>$4<@h$tZ1<^BxO!fSuVRV%vo5Y`r<T*a76yMK+z7
zys1t<N0B`RCot&EJQer@>B-f9A}IVM8^CalWoUe2hLR_hstoDoivNc4MPRK`pEZ$o
zg4B%1+>QF*_c6?bv3*j~Z*tm|h~_gd4y(OV^6CnUzz$6u2oCBt(<^>NHs#FqxPh*H
zk#lCKhGW^w7(9nXh4>z!on+3?=j8850gT1@XfSlzx<SBD1u%Rv4^p+4WfWo03qn4i
zz1^iyT%6^4dEQ1`Q{K&{tB{s)lg5T779VP*AKRUFCUA~*hxr<*a?{<94kqU#VvX9t
znqTe1{k1R=&jX0vQg7jUAW2n)s0yZu2&{~D2&m~4hFA=HO#n4%@MEvUv9z^Aj%dax
z>P5sSRu+c0pd*39{2WJ02%ztHrF(c%kP<#$QtUT3ZSyggMF_(Dw?e<QQyxyh)FrOG
z<QxpF8|Pm)+MN#H7&J>)!-t_u*x-%&|F|4M^|wI2<ro{s1Ymaj84@%N4IR*7-5{Ff
zs|**+cSP&A0#b75p#-B3Geg>HYyK<-ZNGX%mwr@mgf!*&=`$k1*9go~QIzcZX}TwU
zR60C^c8)*N=(6Aidmhg-Aj~<OEqbibNX>TmbbSH~!QBwCV2j5G#S+P8{`W(4vQ}JR
zU>BU2$Bsv{Eu2O=8Cw@inF=l&Yu&#v-<h%KNgM>TOP>j}KW3uZOlUAYg<UV7HKxGu
z+<3Imak}^|Y=I?11d{ec{4QGxBXJRrnC=;^OVp#g;f!gCq382#k0jA?yD5L)xqXRP
zdf+z7r@ZbChJykB>joDwlJwrwP`E(BWkBpXo<P7u;fKUe&To$dh?pJ{Wwl_)lrCo^
zC}tuDt!%^i&7!`C?7Ejkfyxhrg8oSk(2c%N(>R(S+H%C0UW!d#;mJLIRUe64J~8b2
ze!rHl+t$FGhO+fFbvN(gS1Ev;zudG1#jjB+J4vz4#f))@ER|N4>YLFoa8t>~i14gG
zLT_dwPX0@C=zP_;^!?1d318^c1c92A8D0)NHAE@%k$AmKQ?vy%TYM_#$ShcIKf7_f
zMY0QX4A4M5+v6S2E#8#4f<L1RH!%P#k1;*euDN#RX*SfGuLL)c`ra!&V%HzkD#U1Y
zE=Xn+Xc~OU$)IA=5=Bm_lKe#?_JL}F6NT!N=7mxzzkgN+Lyg1TiGno6u&SrK71ia3
zy_-89QzBR}w#gtoyl0c1>J<wk2uEy9%!rCxQ$YEaK$s;4vi-a#P(F`1E+icCBA8>m
zkpjt$mk=zoz}rNhRA!D<a4o!7SnRyoY<g`24<*hEF)sG}TPT^TI3+Ka0<D7Uz)xBr
zH|GpVK2K0_PAHvr!e}t1Q&GCOno>j?CQvqr5fMnjrQ4{8F>As$n{Gj{i^oDd4ENVF
zO45P}C_rW|QbHs|w8OWeUyY}kSZ9q1#DS9I)A@xwb96$gVdAPceS1XdTxoED1HC%Q
zrs}w9b{f@ePw|LZnic6?Z{7Z<)XMNGo(wJwW|7l>^d0{n0LTfLt76s{xU(9kYjb|=
z4L9Q?>-*qJNmIDm&&^H01fe^02EB#rI_yarN{~>`*p@S`81j)FKq`qhtp#IP5s2s$
z!$AB8C~7le)Fk)U917VWaT>&yYQYPC9CE)2D+=X?K4JlHHiOuxku&@Tx8J>E8bLu0
z-n+~%1K%PRu9cf9nX~oM2^RQ+SD2(AUCV-mPa%9w**ymlOTONP9~(z`;sueQpH^6l
zTW<GFkQJ(_qDS)Zk?I<qp!G`V2Jt>Db}WVJuMx%yC;hpi8nifO6;A=%*J(`Uto2K1
zH~B8ex|r&tB)ij&egHP2O~dt#kpC6OrgM7L`aD(T^7=d+@dC;I61z0}!@&a|vIbn+
z3L&A#U)Z*OndcB^=nXe*QQe1LmNt)U!@>y^y+y_mtXQ;QRVQd#*9h6>_HT4&(QJY}
zi2l=KQ4|c;xJgBI%wENT(K$k8klu8OnL!YZ4&g(XY5Ync!fEFXsoI&4D35>kHIK{u
zu`EL=@&O^p(yQwwHO5tK<akGQJY48*NS?j>uVnOWs~NawbyA+hx)p96Ut}sH+cDHn
zhaZ8p;8NQM?;H;q#(%a_yGUtfsE-bHl*BnF&}`MG^V59C0#uL`d?R17M448drf9Fg
zUu|PJd%`q}E8b9|^7M9b6FW598pHWed}*E^<fYS2eWQXAeZX~05UZjwAI!}p#hk3w
z>Dbrapn%{lxSgq6a>!=B!T$guX|p$RcWO5>Sy5bczvy0w<pkywwjBbFJCfjIi)Fq-
zdz})xyH{V5`?!UAe|n4l$;N`MKcZ^{U&?KlMk8=a$m$@*?gQS~pS_8-ROYU%dC8JJ
z)A$tH2JrkBaAS}@<%}(Hk1J$2+2uoh)oJ6;k2GR^om98jw8+p*EC+jojBLW1&Tt>N
z&QLTI{tKhiS`O^Oav_3myazSY5xB+UKVKcc-|@}YT^l(B@U~BmEEKUJ<Y*8Hxf;I(
zg4ue*C{o&~FFr60eC(uQ<l|!DMaW%>=F3L?Q9g`Sxw!0w?*c(mq<W)OJ5^6mA6-P@
zIPv#Q00EVQBe?Jk9j;D~QP0TZ*5mMfN>v?-_;zviKad%{Cat*{ASFcDRRncOs1?to
zzP}h6T$u2(Fq%>u-x03C(1pqfN@)N$#<Hj$+FevYN(>KLAoc(VT*)CEv8Ye+v45W>
zm5p!G>P0LAHE@GGncx#U8DdG!l0@?PyTSbiu^9O^pfdMAO@H^FAx)seXKMPj2U0VP
zD<<~qRK)c|n-Z!-@!{%D;1MAJJp4^HqoO?+&7kpM6;Uptl0mbNK2ewiZguv=I@~kW
z-;t;ivf(4*NcJJcfm@HVBA`QN?5lsCB3e^Br5bD2YHsO{QV^kvCgC!PgbJ2$u>(&g
zb5;+LBcD1-d)HtQcyQX>UFQPOYltC0!We^=z-FYe2D-c!`ri17Exiqmp)2Id;yk?y
zO{f-`@bSiLRtZ_o0RfIR)=$v(j2ffO{z389cg<J@Y`lFte0h0X+Fl(`=^1~|@Cgp`
zQgH?Pvwwg!&vN<@e%Q~{afPxG2uMLq8L0i8$-2cwgG;?8AdfVtgf@dy7NXT(>a~=u
zrU_RSGnNc0qO<cP4qWh2jjMt*&wYlx)an-W$5K*9^q6LDfwf63m~y**^^&?{!Hba-
zHr5cDkTx}z*o>`Pyrp+3x!5OcqjLaO)q&+M43Hv~{1A*`;~x^QL*F@qacSzCrK&yt
z4UXT?5pPWII(-GzxK``8fiX|Fjj5zwNUh?R=ACIv0&2MS(xESThSQU*MsVjNvo5##
z7=aBn5E0r6$3FGO<v$KTr_#d2;$Ux~zV#7oSBJ*=gJcRpPUdmDi#0m?eGj486@QLy
z&jAbUB63NTkBtyUM7A3y?Ghs#TM8vzqQh&VT*uIx!Vn&}W(MJ8IoDHSZM?y&U$vX>
zqnL}4c7hU#c0|dhz*GjyNWp8pgts`Ba8fJZ6%<DA{B1eWY;I|2K*dxl?#6GwIQ{MC
znm;v{M{#heJSG?|R5^2_ud5t(x~DNeW#bkXGs4d}_g$`6M&_JYcA`trUNaU+Yp-Al
zVUDzO+s}z7{4(QNQdyl}3ni<*KZKI}y7F(7;Io}^R<g}Lzo$6zfcY~WiFN)%;N{x;
zV0xy7%?*g7EX~D)0x3DOyK>n+vQNHP#e}NxCYmDTvXe8fs73M}^G=ZMopCUFO0R&*
zOf)FTFBQY!lvBrOGJ=xGw>)Kt)}B55L?n{^a-p&3*aeAdomS#)EfOIc4utra77Tk$
zXHY05pW$3&YcFs7u{bN`n)Ck6^n31)e}IObyA6yeAr&6Fr|3G$m}-z~_ztvSdX%!=
zBhdE*L_*XZn>el&?T7ppHbX4k9iI{**0~>{v=+llymqIQ0wgL}>S~TZo~p-k?L_-9
z>IgS9f6|~WJE6!Ay}YuHXD;N9l@84{+I|j9LX-dE%Dg@frVSs;+`>1OopDWdMQ;$B
zJkN7-b(G4Ed5US~a5!v%Q%)zqW12%x;%Ru3@MNCznkZDk4tvy~aK*gl?kY$dTfaWJ
zlt||64SEb_Vm8u=t{fu1!vrgfwg#^@S8U}en^4dydhP7<uiVp@wA7A>pi<C=;#yfA
zgkTFB0te12U<GDLly$C_Z6>eSvF(yqcZ@sZAC=<%27L^}XzBCag~y{KlhVVQA$0Gm
z=;e8Lc=rP#IJyqx@n?=OmQKIi6E$90w}-|89oAzNOHqrEv(??yKf<=i*X5QwT6)P)
zSzx8mqK_H`3%WPtlFnp-cwzy{;98uGt|0_a)mr!h{geu(mgBAxwD$BLi0WFZeW=+K
z4+f-zYAy_%_L&nxOq2ZM96{aD7&ky%xEr1Vd`*c<BPu|QP~uTqG3bL0plxPGsV%kW
zHTRc7ciLi{mRzR1U8Ry$xgwf;oOQDk7f8`AyL&w|7Ar=#V9{kc<Fs=;PFY4JMHC%u
zAkL6FT~6c980;kZJVQu_dg|2+8d68oBrG#fG+a3pZYYk>?T6Q((<%81B6UF4Q3w%y
zf7l2wS+dAq?<r1w2(__nih{r^iWS!j3>dlO&eGQzlYim}Fg5jN^+ce?8#>d~Atdqo
zVXiT6H0$V48FI6LhPWk@ZjeG)WMChf80j^F@_6r8sis@qn(;_Swpv=oJ1letdxrGv
z8YwK5g<56xw_I-kC68)M3MP=q&H?^As%{r)dE%OWR)`g9f6aM!I2@E^2c1&wa11Y-
zg51?`wrR10$%j2nwpyloGE<*~WDBJxo9R4Rto5WOH7J^Vpu2T*fkmjKljuSLuKgcE
zK=;d3bUu9SI$kXE1v4Z|lY9`VWCAwkMZv7!G61F(v*RUA(_7e>V`f31PFr@J6&i>n
zhl!>aorjyfON=>10<Tz2CM7mBBSk+mpLRwAh?J75GA(KRT4O&f^uc!>HOSxiWCu`m
z11~wPOElYR4Dv*Y+&#ij>NE4^zlJa?!Jj0~-JmCCyi=h2C%_lhzlK;$5GxPVZxBB{
z1;W3J+>U`O!V|!|5eN}-4jxRk8I=p<aGt-O8Hwm3=rHx5r=CK1mP0<Y{^c{2{m`o1
zKnPK7%(7DlCOqyn(uQUQZqnJF*|ms`QpO|=cMMWKa07@+&WIs9>`3uJEP^_X?@5^5
z3PTl$Ah;DTP!JAYXLeS0(sjYE5JyRC(h+k8>u~oi)gIJ<z$pb|Tv}<APWl0uiL!aJ
z+_m>6keV0idt-PHm=(_OW7{*B)TQK=$m95_OutI*P}{;e$hXt!<(d(1fk+ZgP~IhC
zw+@XZ3Abi1Lj7qV;^`m_;dPh1*eLgd-ZFwSTradxK?o1u$Rsu;A1RqKjg_thxZ9m1
z5l-_P^YZ?Pvt445@=sOE;DJV&yizL-qZnX7JTOpY0b9E%MQEdA!aqjQXn%+k{iots
z@<;uHDA&Xsh+U$_5gl*B9Z(CV;3&9{^BbVb^n?EzUw3H%r-mfSw|X;!nR-p-S=Q-`
z5vg&ecgH#l%Ud15%37AUl<YQYW|;O<yxuf|D_E;=<IvnU^pz-rt<_j@EWy8;kw<vM
zztYS!wdFx2K1*p^a-4rgn1IOK7zr%e#d)d0x<=8KHskaE0YXVwzKZf^rJE!)BXeUi
zsY^p9M$<d~V>r%1Zb$`UB^xo2j(N)rmPEI)OVz+gtMMUaP#o{7DH!Fe)Z-dpLdj(W
zjcs>x1|dAF(k0d!+pZIFHz}D{lVPT9hp%`){r47t+jo|>6qFfBn+=T+m{XRs<ZPGs
zGdWXWQi`>Cmj3Wzgj9|DQb$N>>5`avI_W+I7EVJ&*-(^@7IN#4h<p~gHKLY;0c#wx
zf~{41+O<nC5*?_m6kd?CAi<`dzDEsGUgC;a>JsLh((@=`dj1`D3@Dg;I&IEetU*6o
z8D@r-93^>Q<oLDC21oYBWZEbtoDRnN#?*vOR>2}@B{kw`#?ao)vFFgd>oHTd+NrZO
zTc$2Gn)#7jgV(Y4(Xc=+;}K%h#H0|>Y4hBGzQGPfGm|b2URmVEcmK0J7B%?qGF_$t
zJ>eP<4SI@|1JB>gwT<CqeW=89+P;Kaj17AmSbz$yJPwt<@WZg#fVl!e`Z?sOeybY5
zFAV*2&0zXiRi-egNb#xq5u-U%UF;Y(FW>@eyLOmPl%t(NG?UJ%*Fk=kgk<vR34Z)E
z<9c|QGd@X_LoB9M!%cWST20OGNab;BLXG?Qlc(@1t=RD4`w}ebs%jlDXm_N1c;Px^
zqmjBm*$K+4jwr6=zR0b{GHx-i4Ih{*>DAsL3ZBYg@KP}9isV*1zGB*-F5zNUOX3yF
z1;9ir6tt#@4+2rpZ@gCKD<!q!#5&Xw<<cT@;+2qLEIg(@LgtwEZehi@1a-6<<_7gW
z{o=izS%M4>)EM6)AEMPgQVQ0y0r!-R(-T9}?W)!VmGgYkIa1vt{Ta>t7vf%%uvw9z
zU)a_gYqbe~{2A|;G}=iGXqZ$r8Vg7k7-JH|3N>=(i^w}G5Ch`Cwg?}HX}?RV{UYc-
znJ}!BW*yMW+S@8pxmmuM?Eh|>0DygSTcC99J5LNS7VoWd96gqV!vIRX=4Oy&v@k^d
zD_RB-A}B&O9c@XKL7KKE=kcN|Fh+fTZT^?bbRXk9kM$lL&a}+AqGMd;qv~?5Pa^r|
zpjMa;dlkV1Dm|;QPNEFW%U6Pv!`;Eu%ig%mqM@s|{EzQ2xSo5ZTQ5@vt<TSw%3$V~
zT+fTzs2>~rk~e{;!<;>cjjDFt4f48}o;XfgkvMiS>1?g{0gA<!bZE=^s%YnEu?#(Q
zGowxKRJ7+2=i;7W8=joZnm4l<Ur)+l!<FZz9}=NT#iv&qPUx&`1g7pb!U2h4qDvAP
zf(NwP6;x9r^$pui%<A%HX4!XE;Hwx5Ao@=E7~mD&XnI9YGTS@Sxi_VUnrdYhm8G#w
z>Q{Do#xmR;`Ve5qP(m8dEg8S>b0$8AS?in!%nyxW0LwxQyI^0OZUg!bQ|@*5ak>Z)
zX_UXYuo)eq0?m}bY*J{_`x-h-k{;7gYhsuh;RI%4@<Y&l&S`EHXYwLeYAO^cJ=-y%
zYod`O>}x}##MSl#XmzzukbH0)Jd!ZDlrjH+#WQnAqR5kQ*fH3Vy*G0OG*;=X+W7vQ
zAU2$Zx@Y0w985w@`vY&UU#|za*xO!sX)p-f{wDOK$p3XXe3uS`{{IL)@Eg$+IesJh
z{+s{7`Tsf`fIEFoPJe3pw}=q>+4tv`j&zcS&$#SZLI*b6UghXB&89N10-akfJ#(PX
zwqZpqM+e`TJviWxq<FP5<3J{jpbA75dN}%HtD;ehrvu`-0yq}Kfr#`C?ol4h?qttS
zx+{IvJ+@apIrzML+bVe+?T}oM))t9!@=o$FU>IXbsiq^|WWiyf5|gfjxI7aJ=Tnn8
z(WC4(tpye?d|CehzSvTZB$-Tm4xZ*XcpjkkdPF3A2SD3D?Sd<+byO_FPAu$Rxmi)C
z;5VLObC&xIW`bHePf4LHH8NM6O>qn14}U!1|7vmsOt57+HLtevAi-^@d<YfpfAQJ}
z4G>)6r#y)FZ-#quy!EQcgvkbpKOMOSaXJdbIq=xr>1eZflkU?72_QI*7&%G;jv?*W
zi5QgOYWJi6xQ^bF9lx1SEq}&#BksoynojzOHR#rY<S_doAs;b24<`#14kte**L&AZ
z##3v&f3S?w;t6&=BdT2S($ycblNRwJQh+*AupI2%cpehuk*z0zjnVYzf=<x3wKdQz
z-1!y{l7(TE8Is<b;EWyInDD$1>1mVQB9w8uI|>OzSy+CG%YEU(@`^?}63^3-z6Fbb
zVta{IZgw8rfGmv_3cUuCE&%Qb@|dWO?t9Dk<6mrcn$Nw>V>;<~yc6l%6wcl<NGh)=
zBt>*O?px$2nuvmk%6O7RgTyW@r@NcRNFmraE%59^KwI)yu9hY@E;237oFN^LYr;$V
z_ixU|G82Wa7B%q&vCX`D_gp#ySCLOAKH`I<gS>7mls=_yMy7`wgk=V3$zB@mipL<|
zT6|j}Loy!f9)n(O{h`DHD0elWesl~*@J;W;!R(z6th<{IU0hagt~$9=A;u}elm4a`
zc5h^e30=+M)JA0}@>uO?nOmq@WuGt~3SS#{WeIkElUyDQ!ho6^kccfz_}kw2%+Umc
zd^AmLc{clKR?dN(TU?0YG11WCNWR&Ak<iMRtn=v=-_`w=a+3S?sT*_zBCv!y?)~0A
zVd9=aGIYoInz>x*t)q2rHXWY>siBKMR@X+m(I0qVNnz;AM1IQY+rmC|!vb_-%_idX
z{K`O7S;G6x4)`M?R~;379=;>A@6M}<;#(A9mjK!L`_GR1-T{+;Ra_8K)Np_5d{7VF
zOLW+K?o4_pBh48eL6x0%FMsuUi3t5f09u6iQycNKOF1*_-GQL{@@Lb7_>QiNjx8(6
zJ)tyd^hJx3T*5WWt}~W>%QQIH0P2dri^h{gKbF2IQruv28d;pWBZV6qA^%T)h*=hV
zvWaI%og{DZ>7`G*ZV}eTaOKQ2o&<V~mu;{CsCg%EC)#z|piJA)xw#jHZ?Pc%|HJ|e
z015*B-?s2y4*lEC^DP!chX090J!Ajw|K#GoMhKuD`+PWOW7gUjSX7ZsspH{V?4|Jb
zg4I;UV*chf`}Xz#IbXbV{iUNLxN{|!Vbk&faq$)G+OvakR4z2kJd!c|_3cDm%)N<|
zlq1!UVpR>yVg?pHn_G5|Dp#<fqlx2-`#zy9=tl%s4h|RC4}9}7s!Rx{23IJ6&C_}l
zp}GJsy}OyC<>Cm;kS|tFBVBfL5br1}$upS=FJ>+jk_Dns|6@bsG?WE-;1q^%8LjT4
zAt$MbXTq8KMl0J571WhRv@YWnazMmE8-E-^puY1auV8aH!XQc?IMZZ&SzqJW&6}?X
zYcA?$&`CGIh#w)(1Dal`ooIn~SphFGQ~=@z@5kJ_+q!MMo^OeH&g5f`BGL0sQxy1~
z;P*heY<8HdF!z@E=~s%bcXbX2>WHsM4<{lhPU_x0EkaYS$F}+a{$ld5tq)=7W$Fr4
zv>#unrj9~}&-8(rA;FToo5wVmU6!X%;v?;c`yjq6{`sOmnM0s9Ju}Y-Hasnb+ZZd_
z(qriLz+fjYKO!%=<7W06Z>-No7`d5mXWd}A1i#t}V34()#H*2%+pXs+`Yhu7HTF1n
z&SnWk6C!oT;Mo*0T@M>iV7$MPPvBJm(_~r?A}NXv?~!!F)excGFs4Cfti=M+of`_y
zG4vktuh@j~nfD|1J`C-)<4MpS3}kI`C91J~QoGvGn7jf;u~28>*DU1e`TKU0-e~D_
zV9FVi;@47_bUe~#&{pcpqsdr_pyUS!oo)OX(PtW&ZSQUJUisA{n|izb+W_39^LP*n
zUg$RX^RCHnzy6W~EH|Fp{>fy>TTsNyfgQgLxWQB+dBuHug#ph$B=XL$l)4SvH+5-q
z?<xLaZRwg@8l#^KZd1xbyF{{m(ODm0+#iZhoWkqlf8dDMFqctvgpyW!{zPQAUNT<^
zga4dUN0j(;Mj*Ydqt$c}6%tZO2*)Q$XJeSUX(M90_mm~ha|RH(rxSZ?mA~hoUto1M
zUl?@V@G^U;n}yU4DaxGWAY-Lyx&46$`f~f$*H}m%IGtJ=@M#A!jWSY{kiJ?D42M*&
z6i!+F1u6kD(WzV7&Z<eHY&c7IgVVb|l-tXP>-_}7(Zssh{Iuh|ad!m{`RL6%66PGc
zF|BJyl{5h6=T55K3zqp(j~moJ9D5+!N%V`L>TGH0Xn9?)T3Tu85aeqAupYJT#lDt*
z?C@#4TX1I-+110_firny<K^hm-dtD5-Zs~lOH#h52Z6NTmYtpDJq-Ezh3;4D)%QxL
z_@8m_{~ineXWjdM#=ZXz1;G06Zi!o>|L5BS3Z^K}n7@~uAZjotsh4}b&vW)l#vR4w
zpG4>`)jl#2Xt4h{Tn2=!>W;!ma+4|sxS>bUHzHsxLxpX$Cl|3|hF5cvwXu{{Oqzf*
zv?WrHKOp%r1~>sg>cxsQa1anfVyBYR+a-kYMr&UDXD%a^st?QUPuN#1Ut<E?=Qp6q
z@+kP~yzB?W^0L)FhZH`4fS6wY10*vmn5a7&ehw;r**}3ieDfW62xU9&$YgXMKj-ge
zuAZGfga^WnKgZRxV5nap8Mkt9QE@x<FoDF3tjhM7@Qm4MO1CPoYjBy$_I<o*ZM={`
z9g~@H5tqHQ%Hi$SGse`G)^ckcu0t@K%9|_tMbbaNXchX<vOrp-8T;tk9)M_ud6z;R
z?dbl)(4gX*>P&9R#0-x>++-omP_&r3p8TA^8t6p}F)ke3M{K*sL2qAT&Pd8q3)Tv}
z1aPA;5KL=7WLSFGxi<Cknqke(?ommc8lz6b5NV~dwi6I$u$xrZSO`&-D_^AqUh((z
zwiavv7FZ39c7!gi%TP;dRIww_d`p-G<eD-JVKdJHd)&7Haf0`H;WPH9MXwWKyMODc
zAU0Hb+`=o40+Y(DSbr$r?&m}S+IZUwR-SBE*`w{N0XIRR%CzuDzYhLOr0%Hh9HR!o
zRKFW5JjWL4N_Fgw$VgdrRYpR2^$sL#EMzlR?H%Bpqz=*Pig`y3r-Cau5IoNc!K(({
zea!vEfDH5uQ{<8Ed)iKjl>V$)P)&Fd))@*ZjnDydd8R~y)%-HPaAm@WSg7KF*UJd3
zg`K|XF>?TI^(7@L`eT;2tFcoq&v*eL(w=tnBK6F=F@+uu3S~$MIsr38MRU+p%NgA$
zgYOlAq~oHlkd%*B><-kiyW+!I&bX%9c!)dl`pdP^{D-r9y7h3w(?Avq%HZs<%FsK?
zU(Zss9deQhp5m_#;tTFVkH|j&42|g+D**waqkOhMsl7wBt0kw6=&uG;wI?(=qa9`t
z`f0vV4Ew$=o?FmDx<+CbKS%x4@u-VikSG$ty>tY6N!f0@-KzT15~h6W4M%V^Alro-
z-D4c{N|80+Z|x^jJiCdLAugR>9V~;8I?M9Z0m8e+aeniB$ip<hV82sR3rdi_<Ac~I
z@$ACz&@d2&(o)h})Vf+)^NID0^M|OTBV>siEzr-tND|(S0`Vkpbb^II7o-kC$4b+u
zb)%U?*rt+<pLia^)mB4qQu|58BwgukP)CK;4uT!O+9ctUt3@!nm*D~tA1dv@1zY?L
zP|ITDLZ{}>)L~E3j!Nj$NjTIMF!!pU$*&`9z*rp-s<kbYqODPJs1`Ob)Wfgyc-76e
z<*uNK&RA-J2a+~NhlGUZexTfT#*@gC{4Be48`DEWG+~*s3yiLR2mS$K4;$$<8`ab!
zM3GlT6?Mrrdeid;zky{`a_bQYoF~HqZ5w|vFG>k8a_2oc;P5qIO_&(^yFAZ<-^eCU
zI&climuSM;zuEZ~1^@Yb4_^pb@Vo~YJF-c5A;*@5M*Ko-8P-DM2s$)s9Z~9?VA3`K
zjl1IbBm9fsKo<i{a=2#z$_=(6_iYe&An1jKdP;##4rBlsfyct?ycbypDqlebh*E*j
zy1q~Y87Te(hryo3LSswL6ct7k?Z;k7wQGo1TUeV>7xkXgUk2ogP+sZRoW0A?CnjZ>
zC-EwQ@<Z%XpldEXB<4Rr#LYM${dUINosPOq`4q;ll|hR9c*s2l=j0a&i0VCFc>%)<
zP1|x-|JE~5&48yj9j$?dz2wti0<y(Y=rt(#t`C{%K<J7v1gdIaWoC;s*<FxkQ!O-~
zQq9Y)7|Um~v%^6&dbJ7XZb8agFy>L6cCwIKttO*uAp8juS$4xL6GG<XJ(*alIl}RI
zD^5o$m>V`{t|?FHc({11MMLA(L3F8(n0!xaJS>6d9wx@bztQC(h(&B}!to5=IyLI_
zIWxAOF_4#mppaFo0OB@$OnuFwOFBrkqtvn%;x0L;Yn&_h)xj&4WkiJ~l~&K~4T%Az
zACv+t%^FRdK7s4{OIj?`KQ2N!Bpo7Ze~36zEvqRZ2O*{^rAtH@tPq(lw}l_G7;^qb
zybkV&uQysm$Xghayc@juAT;^wCJ4WtK%>0l7CfxRp%0O{VW(_q91Uu?7g;_ZvXAPS
zxklQF0Sejm=a^5lp+>0VX*gBnDpXj84YP*gIO2O4nIs~d69LJRepgJ(&?9CUgN9pw
za=a7!O#YecmMQT}7({H^=L8cdkAdV{COLO?aH*@3?1G^XnJQI4q?ntN^5_N)rqxQ0
zm8K1P{wg@KrtBpyxj}B1)Ls_IUln}yukiwc6zoC?4)ZgJuuB02+=qCE4aNl;)rUz~
zZM5t-!+r=Wc2)f{n$o`HmMmu`VO*K3GB#HJ$oQR?DCFI#6ENVsgOO$kMOl_r{!Gpa
zLLu_cXBKS#ep_L~Eqqu;R?m<k4i)*2G^L7W^03MTao4)+ll;B=Ua1OQ<j+x2jk4$`
z|J6H+Gi88BEW{ffCbWwb$`(x1M7|I!2%Bj0@dmYPvPn-W9LbofZ{eB8h*_@UHc6!$
zr<sQzsXfIj0EkB(!H*DHQtyX67U0D@{fOVzFJ`m(NK<PezV2s=j3tGzS;A=)gCUf~
zZF7VsgP=hXAxqhbdtq#Cq81?w-%F(mn)ZyVu!OIQ!lIs;zJftF<W$ZastQecp>3J>
zF%H&Q??pU9XQUIy#**E?tSFNuijyoTwAm#sAv|1qO$>cO+YC3Qn4UyiTOzt3t-A$q
zaq^Mrh9{c-(aU%XSYvJ*Do-GX>tuUSPiJlu0_$((>qS)_{a`iY>?v+$1auGj&nm7K
zfoLNkkpYrTtqciLrsrwNIm745YohxGLjCE#OvA}XtWwZGe8N}qb^!Z5*w)D7>(m<3
zYa_no4%A(D>SZyJ{5;X+G(+l0%!3ybd~2wR=5=cpgC%6iLW3++(nVgWtl{v?MnK{s
zA@=?fJz-S&v#(uHI?9S77B_88Urp{;c}411Dcr%gNZkNel%-@w1GR_%uIM2OP&Ikx
zp~`2)q^VxuFoV+63g3EL!~-9sc7XB`ZD^&yL3Qeaf(wvsg>M{$+ET>cfe><a^C`x#
z(m{MYY%_GeeWJDBhN6k?AAo6eDp$c@JdYAFd?gC{B91|6%#VU!iG}aDyU0^IJ!IXG
z^clNYTh4aN>R2M{x!5nb;smI;u%d!aS^4-uwDrsIf<&sWz^DuPK<sA+L6#tnvYTZp
zf3h-l3qI3w$Q`!-plFeG*dr3Ah#3#@nb?)O07|4Ron=$JgrSN6QwlS5zE+rEgl16k
zd1uBTS2mwL<4+0Giy>;tMpb$-3v~qL5{<w=T5C<F0du>WJW^doaE8B>a@#dV-h^X~
zklfy(%JoR$G!N@oh@NDR)6!<cH^vZ~oR*P!Y`U<A9jQ`UVKo9f$k*S(BdDn>bsK5j
z(;mp$1`I8eT93V@VyuZido&#anEoEp{LcOZGFd?dlsSGAd6Jp1eD%;X97FF@)4(6F
zInt=FaB3*6(7YLPJfrJL4J*+hyj(hd7uMu`RJDF5${FV)@Fm>^QPeNTJ_W-eys(0&
z5drUEL+P^{Gt{#2n(R}krM7v=7>y7o-bBIUyoJ$zGSueeKtvHXigSbqtXEknD;!`p
z0#%{YKxztRC}7PL&xb}2Ssg)4-+PSN{J}rt$A$A=#W1uH&y&mZ6zDv?8}fflehRmz
z97=76V`5P5(^fV5Tc9IB0lgLqnj8rxA=Y$3Ca8a<N;eR47c93afCvQ^Gi?-Hu^yum
zrZ)|XT9BXRmi52Km^Sxf3lzoHkPE(>N~&=-Qlw{1xJ|-Vi3}e7{(GW6G-_kQaDq9G
z%eJ7WgO`)emuP`D=NVV!gUWOd6N&TUuP`q>nxu>yk*J4Ui(IROo(Z-`NK7}yO`2?_
zjV|hdaoan9$(Knvr)(q>rzj`Fo!Zh1oMetMP-YaKI`#_kzRct(vm<x)H3pz}sZ@8G
zaRB}TMSRoSI<Tl@^Jl3cL9gb2XNU^>{{sxDsyWD0C`KXtu=Qii`^~{G;(5c(a>}@%
z5|G($mfp<bth8lZpV}@IUU)O*Nm!(vwbKRjb!Ab&=m%LuKZw%C#wg!_zyyVb8}8x#
zbwt^REre(5i^PP7g2C#35S|^S)y8*4JrL^-9;|}ns?$$9>B+0^7|guL2HFXzMU&ue
ztCG@a$q#eW+F~7rJSkuq3s)ftNg2n)A;Rfs;9%Y{D?dKNrVb*79x5Ja-A2wqhPItv
z<0t-I3eI#ro$4d}lO5oH6F<JW19rp<F`KwNJXuDuO0K4VIqLj9;P4lS(vl4sh#q=N
zbF?gLq8&$wky9$?t<h4e;0yVR$SyyI78nE{b7-QT&@e~DOxI73BqE10;`!m@^V`?&
ziBJ6aA<wTgJ!%P3VGX?lm%FSj)RQVIl4%n<qO1mogC?9Vo@^@sVgNb{XT$d=(>9WX
z;<16CF;iBH&_ThDUaX2|v>;fZky6jye%gk|OcI!m3=l4mx(j7t#;U+%>TWTfG^)v2
zmpPQ-_g}wg0wn^$PQmrHOHYla5Xhz7e*iF6?MW@8K`LRn6}nKYa$l=qPvm5Jaw`Zw
z?$n@UlN8z(W6)<fY=e3H2rx!hzXZA|bcTe%mC2@w@%>Gq2M`sMfqH{d1^NINf&>Ee
zGkO+_jR6}u-i3l7@$-nEPyYbv&3jR_b`escIK@V)y!)CXD}a>5_h_qD{Ldj4i^yx{
z2j&o1>xgKQD*2OxmoK!tsrJ}w=9eJFa`d6r7aMW{U=@7Hp8WGB8a2dQfADfVq3w?_
z3!GP|?~c=5E0{?Xv|uGLADLEH#KhxrOC@1A(gBAoGQ}I)xtkEpZALZnqJID)mIdxD
zRTKhd^tTkX5Mj4Q5|ETqB6R*@`}{CtG$z~4FGmwDDIC6|XYAnF=q|Q$5r)A)+$y1j
ze%off9eq8<sd^U$-9?{1X|l8?g-M<*GJWiJ)B6~8x;ykcp#=*S5J7M70_v_21D(~d
ztjW9~E`tVd%U}7l3dXJPicIUes0Qg)NrzvEE0vANh2a^o-vu;&!^5=)W0QYXrdW?m
z<0gSZVn$-i1h%~&W`^VMSk6r8_-Pe>$i#Fi+LhqLZyeo-2&QN+N1W98HtMO?8VbIe
zA9bdEtb@eIjYZ@?A-E@B4fhnk{vLQf;*ge&vU2f<tNI`ie~LY~%D)PVFI9k@89<Dd
z_DnlEdRJpJ)B@%40*%I<Pd}uNlL#EC?y2B6>LUZE1U;l!#Yv^)f=0(~u-LNV8pp>T
z<&k$GT572`Y(Ef<xF>xHC+z<LKw=Hc(=+@osGu_Rjuyi~c`UUFyt_tcMJuIxlg^M8
zbeB+5uD~hbx^y(yo-J09gv(EjYZ%+Dr5ZhJGjxqM%Tx!ejrY`kQzU->igwlIXjA5%
zl>=dZ&#YIv;~nHDh15Vq01;%s2gvgB*xgqJ+e9<NEhh3P3x_fWW=al`6{qz=F=xAe
z{jk=6NSa$E`bwuCk;G0@?HxkmX<6)!Bc+jagjSS-#}~pt?$Ef7>G)nurVi;w_lg!x
zc3eJ|_404fmb=%Xs>-Ss@Y|M}%rv)F7zouDcL*tNFr3Diy=T~aQ*JaEgaGchv^K0e
zc9z|D>4&29(O&F;!DG0-Ghs-s4wT3Gn1Ez(j?H_NDETQNrS`IRH4Uq~VC)k?59mfw
zIfJ2KD{0N&(#Mi1eV37~U~5Qi?QD_gvomN=An0rN&uABF0t~R2czz^&Jk3c19Js@(
zCfqFoz|~8#)wO>BG)f#T0Vty<;G8lTP>M8V@C6=AEQm2Q3{G54k;0ftyv>P?6;RiI
zGdeU0P+qOgaH$oOS5RPj4QZAFf5;N+Q{@gW+l9!GZQDEYqNsv)fYY@j9g9r_vYI3N
z?ocvq3V&A5h&O?{_XX2Li0eGkR;(eJEISx9n!l6E@`SUrGLj@t?<N|NR!gY}AG0(<
zysGyjKr(4hMc$37gGq>Dy^e!oYG@M{vZ^dX(HS?@5*CxYWFja%<Df^uHnF*+5hgjS
z&$EW??omJ|v`2na+ly%Pvc!qTh5bVM&5u2DM`IULi)f_$eZg>;BDJmz8xkjBh6UnY
zM~+i)zRYOr9`ScXDvM)}pAydLA7GpTNT|WOD~sK1h1Mb3|C`DVuXuz7rh;=hLc<=~
z1`@D0{2IWZW3ygn;!@)AAy(mmY#lCLs0kX`k>NC6(zAmWgDiG(ly4+E29YMgM&LAT
zt!QYMJvr|@xf=&xr_XpxfN2PE`s8cMYS4#(zDMZ^v8QA9T2@14tZ2}m`%F&>g?F--
zyt!qFVufE2g(JVPmX-PAH)Ogd&LMNfzRC^VP%B|L?DP?ivVGf2fv|QUkX-E=56KU;
ze^KqS{cc*Fs#rWRkNP`>O-Pj}1$`1A>$nQ=#E~b{4N(lnY3ybpsCMz<;*6C}C#UH+
zqFYR}(Ngd5Hd1p<ho>%2uAN5WC01bWE31H6tgnU8#0;NVQXJzQU7X=H$pEd!Ox85Q
zFpEJhVqyHLMx^$LXtc9OEM4UgJ~xnR`iGX#o4H{CUV3VnT(8NCK;qjdwz(E;)7TM$
zA+E=GK;sd0o_P;i(c*VOUKpwLDL$+%=#l3~{%H-ExQ+PC14)tc75fKJMqaDL(cPyQ
zgRU;1(=0kL#Q9l?hkvZ^yHk!}QO`rd*gT}aJAs&|!@)|tz@Cw%Av!aQ@wo2%yE1mh
zL6Pb(n}Xh30I=*Iu-9H<F$-c#MKGd)oNXKUr(bu$(@D$n#+|Wh25}$u+Zo`qW~6|{
z5d^3k>5#T<t4z7h>ceQa5Im0#cS*2PO+#g5VX~tgHosvYC7#B4fx(`r#L*)UDE?Wx
z(E!6(v&5D<gwK2D6-WXx^nU_j8J^}8!MEUnj++3qg5J?%yn7fLg)+3sm+)~37MatS
zg2WZVQbo$^a~<9jO%R3YkY6Nn4&B0@^!_6%DEV#;>+tdFQNSTG2<d?NiPT1*R5?K}
zP@@}BDFm|VM30~W+d6|%t-|;)jSO}fi*30?7|4c52iUUOdx{rM>sg+nAar@9?@^Gc
zy4&hi0_ER!YcmEb_?NaST(8YTT^9J25y>q%6t~0L27V~B+x|tE2y_;`Lq8E=@k;KP
zK_Z+=B4!<>AuR4XAXed>@KS8@G@eMt5kXeJ89tjOxlzsL7o2Tj^B4H0QMTM-;LudF
z0p0q5lCs@5)40UNg<TbICT?HCyeLHh^Zx*4h8?^A0JB*hKkh9f$V}!A1PwZaby|QO
zGQu~gPQ?DT5hzS4{{Rb@HC;`g;*8Hao)B`!P}F9Q3c&Ys??)*PyA>Cyy7|$X49g+L
zii!m;f?4>M)oG0%cTjq{%_;F`@hs7IM4kTD`^2Zqo835J%2hnxn1yq(a#{~-4tFP`
zncBT!S$6|H4g&3@c(cqAL<j(A0|DYJa6)WatQxlyBg;{pDhtaY=JhBSQ~<7q4aX!?
zpsHROGI+}w04i5Qm3X-Aizw-ZpohQWr74!1iph?Zdwc%?6PWd7A`erw!eUhb>{fY+
z11!<Qo8gt3Q*<jk5-;gkmuLrs{lOwDI!LWt{LH|QRK=-3xN$D^aKI;}2*v?*=3Qt@
z)C(zIDDEVoRU*T@jLcS)WJUx_2zsA<`S&e@Qs6Cywp<E0qJV9+3`+`Cso@m+VjEhE
z0yqW-KoRUq?g?lI3sxS7*$USBiXTu*ZrbVP06f821owz{`W|Br1C~z28t==AlUDjd
z3dh^^8>NaHJhI0&=@neK7fVCkN5VZbs8vPVrUT5XkXBCCseVC?kofvVP};4{Tdy%f
z0lKFz(FMy70JVH-{J~#MuGFT+OtD;^A!Mehunx}9GV_>f$#SYE(kx4eF~t({Xk9M`
z&nqIiX0rRfxwPAc8@+O)twK4oGRy>0-#zmZsW`Vn{nQq7F|@Lu%9Tgt%D>YQwURTA
zQMxI_E;!TnOM8K);Jp6h{DE~BI8C1)m?#1-BL%_SUZz94T~D_!fnoxs5eL!^j6|%b
z0s;+*6G1@QtpO@+)qv&dR*q;j?t#`)>(#u*clJ<T(yyT^m8f(&qaT<R`X--@{$N=S
z%%GoX>rBw{5Lo?=dkA<3+>J^iEfwEK_OPJQ&=RZjiub5TF43Tld~ZEf#C0y(&2I2O
z1{D`e#_6jy_DoKx>U`mKXBPW~m&9k#D)zPFX5(;5%4g#8N;-&T9YCqG{8WP(+x*0=
zENs)Jaa>FDak!UsHM$jjhq-fM7rdyne8g2&CV?pWg`JNi7OUz79YlB!sM643s3@AZ
zUn~kFY~Jr98h949;$5i}v7_o{Skq9hD`~sq^wNC5o74_!H)lD^XLJpG{ASe$c0NzI
z163#s02s`ugVIZ7s%H__c-P!f?_L~y-{PiduU=3O%Pn1dp;ef9y~1)t!lPHtWwo1t
zqY(xM^ZEOT?}UKJd!RuBi)hK~*1f=lk#s9x`-td+AZfwxZ-`8!P^@DPc+XJq`U2GQ
z{1X+!!WjbN<2#rtB_P@6vw&h?QYlI}R|Q&lPvDhI7-b4fkVa*Is2ivng;r={=|wOe
z3ZlWJS02QvX;C$@qL%7fTvjP#Q7JWLIkPRL6U5&V#~Mc@C{++nh!%vrLacY!$Xp_0
zM<HkTF$^;+F9BERVuM}+nh&I3h^oA;!Cs({!F2HtE$9nX&=8mq(R@{Z5H4~yBGW<J
z31ak<6gFgdiU6*fO?Ms2SWO=Kf>3WQcY_g;w_HZDY4#aEBpco+uU^z;ZO7aJrS@HX
zF%hD&3cns}8|q(|%FI?i{{S(owFRcV1O2E9h`2VVo~(i;Ho&X?K*FkWhAHG*JcK?1
zK&ybRE%<~<LA#4m#4Q%3JBOV@usNJ4Qd7_IL4^1r)B@Ix%CIJ`kpmcf5KpZtRe^pc
z8B2<BAEh#`U=~u8UF68|<f!17^p))T(1USw?H53#4ke+Nu(bUfJ_xW8quC0nZ;az?
z&t*Rm)gUk%J~tJEE&OGfgGEAMIvln$#6*fzrTL1=ML^KqJrBh&PH82MK|E$DCKzdN
zd85(R_^3!4u|y{u;n&1VasUmn#kc)Ls%pKC>tDFYa5I1}Im}nZtX9cO=XdK0KncA#
zl~*b?!QXM%BP(}5xHsj^;Qpb4M)bq(P%jdeVy9QF8k^1hlQ2Y?G5lRif15)oN>dIO
z2f2fopa^F-GoeR#=PK=HVi*9c)U+awE8zIV0Jx)Sl<Jjn4NcDzUxwgHnJWmE{V+()
zENbxtqWfKpQot5WSzAV}l3!IrzFNP?nYR%AGs`dNSIs$(dLysrF{j`w_cH9@4)Yw!
znysd)_z$URAT#EWwx1Dl{c0dl>gYc`%4_G4xi9|!gv|xrdKk6Vzo^<vE*4%3?;j9R
zEs_xn>#lEX6Nc#;u7jTQPf>!y3QL#3b&}<y19X6Vz?Q?V2j>N6q@sfRs5a!XVgY7S
z+~s)r<IDm;+N~<^2hZLhl2b~DZDJ*ckZTEYZ7tJ!g85SgA|S%%PyYZbP=4kA01poC
zVAf){<`b^1xlnBVEDfg<I`R)#5cj1h7$9v(qB2$DCYwWGAEz+a*5Y$1X)eb$mtBiU
zqVE1Dr!LHB_+VK@zkC<<EQ~o-6|(f2vQYIATM#Eq;cRY;g?Ue+TgP$zO-sT#;=UFK
zdusrHz9H$|5rC)K#{3FeHA=V7+^cQV;n6cvO?Hb%;)Sy|EkTSrU@BEgfV*sc2Okj<
zShbXJRgOF)w!L-~rJ~%?YU<@)<xciK2LAvT5|DX-H_Zir(o{Mm`4N?vj#Sf1tqnG{
zi(fSs4ZA^Ek5qBKU;>M-KCf?sFt|Bv)f==k##Mlb$Pq?>v5^Mu2;v1no~A-nN{nFK
z)V#u+Mu^Lpe7ql=#$PxFvCv^0fOcDCz}Im=$kc&)Jb=Jn^BP58D3ef>2P!&>W{X(m
z{{UeQaJCKr`-DOp)lIz7HNIQf1L?V&)#VER0C1%c21wf_3G!q0FX3id3)9Xw$YTn%
zw7nzhA$3#fVEoY$U{$9TD1D(aR-gNm<`a?ZxuXRj@`>sYR9uHCiGi$on06Z_u~Svg
z4qa>U#A0#FEsSQn{4&tCaJ|lO^;cAW6Zn}_6iZvJ)gOsm1--!YHjo{wqF5D$4*dRV
zRO(dZl08md%_3_&kPyg#N@-uHj9S)~JFouRm(Weve42kcVV^y!=WhoiD;7@CE?05H
zTK)}n7cgu1B0i${WAT`|H8-MmCO(kmadX-p&N#279ypeKYTn#1(H5!(7=4r3NM3Z2
zr*K~UqI!pNMKsH>E3BmwDWs`=YcWVm=i-2qXJ`)Aq7bB+QgcnW+ZaOilh|W$fYM$b
zV~Px%#zfHeJDzNt>_1UN3mdNH({w1SZ-S6$vs4%K{{UZ^d<E7zxkOUI#k#Gw_xObu
zMJ2Dx8i1mOpvodc#Xm4ft{cvuE3bhr-137&DuvXM#<MGF3SKfT%!{k|IYIPBJ|X*7
zFBr1>LztP)kP3h_1s@R921goM4!dxY!MEtU;Brg+^78aY4QOPA9+l^N{iEiiX;$YD
zOyk{@sdpBbJ;g31N9Xq!QZt4-r$-XB_G{i(hFY2zUMPV<*B4!b8rLUbVBMa=>xzL0
z0S6Jg_bPQ3S)UTL*cOG&OA}X#ih$K_Dx_TiOA0()ToaI2u;LpCAORHvW#2UmAdX3L
zwgJ?lm^KIu7rz&pP^EY>r#G#`2&#+Vi9iN|@NI~Np1fFD>6BVAX0Y~zC8VIjR-Xi2
zAoA<*f5~Rf5^mLT_X{9*`c%i}7zDE}=_s$2j!?iRf<b9z6$rS~Sr?I6V@m%3wB*3a
zGmxfT9s*h6K4X&y_+F0^u6ZieUSU}*UA;kk)KX(Hqa|2JhFCHHG3%rKz&7^HI!Muu
zv1ir)0F96yMJyi9d_iqXbqvVN{2-<}j-u8g55`Fli1GdtWSW3}r9l2Z&rT)xafTBG
z^Qg>$E_9gWxT*%C(&3bPyWUvuYhnZk{FRvsCSs+OhtV9ge8T__qC#;!$}Q)_DyMFt
z!2-{#f_99Oxk<|6kZ>qAG$q>=6$003M~1FatOaW@jnaf{`kFDObC>Q3rxPaSo*V<2
zp}P4O5_~Ul{JX(tpXn0py0z(mXz*e)8+Pq+aCwd(y*-!sh}wn~TbB~B-*|xH0haSK
zh;o-Pf29l~wj*93I8}HL5KtHZH_Q@;x?)^WH!TJBIXp%zcoihisE^ZZKKJHcpaf|4
zhw6ENscmQSID9#rn}RpQEpWQ|fJ%6so+dWcgv@HC0M-SFnke-xfq0y0xdGMO`isEM
zI7@gJzT(>5&;cEycp^68dN<9%O7MWY6<;hj(z}+qloIUbK9YuAt5*JDW4l1U8#v4y
zTWv(|$b+#HB0MpNMKO;q!EHc0i{8O-^<^OF16f8F>Z1X`Rs#ZPnjfAn2oysS$o!x`
zxE6pfKoB);pf0tA)vHqHXhDc~c_NZ3X@LzXO2;Su?mFQ!1y@K7`5BxFhNm>Q>|%3@
zVN7H4-eyD2EsSRiS82e2(e9%`1cjR3!3ZAZc5Xki2Q0lm<WAv5f6G)nH}Mj@;^D5t
z#wJ|D(wkK9be<7|Vvr&7?Y>8-MV1TI)*n^oQvRj7jRxX%1u7JQCh?4lG@~&U4YV9b
zn-BqL;o@D1O{KYBAPvo$*2O;<iZR+OrdQKL^D}5iDAse;68tI+a2&IC?p02O#5E{D
zt3<pUAd!Bhux9Gl0Scwkfq&c=S55$jgySlg(rGydJKrgk%E_%n7^%$wc)rldO5UaN
zMDb9AoR7#uT9&5Nx`oAj6&gkA6POhR^1Q5Bp=>m04ppEzST=zGwbHh#6^7kg$rnz`
zUE)6F$SfwW(n?Sn9x)&SS*cs7>fz!2%#oKL{lGHyM<1Borh%~gfu>a)uH}?;Vd2Cm
zc6P%PVU7o}xa|?f639)esq>gd)=)3`G5AAW=3F?zKEy<rY6PCRmGCc^IM^1A(XZOS
z6G%6~DG75@g+s>WErzct=oS;Ks#b?EYB7>f^|EHyA93uE4T0a>HLDwxr2I0dstINs
zho|Nr#|B89ypRuA6e+_@*y)yVGXr(LZ9*(((=dR2nfeJ*LJBs;V@`3PA^3$cS~+fA
zs8w!&=3*a{L=Csg<}L)a{R)%<HArNA&xO8*8-^(Wd=K{)&{b64GTV=FBU07(=5aKh
z35W7P7r5iWCqRDGaJkUT`4#evL9*OLptEI13;mGVN``1%4r_?Rj0X!KcbW=YIfe|h
zat>qK!o$gNYJaG5R-ryF6!T3CY>}{_cxqhJxUaQGYJk|AB}B})JumuN@foC*>2P55
zJ-|o6Q}zrnV)~8ZSq%}OJbg-8uVGm);ewFDS}wP8!b*#L?hY8pXSUJqEQjtFDOtT!
z55#VepDjfNQ0(ladK_XXqm(=aC4e_eD_ZqP!MCKQn<yo<ihw_=lr9ZxGj3yf=u?oR
z@fR>)<^iBsjU}H`2O25{l)>~s-SA!g%aWocFU(EOu>x#P<vD=n{{Z0=vF~;@4CMK#
zoUJ)MAEdYy%sUDNA0ho<XJSV|N6Msyl@_>)%9x(pJBSGpLv_Sslq9%S%em+RWz0ji
zZ8U)?wWU@X{n;p4wa!Peh3aJy6=i%f?9ikEAgL=umoSVlF)<UFPer!0;pp{5c%muG
z10DG#nT7hLYvdQ^7a%N4daBehNKml4^AkX=>xws42=rCdw?0!)6`ydFI8%Tjn;;cu
zjgVXTLPr4gRq#I&u#ChAFxfA5<lo$(T1CSJnwE<@7pj6-opoMIzyiuBU@#MDxd1o8
zn1@#$O0VW6^{GK}n?i+L7(b{HWsL$L5LDQcD|>eySx~m*h~7O=msAqqG&MO`e-I&1
z5OQ%G)2hJGe<%S8z`w~AeY<IHp!hCkVCD-rnH>qKeM)B?3=ouXTS|msMeSH+A{lNY
z9Sp%#q$c2jZhJ$R6^L-aa|L4UjbH~*%V8;C3{xmRVl8`_s*74#nrvgH7`+_$g25^k
zwxmr3XcoMaJ}!Z=t;F|Y0BCUgfT_qu7O|LGb)iOChsOjeEicrm+GR^(GUmmH2o~Q^
z2qzeEtAy;JSoNa>jS{#LQANgS)_fQ}lo;+WB37crB-$_fq{K^X@3~Ip;RR16gYwIW
z7dHezD>G>|lWQ!$?g7ETU|d3tw>XRheS42|B@d7B{X-xjD;1L-$#RwQrAm;ce25A3
zM=K>Y2)vVs7r9<=vv$To7%ldKB-0shaCK#pltPLYLH_`;IHHkB;wC662b(?+8dChk
zCWeK1G1M>;(K00j94uRY%_rd~KWPSImoch<@QA3)!X^edZW+MJbh5B?naK$814#HQ
zxjyFpU=qx&Fhn*H#6Ve~-?X_R;DJ#3vm9hy;Y9pGiom@vocRh5a2lh-3b>BNO?2mX
zHv>sGs=ZNvF?*OQQp8G#P7z@YVPTa$wH`En<!Np-H-Z}*1!u`qgCi!pxCFCtadCHb
zDHOoV4XQkHe$SbhGUvdr!ySvCNKaM+m{_yZ&Lcvq=VA!H&K_mU8)s6$$hKE;<Vh)n
zs)Ws3FcR{{Ofmt7F^>~Oc_#@j<_CrWuc)Rf1q;>GQ8|m31RUHDT6#?1=nl6id7Zkh
zP#Kcw+F!auRU4^5v#fz(?<Bb_kxm1HGOi*p6<raaGF0F@Vhx)y4q@M$bE?pAX5rSA
zw3gf08H|@ZLoK<E%FBWB^DjbJp%R)CqFNeChU((`oUP~!fk5*V08k$hqq()*1dnzR
zbmIKP-rNkRQiZF*F8+|#h6oG*OVHp{DsrhxdE8Q9_4}5UZiK4@!nn{nUWtih;x=B*
z38DuA=3o+Sv3F~ggn1Gh0tO-rC2z$$SOsb_4R|6D=q%(#1k9$iiI(iV=`3^O3PAY0
zEPNr0ghCfF;b15m@?s>bwx7yS+PY@tOjI4=NlK(@r3Zy1p%p8zf;G!Zw+w_18Zi!}
z0hK9sti=a}K&h)OB3F)(2TqA?oI7ZAM|E&hx7rmznQK;Ic|r!mhjNXoowBbj#@Ap4
zTDg^(Z~I9DQ!uK2k~jV@<5Dt&R-~<0<Bz%8#IbU(62r1Cqh76i{{Yjc%(;7xX=kVY
za{(ZiS`g}2jFv**n1%wI&kV)E>Jwlafdvc!;ySVNB?~mh^AEtXf4D_BJ_FIIW^m)F
zZO8urKn45+LPIo5`Vn&B2%-AT577;g2P*9&knAPBptS-+h7-F7P4NBEk>q}3K&exC
z{KKQw1))3KKwG^P_cGn^587*;b7IfTTm&UjaOW{F8rqNMUs709qth_}s9SP)%vEKA
zOGp5^Gi)&ey?>+-t#B3d49E;gWtOn~^Z4yw6VIXr$Q259D_(gDm0VcD^Kn4vvD+O+
z;Ean>@tU1@i7vp|s}+IEMY!q=_-fwe@|F3Piiq>^5DWN(SC|H{RW9FSLzzNLX`nt_
z63*KA4+n_3{i3Ci9=V3l@<p>J#Z+J%!CSEx%k+otJ|%3GE^;t9A#4@U1A<vB^;jgO
zF&bnqgEq(3XUgRZdg2;vd<}>l*4l#<URF?8R%mDi#^VMnvR`7Nt?57M8<7t`?FVkh
zHU5&URQ~|uaT+JW{{YxfD}BG%HDvD4EvbG{)<blTU}01`1|pz>pxG40XvK+&DN{~N
zYF`@xS~ML`5V|spS&=&$pq8tAM$mxgtu?r?j0i{?6idKpA6F>iHG*bUOM*EL8#Md<
zAh*FNmGuTP9LAy{=t~4{-X?~gqD4!%04VT78h8@(FD*sreRBbb9tis8*rSX>bODw*
z<tS_~xKgzQwUDbU2MJJFPN}#|!L(JhR8_HsjWFv@#|?u}sa+8Jcf)Toz!TI%oAW#|
zoK-!+)N4?kgqDds5Z$Fr7!z^lGbCXJ*^cECN;fYIJT3fA@;y}sszGg-o4H0UIhQz$
zjIi-iIaD!F(S`s!^YbiFJ_mU|A>WvvHcM&?2pa)G!2-7^yJB*RH+Yvg)XG?+Vg5-B
zcb4UX-AdT08*kKK49Xr*+Cg%)EG3OgxR`K~@hA&X8wK+Rn_v-Nn4$q*uA7QQgER_^
zCzL>6Sc6MI=5P9znBark;t1nOW9^7i+<@wga;;L_<_faI&kJ<UIdKKrej@JY#I0R1
z5Xp4S9mY*K;0|+s<_Gg-8m3w{3rqD0Lp8{A0|LX&pD-mQq+Yu1V5m+NM;0q=e1<0Y
z1&(s3`-p5<au}jlT^|1cM}h#^E#je9Y(kpP5|=EZ#;3AW3TRWMKU0(wV%rr3mtN!L
zDhoLQ*v)>TSsWUsQ3kI1oxuP!o=btze+5Gh@H-#axUi3~{-A*}TF&7!0ZO@rRwKQ_
z30b``1$$I#W#r*vcI;pN!}_5CiDeeFkeojB1CPy%<MyB$+vS8;=(G{f+Kaz_WuFp%
Xal_GM4Sm89Yb<B(G9zUL{LlZ{JNy=~

literal 0
HcmV?d00001


From dbc3df3bfe31ff03f083402c72ea3eac63177594 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 00:49:38 +0900
Subject: [PATCH 004/638] Update _config.yml

---
 _config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_config.yml b/_config.yml
index a72bdce81..c19b807ea 100644
--- a/_config.yml
+++ b/_config.yml
@@ -12,7 +12,7 @@ author: Ivan Ivanov
 description: Self-study and blog every day
 
 # URL of your avatar or profile pic (you could use your GitHub profile pic)
-avatar:
+avatar: /avatar.jpeg
 
 #
 # Flags below are optional

From b878c9ef2e8f0d9b032c6c6f7a5755e1fb97f09d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 00:55:53 +0900
Subject: [PATCH 005/638] Update _config.yml

---
 _config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_config.yml b/_config.yml
index c19b807ea..a4cdbf12b 100644
--- a/_config.yml
+++ b/_config.yml
@@ -31,7 +31,7 @@ footer-links:
   rss: 
   twitter: hi_iam_ivan
   stackoverflow: 
-  youtube: user/ivaninkorea
+  youtube: channel/UC-nprSI5aP4O7REgzaSUXFg
   googleplus: # anything in your profile username that comes after plus.google.com/
   playconsole:
   mastodon: # <your-mastodon-instance>/@<your-user-name> eg. fosstodon.org/@username

From cc705f612c083cecea83ae48cd4723544d2bef69 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 00:56:09 +0900
Subject: [PATCH 006/638] Delete _pages/getting-started.md

---
 _pages/getting-started.md | 51 ---------------------------------------
 1 file changed, 51 deletions(-)
 delete mode 100644 _pages/getting-started.md

diff --git a/_pages/getting-started.md b/_pages/getting-started.md
deleted file mode 100644
index f7946a125..000000000
--- a/_pages/getting-started.md
+++ /dev/null
@@ -1,51 +0,0 @@
----
-layout: page
-title: Getting started with Reverie
-permalink: /getting-started/
----
-
-### Step 1) Fork Reverie to your User Repository
-
-Fork [this repository](https://github.com/amitmerchant1990/reverie), then rename the repository to `yourgithubusername.github.io`.
-
-Alternatively, you can use [Use this template](https://github.com/amitmerchant1990/reverie/generate) button if you want to create a repository with a clean commit history which will use Reverie as a template.
-
-Your Jekyll blog will often be viewable immediately at <https://yourgithubusername.github.io> (if it's not, you can often force it to build by completing step 2)
-
-### Step 2) Customize and view your site
-
-Enter your site name, description, avatar and many other options by editing the `_config.yml` file. You can easily turn on Google Analytics tracking, Disqus commenting and social icons here.
-
-Making a change to `_config.yml` (or any file in your repository) will force GitHub Pages to rebuild your site with jekyll. Your rebuilt site will be viewable a few seconds later at <https://yourgithubusername.github.io> - if not, give it ten minutes as GitHub suggests and it'll appear soon.
-
-### Step 3) Publish your first blog post
-
-Create a new file called `/_posts/2019-2-13-Hello-World.md` to publish your first blog post. That's all you need to do to publish your first blog post! This [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) might come in handy while writing the posts.
-
-> You can add additional posts in the browser on GitHub.com too! Just hit the <kbd>Create new file</kbd> button in `/_posts/` to create new content. Just make sure to include the [front-matter](http://jekyllrb.com/docs/frontmatter/) block at the top of each new blog post and make sure the post's filename is in this format: year-month-day-title.md
-
-## Using Categories in Reverie
-
-You can categorize your content based on `categories` in Reverie. For this, you just need to add `categories` in front matter like below:
-
-For adding single category:
-
-```md
-categories: JavaScript
-```
-
-For adding multiple categories:
-
-```md
-categories: [PHP, Laravel]
-```
-
-The contegorized content can be shown over this URL: <https://yourgithubusername.github.io/categories/>
-
-## RSS
-
-The generated [RSS feed](https://en.wikipedia.org/wiki/RSS) of your blog can be found at <https://yourgithubusername.github.io/feed>. You can see the example RSS feed over [here](https://www.amitmerchant.com/reverie/feed).
-
-## Sitemap
-
-The generated sitemap of your blog can be found at <https://yourgithubusername.github.io/sitemap>. You can see the example sitemap feed over [here](https://www.amitmerchant.com/reverie/sitemap).

From ba76608cd3496452ef9992e2c930c171f3fe813d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Apr 2024 01:02:31 +0900
Subject: [PATCH 007/638] Update default.html

---
 _layouts/default.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index caa586258..708a06d23 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -29,7 +29,6 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
-              <a href="{{ site.baseurl }}/getting-started">Getting Started</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
               <a href="{{ site.baseurl }}/archive">Archive</a>

From eb4588c57def029120deec548e374f2e1e1bf549 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 20:44:32 +0900
Subject: [PATCH 008/638] Create test_project.md

---
 _projects/test_project.md | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 _projects/test_project.md

diff --git a/_projects/test_project.md b/_projects/test_project.md
new file mode 100644
index 000000000..acdba69e5
--- /dev/null
+++ b/_projects/test_project.md
@@ -0,0 +1,2 @@
+# Hello.
+This is my 1st project

From badbf413582cceb115a147876da3fc24359dfbbf Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 20:44:50 +0900
Subject: [PATCH 009/638] Create projects.md

---
 _pages/projects.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 _pages/projects.md

diff --git a/_pages/projects.md b/_pages/projects.md
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/_pages/projects.md
@@ -0,0 +1 @@
+

From 54ea09cfda43c55d30dfd9c5d8468108e29f39b6 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 20:47:50 +0900
Subject: [PATCH 010/638] Update projects.md

---
 _pages/projects.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/_pages/projects.md b/_pages/projects.md
index 8b1378917..91bdb33e4 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -1 +1,29 @@
+---
+layout: page
+title: Projects
+permalink: /projects/
+---
 
+[Reverie](https://github.com/amitmerchant1990/reverie) is a Jekyll theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for [my blog](https://www.amitmerchant.com).
+
+This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
+
+## Features
+
+- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
+- Fully responsive and mobile optimized base theme
+- Sass/Coffeescript support using Jekyll 2.0
+- Free hosting on your GitHub Pages user site
+- All the SEO goodies comes in-built
+- Markdown blogging
+- Syntax highlighting using Pygments
+    - [Dracula syntax theme](https://draculatheme.com/) included
+- Disqus commenting
+- Google Analytics integration
+- Fuzzy search across blog posts
+- Pagination of posts works out-of-the-box.
+- Categorize posts out-of-the box
+- RSS Feed
+- In-built sitemap
+
+Learn more about it [here](https://github.com/amitmerchant1990/reverie) on how to get started.

From c4762ed2dc93ced623fd8c780cf9614ee47389eb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 20:48:27 +0900
Subject: [PATCH 011/638] Update default.html

---
 _layouts/default.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index 708a06d23..9ae27593a 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -29,6 +29,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
+              <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
               <a href="{{ site.baseurl }}/archive">Archive</a>

From 96e8352c345498f047a980546d0862a3c737bb90 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 20:51:47 +0900
Subject: [PATCH 012/638] Update test_project.md

---
 _projects/test_project.md | 88 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 86 insertions(+), 2 deletions(-)

diff --git a/_projects/test_project.md b/_projects/test_project.md
index acdba69e5..34e03c412 100644
--- a/_projects/test_project.md
+++ b/_projects/test_project.md
@@ -1,2 +1,86 @@
-# Hello.
-This is my 1st project
+---
+layout: post
+title: Test project post
+categories: [Miscellaneous, Jekyll]
+---
+
+[Reverie](https://github.com/amitmerchant1990/reverie) is a [Jekyll](https://jekyllrb.com/)-powered theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for my blog.
+
+This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
+
+![](/images/reverie-demo.png)
+
+## Features overview
+
+- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
+- Fully responsive and mobile optimized base theme
+- Sass/Coffeescript support using Jekyll 2.0
+- Free hosting on your GitHub Pages user site
+- All the SEO goodies comes in-built
+- Markdown blogging
+- Syntax highlighting using Pygments
+    - [Dracula syntax theme](https://draculatheme.com/) included
+- Disqus commenting
+- Google Analytics integration
+- Fuzzy search across blog posts
+- Pagination of posts works out-of-the-box.
+- Categorize posts out-of-the box
+- RSS Feed
+- In-built sitemap
+
+<div style="text-align: center;">
+ <script async type="text/javascript" src="//cdn.carbonads.com/carbon.js?serve=CE7D6KJY&placement=wwwamitmerchantcom" id="_carbonads_js"></script>
+</div>
+
+## Using Reverie on GitHub Pages
+
+### Step 1) Fork Reverie to your User Repository
+
+Fork [this repository](https://github.com/amitmerchant1990/reverie), then rename the repository to `yourgithubusername.github.io`.
+
+Alternatively, you can use [Use this template](https://github.com/amitmerchant1990/reverie/generate) button if you want to create a repository with a clean commit history which will use Reverie as a template.
+
+Your Jekyll blog will often be viewable immediately at <https://yourgithubusername.github.io> (if it's not, you can often force it to build by completing step 2)
+
+### Step 2) Customize and view your site
+
+Enter your site name, description, avatar and many other options by editing the `_config.yml` file. You can easily turn on Google Analytics tracking, Disqus commenting and social icons here.
+
+Making a change to `_config.yml` (or any file in your repository) will force GitHub Pages to rebuild your site with jekyll. Your rebuilt site will be viewable a few seconds later at <https://yourgithubusername.github.io> - if not, give it ten minutes as GitHub suggests and it'll appear soon.
+
+### Step 3) Publish your first blog post
+
+Create a new file called `/_posts/2019-2-13-Hello-World.md` to publish your first blog post. That's all you need to do to publish your first blog post! This [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) might come in handy while writing the posts.
+
+> You can add additional posts in the browser on GitHub.com too! Just hit the <kbd>Create new file</kbd> button in `/_posts/` to create new content. Just make sure to include the [front-matter](http://jekyllrb.com/docs/frontmatter/) block at the top of each new blog post and make sure the post's filename is in this format: year-month-day-title.md
+
+## Using Categories in Reverie
+
+You can categorize your content based on `categories` in Reverie. For this, you just need to add `categories` in front matter like below:
+
+For adding single category:
+
+```md
+categories: JavaScript
+```
+
+For adding multiple categories:
+
+```md
+categories: [PHP, Laravel]
+```
+
+The contegorized content can be shown over this URL: <https://yourgithubusername.github.io/categories/>
+
+## RSS
+
+The generated [RSS feed](https://en.wikipedia.org/wiki/RSS) of your blog can be found at <https://yourgithubusername.github.io/feed>. You can see the example RSS feed over [here](https://www.amitmerchant.com/reverie/feed).
+
+## Sitemap
+
+The generated sitemap of your blog can be found at <https://yourgithubusername.github.io/sitemap>. You can see the example sitemap feed over [here](https://www.amitmerchant.com/reverie/sitemap).
+
+## License
+
+MIT
+

From 9d2bf7284d2bb03d4ceeff670110d2526ed838a4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:01:40 +0900
Subject: [PATCH 013/638] Create projects.html

---
 projects.html | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 projects.html

diff --git a/projects.html b/projects.html
new file mode 100644
index 000000000..55751ad48
--- /dev/null
+++ b/projects.html
@@ -0,0 +1,49 @@
+---
+layout: default
+---
+
+<div class="projects">
+  {% for project in paginator.projects %}
+    <article class="project">
+      <a href="{{ site.baseurl }}{{ project.url }}">
+        <h1>{{ post.title }}</h1>
+
+        <div>
+          <p class="post_date">{{ post.date | date: "%B %e, %Y" }}</p>
+        </div>
+      </a>
+      <div class="entry">
+        {{ post.excerpt }}
+      </div>
+
+      <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>
+    </article>
+  {% endfor %}
+
+  <!-- pagination -->
+  {% if paginator.total_pages > 1 %}
+  <div class="pagination">
+    {% if paginator.previous_page %}
+      <a href="{{ paginator.previous_page_path | prepend: site.baseurl | replace: '//', '/' }}">&laquo; Prev</a>
+    {% else %}
+      <span>&laquo; Prev</span>
+    {% endif %}
+
+    {% for page in (1..paginator.total_pages) %}
+      {% if page == paginator.page %}
+        <span class="webjeda">{{ page }}</span>
+      {% elsif page == 1 %}
+        <a href="{{ '/' | prepend: site.baseurl | replace: '//', '/' }}">{{ page }}</a>
+      {% else %}
+        <a href="{{ site.paginate_path | prepend: site.baseurl | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
+      {% endif %}
+    {% endfor %}
+
+    {% if paginator.next_page %}
+      <a href="{{ paginator.next_page_path | prepend: site.baseurl | replace: '//', '/' }}">Next &raquo;</a>
+    {% else %}
+      <span>Next &raquo;</span>
+    {% endif %}
+  </div>
+  {% endif %}
+</div>

From dc2c0950da6ae4d8d69f8e56fd833ffbb80fb0f4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:03:11 +0900
Subject: [PATCH 014/638] Delete _pages/projects.md

---
 _pages/projects.md | 29 -----------------------------
 1 file changed, 29 deletions(-)
 delete mode 100644 _pages/projects.md

diff --git a/_pages/projects.md b/_pages/projects.md
deleted file mode 100644
index 91bdb33e4..000000000
--- a/_pages/projects.md
+++ /dev/null
@@ -1,29 +0,0 @@
----
-layout: page
-title: Projects
-permalink: /projects/
----
-
-[Reverie](https://github.com/amitmerchant1990/reverie) is a Jekyll theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for [my blog](https://www.amitmerchant.com).
-
-This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
-
-## Features
-
-- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
-- Fully responsive and mobile optimized base theme
-- Sass/Coffeescript support using Jekyll 2.0
-- Free hosting on your GitHub Pages user site
-- All the SEO goodies comes in-built
-- Markdown blogging
-- Syntax highlighting using Pygments
-    - [Dracula syntax theme](https://draculatheme.com/) included
-- Disqus commenting
-- Google Analytics integration
-- Fuzzy search across blog posts
-- Pagination of posts works out-of-the-box.
-- Categorize posts out-of-the box
-- RSS Feed
-- In-built sitemap
-
-Learn more about it [here](https://github.com/amitmerchant1990/reverie) on how to get started.

From c73be3ac8178066f2543bb5d77b54ca43248f602 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:05:54 +0900
Subject: [PATCH 015/638] Create project.html

---
 _layouts/project.html | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 _layouts/project.html

diff --git a/_layouts/project.html b/_layouts/project.html
new file mode 100644
index 000000000..ce81b0197
--- /dev/null
+++ b/_layouts/project.html
@@ -0,0 +1,39 @@
+---
+layout: default
+---
+
+<article class="project detailed">
+  <h1>{{ page.title }}</h1>
+
+  <div>
+    <p class="author_title">{{site.author}}  ·  {{ page.date | date: "%B %e, %Y" }}</p>
+    {% if page.last_modified_at %}
+    <p class="author_title" datetime="{{ page.last_modified_at | date_to_xmlschema }}">(Updated: {{ page.last_modified_at | date: "%b %-d, %Y" }})</p>
+    {% endif %}
+    <div class="post-tags">
+      {% if post %}
+        {% assign categories = post.categories %}
+      {% else %}
+        {% assign categories = page.categories %}
+      {% endif %}
+      {% for category in categories %}
+        <a href="{{site.baseurl}}/categories/#{{category|slugize}}">{{category}}</a>
+        {% unless forloop.last %}&nbsp;{% endunless %}
+      {% endfor %}
+    </div>
+  </div>
+    
+  <div class="entry">
+    {{ content }}
+  </div>
+
+  <div>
+    <p><span class="share-box">Share:</span> <a href="http://twitter.com/share?text={{ page.title }}&url={{site.url}}{{page.url}}" target="_blank">Twitter</a>, <a href="https://www.facebook.com/sharer.php?u={{site.url}}{{page.url}}" target="_blank">Facebook</a></p>
+  </div>
+
+  <!--<div class="date">
+    Written on {{ page.date | date: "%B %e, %Y" }}
+  </div>-->
+
+  {% include disqus.html %}
+</article>

From c8ac3ba157a59b2b630c9d2ec2a718bd2a37e90b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:06:41 +0900
Subject: [PATCH 016/638] Update test_project.md

---
 _projects/test_project.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_projects/test_project.md b/_projects/test_project.md
index 34e03c412..1df821f20 100644
--- a/_projects/test_project.md
+++ b/_projects/test_project.md
@@ -1,5 +1,5 @@
 ---
-layout: post
+layout: project
 title: Test project post
 categories: [Miscellaneous, Jekyll]
 ---

From 2216c98f746073c44e1e6f473180ef3f4d0e15cd Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:09:24 +0900
Subject: [PATCH 017/638] Update projects.html

---
 projects.html | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/projects.html b/projects.html
index 55751ad48..3a75a1545 100644
--- a/projects.html
+++ b/projects.html
@@ -6,14 +6,14 @@
   {% for project in paginator.projects %}
     <article class="project">
       <a href="{{ site.baseurl }}{{ project.url }}">
-        <h1>{{ post.title }}</h1>
+        <h1>{{ project.title }}</h1>
 
         <div>
-          <p class="post_date">{{ post.date | date: "%B %e, %Y" }}</p>
+          <p class="project_date">{{ project.date | date: "%B %e, %Y" }}</p>
         </div>
       </a>
       <div class="entry">
-        {{ post.excerpt }}
+        {{ project.excerpt }}
       </div>
 
       <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>

From d02ce507665dcb8c2d24132029d3e1dc1e84fe35 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:09:47 +0900
Subject: [PATCH 018/638] Delete _projects/test_project.md

---
 _projects/test_project.md | 86 ---------------------------------------
 1 file changed, 86 deletions(-)
 delete mode 100644 _projects/test_project.md

diff --git a/_projects/test_project.md b/_projects/test_project.md
deleted file mode 100644
index 1df821f20..000000000
--- a/_projects/test_project.md
+++ /dev/null
@@ -1,86 +0,0 @@
----
-layout: project
-title: Test project post
-categories: [Miscellaneous, Jekyll]
----
-
-[Reverie](https://github.com/amitmerchant1990/reverie) is a [Jekyll](https://jekyllrb.com/)-powered theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for my blog.
-
-This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
-
-![](/images/reverie-demo.png)
-
-## Features overview
-
-- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
-- Fully responsive and mobile optimized base theme
-- Sass/Coffeescript support using Jekyll 2.0
-- Free hosting on your GitHub Pages user site
-- All the SEO goodies comes in-built
-- Markdown blogging
-- Syntax highlighting using Pygments
-    - [Dracula syntax theme](https://draculatheme.com/) included
-- Disqus commenting
-- Google Analytics integration
-- Fuzzy search across blog posts
-- Pagination of posts works out-of-the-box.
-- Categorize posts out-of-the box
-- RSS Feed
-- In-built sitemap
-
-<div style="text-align: center;">
- <script async type="text/javascript" src="//cdn.carbonads.com/carbon.js?serve=CE7D6KJY&placement=wwwamitmerchantcom" id="_carbonads_js"></script>
-</div>
-
-## Using Reverie on GitHub Pages
-
-### Step 1) Fork Reverie to your User Repository
-
-Fork [this repository](https://github.com/amitmerchant1990/reverie), then rename the repository to `yourgithubusername.github.io`.
-
-Alternatively, you can use [Use this template](https://github.com/amitmerchant1990/reverie/generate) button if you want to create a repository with a clean commit history which will use Reverie as a template.
-
-Your Jekyll blog will often be viewable immediately at <https://yourgithubusername.github.io> (if it's not, you can often force it to build by completing step 2)
-
-### Step 2) Customize and view your site
-
-Enter your site name, description, avatar and many other options by editing the `_config.yml` file. You can easily turn on Google Analytics tracking, Disqus commenting and social icons here.
-
-Making a change to `_config.yml` (or any file in your repository) will force GitHub Pages to rebuild your site with jekyll. Your rebuilt site will be viewable a few seconds later at <https://yourgithubusername.github.io> - if not, give it ten minutes as GitHub suggests and it'll appear soon.
-
-### Step 3) Publish your first blog post
-
-Create a new file called `/_posts/2019-2-13-Hello-World.md` to publish your first blog post. That's all you need to do to publish your first blog post! This [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) might come in handy while writing the posts.
-
-> You can add additional posts in the browser on GitHub.com too! Just hit the <kbd>Create new file</kbd> button in `/_posts/` to create new content. Just make sure to include the [front-matter](http://jekyllrb.com/docs/frontmatter/) block at the top of each new blog post and make sure the post's filename is in this format: year-month-day-title.md
-
-## Using Categories in Reverie
-
-You can categorize your content based on `categories` in Reverie. For this, you just need to add `categories` in front matter like below:
-
-For adding single category:
-
-```md
-categories: JavaScript
-```
-
-For adding multiple categories:
-
-```md
-categories: [PHP, Laravel]
-```
-
-The contegorized content can be shown over this URL: <https://yourgithubusername.github.io/categories/>
-
-## RSS
-
-The generated [RSS feed](https://en.wikipedia.org/wiki/RSS) of your blog can be found at <https://yourgithubusername.github.io/feed>. You can see the example RSS feed over [here](https://www.amitmerchant.com/reverie/feed).
-
-## Sitemap
-
-The generated sitemap of your blog can be found at <https://yourgithubusername.github.io/sitemap>. You can see the example sitemap feed over [here](https://www.amitmerchant.com/reverie/sitemap).
-
-## License
-
-MIT
-

From fd55bf230f3806af019565de71f2a9700dfd14d5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:10:39 +0900
Subject: [PATCH 019/638] Create 2024-02-02-test.md

---
 _projects/2024-02-02-test.md | 86 ++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 _projects/2024-02-02-test.md

diff --git a/_projects/2024-02-02-test.md b/_projects/2024-02-02-test.md
new file mode 100644
index 000000000..d78ab7d32
--- /dev/null
+++ b/_projects/2024-02-02-test.md
@@ -0,0 +1,86 @@
+---
+layout: project
+title: Test Project 
+categories: [Miscellaneous, Jekyll]
+---
+
+[Reverie](https://github.com/amitmerchant1990/reverie) is a [Jekyll](https://jekyllrb.com/)-powered theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for my blog.
+
+This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
+
+![](/images/reverie-demo.png)
+
+## Features overview
+
+- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
+- Fully responsive and mobile optimized base theme
+- Sass/Coffeescript support using Jekyll 2.0
+- Free hosting on your GitHub Pages user site
+- All the SEO goodies comes in-built
+- Markdown blogging
+- Syntax highlighting using Pygments
+    - [Dracula syntax theme](https://draculatheme.com/) included
+- Disqus commenting
+- Google Analytics integration
+- Fuzzy search across blog posts
+- Pagination of posts works out-of-the-box.
+- Categorize posts out-of-the box
+- RSS Feed
+- In-built sitemap
+
+<div style="text-align: center;">
+ <script async type="text/javascript" src="//cdn.carbonads.com/carbon.js?serve=CE7D6KJY&placement=wwwamitmerchantcom" id="_carbonads_js"></script>
+</div>
+
+## Using Reverie on GitHub Pages
+
+### Step 1) Fork Reverie to your User Repository
+
+Fork [this repository](https://github.com/amitmerchant1990/reverie), then rename the repository to `yourgithubusername.github.io`.
+
+Alternatively, you can use [Use this template](https://github.com/amitmerchant1990/reverie/generate) button if you want to create a repository with a clean commit history which will use Reverie as a template.
+
+Your Jekyll blog will often be viewable immediately at <https://yourgithubusername.github.io> (if it's not, you can often force it to build by completing step 2)
+
+### Step 2) Customize and view your site
+
+Enter your site name, description, avatar and many other options by editing the `_config.yml` file. You can easily turn on Google Analytics tracking, Disqus commenting and social icons here.
+
+Making a change to `_config.yml` (or any file in your repository) will force GitHub Pages to rebuild your site with jekyll. Your rebuilt site will be viewable a few seconds later at <https://yourgithubusername.github.io> - if not, give it ten minutes as GitHub suggests and it'll appear soon.
+
+### Step 3) Publish your first blog post
+
+Create a new file called `/_posts/2019-2-13-Hello-World.md` to publish your first blog post. That's all you need to do to publish your first blog post! This [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) might come in handy while writing the posts.
+
+> You can add additional posts in the browser on GitHub.com too! Just hit the <kbd>Create new file</kbd> button in `/_posts/` to create new content. Just make sure to include the [front-matter](http://jekyllrb.com/docs/frontmatter/) block at the top of each new blog post and make sure the post's filename is in this format: year-month-day-title.md
+
+## Using Categories in Reverie
+
+You can categorize your content based on `categories` in Reverie. For this, you just need to add `categories` in front matter like below:
+
+For adding single category:
+
+```md
+categories: JavaScript
+```
+
+For adding multiple categories:
+
+```md
+categories: [PHP, Laravel]
+```
+
+The contegorized content can be shown over this URL: <https://yourgithubusername.github.io/categories/>
+
+## RSS
+
+The generated [RSS feed](https://en.wikipedia.org/wiki/RSS) of your blog can be found at <https://yourgithubusername.github.io/feed>. You can see the example RSS feed over [here](https://www.amitmerchant.com/reverie/feed).
+
+## Sitemap
+
+The generated sitemap of your blog can be found at <https://yourgithubusername.github.io/sitemap>. You can see the example sitemap feed over [here](https://www.amitmerchant.com/reverie/sitemap).
+
+## License
+
+MIT
+

From 0bb45502ac222fe9131e555cff9d4128bd529dea Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:14:35 +0900
Subject: [PATCH 020/638] Update project.html

---
 _layouts/project.html | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/_layouts/project.html b/_layouts/project.html
index ce81b0197..9bdbc8a40 100644
--- a/_layouts/project.html
+++ b/_layouts/project.html
@@ -10,9 +10,9 @@ <h1>{{ page.title }}</h1>
     {% if page.last_modified_at %}
     <p class="author_title" datetime="{{ page.last_modified_at | date_to_xmlschema }}">(Updated: {{ page.last_modified_at | date: "%b %-d, %Y" }})</p>
     {% endif %}
-    <div class="post-tags">
-      {% if post %}
-        {% assign categories = post.categories %}
+    <div class="project-tags">
+      {% if project %}
+        {% assign categories = project.categories %}
       {% else %}
         {% assign categories = page.categories %}
       {% endif %}

From 61ff756c05e8ec38dbf9300ca9eff6b84b76496d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:16:07 +0900
Subject: [PATCH 021/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 9ae27593a..86b5702de 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -29,7 +29,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
-              <a href="{{ site.baseurl }}/projects">Projects</a>
+              <a href="{{ site.baseurl }}/projects.html">Projects</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
               <a href="{{ site.baseurl }}/archive">Archive</a>

From 5cb1f58fe9688a870f198acf9bd7db6b5e56d59f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:18:39 +0900
Subject: [PATCH 022/638] Create projects.md

---
 _pages/projects.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 _pages/projects.md

diff --git a/_pages/projects.md b/_pages/projects.md
new file mode 100644
index 000000000..3a75a1545
--- /dev/null
+++ b/_pages/projects.md
@@ -0,0 +1,49 @@
+---
+layout: default
+---
+
+<div class="projects">
+  {% for project in paginator.projects %}
+    <article class="project">
+      <a href="{{ site.baseurl }}{{ project.url }}">
+        <h1>{{ project.title }}</h1>
+
+        <div>
+          <p class="project_date">{{ project.date | date: "%B %e, %Y" }}</p>
+        </div>
+      </a>
+      <div class="entry">
+        {{ project.excerpt }}
+      </div>
+
+      <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>
+    </article>
+  {% endfor %}
+
+  <!-- pagination -->
+  {% if paginator.total_pages > 1 %}
+  <div class="pagination">
+    {% if paginator.previous_page %}
+      <a href="{{ paginator.previous_page_path | prepend: site.baseurl | replace: '//', '/' }}">&laquo; Prev</a>
+    {% else %}
+      <span>&laquo; Prev</span>
+    {% endif %}
+
+    {% for page in (1..paginator.total_pages) %}
+      {% if page == paginator.page %}
+        <span class="webjeda">{{ page }}</span>
+      {% elsif page == 1 %}
+        <a href="{{ '/' | prepend: site.baseurl | replace: '//', '/' }}">{{ page }}</a>
+      {% else %}
+        <a href="{{ site.paginate_path | prepend: site.baseurl | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
+      {% endif %}
+    {% endfor %}
+
+    {% if paginator.next_page %}
+      <a href="{{ paginator.next_page_path | prepend: site.baseurl | replace: '//', '/' }}">Next &raquo;</a>
+    {% else %}
+      <span>Next &raquo;</span>
+    {% endif %}
+  </div>
+  {% endif %}
+</div>

From 126cf0846e4c3cc5dc7b2b671abe78827ea3749e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:21:16 +0900
Subject: [PATCH 023/638] Update projects.md

---
 _pages/projects.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 3a75a1545..2c07033d2 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -1,5 +1,7 @@
 ---
-layout: default
+layout: project
+title: Projects
+permalink: /projects/
 ---
 
 <div class="projects">

From f0d2f5076572435c96c6b72366b1369ce424f9d4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:25:11 +0900
Subject: [PATCH 024/638] Delete _pages/projects.md

---
 _pages/projects.md | 51 ----------------------------------------------
 1 file changed, 51 deletions(-)
 delete mode 100644 _pages/projects.md

diff --git a/_pages/projects.md b/_pages/projects.md
deleted file mode 100644
index 2c07033d2..000000000
--- a/_pages/projects.md
+++ /dev/null
@@ -1,51 +0,0 @@
----
-layout: project
-title: Projects
-permalink: /projects/
----
-
-<div class="projects">
-  {% for project in paginator.projects %}
-    <article class="project">
-      <a href="{{ site.baseurl }}{{ project.url }}">
-        <h1>{{ project.title }}</h1>
-
-        <div>
-          <p class="project_date">{{ project.date | date: "%B %e, %Y" }}</p>
-        </div>
-      </a>
-      <div class="entry">
-        {{ project.excerpt }}
-      </div>
-
-      <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>
-    </article>
-  {% endfor %}
-
-  <!-- pagination -->
-  {% if paginator.total_pages > 1 %}
-  <div class="pagination">
-    {% if paginator.previous_page %}
-      <a href="{{ paginator.previous_page_path | prepend: site.baseurl | replace: '//', '/' }}">&laquo; Prev</a>
-    {% else %}
-      <span>&laquo; Prev</span>
-    {% endif %}
-
-    {% for page in (1..paginator.total_pages) %}
-      {% if page == paginator.page %}
-        <span class="webjeda">{{ page }}</span>
-      {% elsif page == 1 %}
-        <a href="{{ '/' | prepend: site.baseurl | replace: '//', '/' }}">{{ page }}</a>
-      {% else %}
-        <a href="{{ site.paginate_path | prepend: site.baseurl | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
-      {% endif %}
-    {% endfor %}
-
-    {% if paginator.next_page %}
-      <a href="{{ paginator.next_page_path | prepend: site.baseurl | replace: '//', '/' }}">Next &raquo;</a>
-    {% else %}
-      <span>Next &raquo;</span>
-    {% endif %}
-  </div>
-  {% endif %}
-</div>

From bb45ac5cbb1f230d64f2ffda1b31ab526a072724 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:25:36 +0900
Subject: [PATCH 025/638] Create projects.html

---
 _pages/projects.html | 49 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 _pages/projects.html

diff --git a/_pages/projects.html b/_pages/projects.html
new file mode 100644
index 000000000..3a75a1545
--- /dev/null
+++ b/_pages/projects.html
@@ -0,0 +1,49 @@
+---
+layout: default
+---
+
+<div class="projects">
+  {% for project in paginator.projects %}
+    <article class="project">
+      <a href="{{ site.baseurl }}{{ project.url }}">
+        <h1>{{ project.title }}</h1>
+
+        <div>
+          <p class="project_date">{{ project.date | date: "%B %e, %Y" }}</p>
+        </div>
+      </a>
+      <div class="entry">
+        {{ project.excerpt }}
+      </div>
+
+      <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>
+    </article>
+  {% endfor %}
+
+  <!-- pagination -->
+  {% if paginator.total_pages > 1 %}
+  <div class="pagination">
+    {% if paginator.previous_page %}
+      <a href="{{ paginator.previous_page_path | prepend: site.baseurl | replace: '//', '/' }}">&laquo; Prev</a>
+    {% else %}
+      <span>&laquo; Prev</span>
+    {% endif %}
+
+    {% for page in (1..paginator.total_pages) %}
+      {% if page == paginator.page %}
+        <span class="webjeda">{{ page }}</span>
+      {% elsif page == 1 %}
+        <a href="{{ '/' | prepend: site.baseurl | replace: '//', '/' }}">{{ page }}</a>
+      {% else %}
+        <a href="{{ site.paginate_path | prepend: site.baseurl | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
+      {% endif %}
+    {% endfor %}
+
+    {% if paginator.next_page %}
+      <a href="{{ paginator.next_page_path | prepend: site.baseurl | replace: '//', '/' }}">Next &raquo;</a>
+    {% else %}
+      <span>Next &raquo;</span>
+    {% endif %}
+  </div>
+  {% endif %}
+</div>

From 97cf155fd6b9384a9ea04bd1dfae45fc2f1e9e26 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:26:44 +0900
Subject: [PATCH 026/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 86b5702de..9ae27593a 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -29,7 +29,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
-              <a href="{{ site.baseurl }}/projects.html">Projects</a>
+              <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
               <a href="{{ site.baseurl }}/archive">Archive</a>

From 704e986c6b1aca43a06367c15bdc145b051338e3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:29:26 +0900
Subject: [PATCH 027/638] Rename projects.html to projects.md

---
 _pages/{projects.html => projects.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _pages/{projects.html => projects.md} (100%)

diff --git a/_pages/projects.html b/_pages/projects.md
similarity index 100%
rename from _pages/projects.html
rename to _pages/projects.md

From dd065d3ffd9234dbafa47b1b8e3d115af14e4027 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:43:53 +0900
Subject: [PATCH 028/638] Update projects.md

---
 _pages/projects.md | 57 +++++++++++++---------------------------------
 1 file changed, 16 insertions(+), 41 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 3a75a1545..e2ed77c4f 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -1,49 +1,24 @@
 ---
-layout: default
+layout: page
+permalink: /projects/
+title: Projects
 ---
 
-<div class="projects">
-  {% for project in paginator.projects %}
-    <article class="project">
-      <a href="{{ site.baseurl }}{{ project.url }}">
-        <h1>{{ project.title }}</h1>
 
-        <div>
-          <p class="project_date">{{ project.date | date: "%B %e, %Y" }}</p>
-        </div>
-      </a>
-      <div class="entry">
-        {{ project.excerpt }}
-      </div>
-
-      <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>
+<div id="projects">
+{% for category in [i for i in site.categories if i == 'HTML'] %}
+  <div class="archive-group">
+    {% capture category_name %}{{ category | first }}{% endcapture %}
+    <div id="#{{ category_name | slugize }}"></div>
+    <p></p>
+    
+    <h3 class="category-head">{{ category_name }}</h3>
+    <a name="{{ category_name | slugize }}"></a>
+    {% for post in site.categories[category_name] %}
+    <article class="archive-item">
+      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
     </article>
-  {% endfor %}
-
-  <!-- pagination -->
-  {% if paginator.total_pages > 1 %}
-  <div class="pagination">
-    {% if paginator.previous_page %}
-      <a href="{{ paginator.previous_page_path | prepend: site.baseurl | replace: '//', '/' }}">&laquo; Prev</a>
-    {% else %}
-      <span>&laquo; Prev</span>
-    {% endif %}
-
-    {% for page in (1..paginator.total_pages) %}
-      {% if page == paginator.page %}
-        <span class="webjeda">{{ page }}</span>
-      {% elsif page == 1 %}
-        <a href="{{ '/' | prepend: site.baseurl | replace: '//', '/' }}">{{ page }}</a>
-      {% else %}
-        <a href="{{ site.paginate_path | prepend: site.baseurl | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
-      {% endif %}
     {% endfor %}
-
-    {% if paginator.next_page %}
-      <a href="{{ paginator.next_page_path | prepend: site.baseurl | replace: '//', '/' }}">Next &raquo;</a>
-    {% else %}
-      <span>Next &raquo;</span>
-    {% endif %}
   </div>
-  {% endif %}
+{% endfor %}
 </div>

From 7ddb8b25e28c0a5d6b912974036df21082422291 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:47:08 +0900
Subject: [PATCH 029/638] Update projects.md

---
 _pages/projects.md | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index e2ed77c4f..6ee210b9f 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -6,19 +6,24 @@ title: Projects
 
 
 <div id="projects">
-{% for category in [i for i in site.categories if i == 'HTML'] %}
+{% assign html_category = site.categories.HTML %}
+{% if html_category %}
   <div class="archive-group">
-    {% capture category_name %}{{ category | first }}{% endcapture %}
-    <div id="#{{ category_name | slugize }}"></div>
+    <div id="#html"></div>
     <p></p>
-    
-    <h3 class="category-head">{{ category_name }}</h3>
-    <a name="{{ category_name | slugize }}"></a>
-    {% for post in site.categories[category_name] %}
-    <article class="archive-item">
-      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
-    </article>
+    <h3 class="category-head">HTML</h3>
+    <a name="html"></a>
+    {% for post in html_category %}
+      <article class="archive-item">
+        <h4><a href="{{ site.baseurl }}{{ post.url }}">
+          {% if post.title and post.title != "" %}
+            {{ post.title }}
+          {% else %}
+            {{ post.excerpt | strip_html }}
+          {% endif %}
+        </a></h4>
+      </article>
     {% endfor %}
   </div>
-{% endfor %}
+{% endif %}
 </div>

From 8d9775481f94bf5cae00efc0212ee71d72c765b9 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:51:12 +0900
Subject: [PATCH 030/638] Update projects.md

---
 _pages/projects.md | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 6ee210b9f..f0401b2ae 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -4,26 +4,21 @@ permalink: /projects/
 title: Projects
 ---
 
-
 <div id="projects">
-{% assign html_category = site.categories.HTML %}
-{% if html_category %}
-  <div class="archive-group">
-    <div id="#html"></div>
+  {% print(type(site.categories), site.categories) %}
+{% for category in [i for i in site.categories if i == 'HTML'] %}
+  <div class="projects-group">
+    {% capture category_name %}{{ category | first }}{% endcapture %}
+    <div id="#{{ category_name | slugize }}"></div>
     <p></p>
-    <h3 class="category-head">HTML</h3>
-    <a name="html"></a>
-    {% for post in html_category %}
-      <article class="archive-item">
-        <h4><a href="{{ site.baseurl }}{{ post.url }}">
-          {% if post.title and post.title != "" %}
-            {{ post.title }}
-          {% else %}
-            {{ post.excerpt | strip_html }}
-          {% endif %}
-        </a></h4>
-      </article>
+    
+    <h3 class="category-head">{{ category_name }}</h3>
+    <a name="{{ category_name | slugize }}"></a>
+    {% for post in site.categories[category_name] %}
+    <article class="archive-item">
+      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
+    </article>
     {% endfor %}
   </div>
-{% endif %}
+{% endfor %}
 </div>

From 386acd9a14b55372299a2ba6d9c2f61dd92c50c0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:51:59 +0900
Subject: [PATCH 031/638] Delete projects.html

---
 projects.html | 49 -------------------------------------------------
 1 file changed, 49 deletions(-)
 delete mode 100644 projects.html

diff --git a/projects.html b/projects.html
deleted file mode 100644
index 3a75a1545..000000000
--- a/projects.html
+++ /dev/null
@@ -1,49 +0,0 @@
----
-layout: default
----
-
-<div class="projects">
-  {% for project in paginator.projects %}
-    <article class="project">
-      <a href="{{ site.baseurl }}{{ project.url }}">
-        <h1>{{ project.title }}</h1>
-
-        <div>
-          <p class="project_date">{{ project.date | date: "%B %e, %Y" }}</p>
-        </div>
-      </a>
-      <div class="entry">
-        {{ project.excerpt }}
-      </div>
-
-      <a href="{{ site.baseurl }}{{ project.url }}" class="read-more">Read More</a>
-    </article>
-  {% endfor %}
-
-  <!-- pagination -->
-  {% if paginator.total_pages > 1 %}
-  <div class="pagination">
-    {% if paginator.previous_page %}
-      <a href="{{ paginator.previous_page_path | prepend: site.baseurl | replace: '//', '/' }}">&laquo; Prev</a>
-    {% else %}
-      <span>&laquo; Prev</span>
-    {% endif %}
-
-    {% for page in (1..paginator.total_pages) %}
-      {% if page == paginator.page %}
-        <span class="webjeda">{{ page }}</span>
-      {% elsif page == 1 %}
-        <a href="{{ '/' | prepend: site.baseurl | replace: '//', '/' }}">{{ page }}</a>
-      {% else %}
-        <a href="{{ site.paginate_path | prepend: site.baseurl | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
-      {% endif %}
-    {% endfor %}
-
-    {% if paginator.next_page %}
-      <a href="{{ paginator.next_page_path | prepend: site.baseurl | replace: '//', '/' }}">Next &raquo;</a>
-    {% else %}
-      <span>Next &raquo;</span>
-    {% endif %}
-  </div>
-  {% endif %}
-</div>

From 7eac420c3ffe92937d2eca81cba47abfad8bce83 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:54:53 +0900
Subject: [PATCH 032/638] Update projects.md

---
 _pages/projects.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index f0401b2ae..6b7f9b070 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -4,10 +4,9 @@ permalink: /projects/
 title: Projects
 ---
 
-<div id="projects">
-  {% print(type(site.categories), site.categories) %}
-{% for category in [i for i in site.categories if i == 'HTML'] %}
-  <div class="projects-group">
+<div id="archives">
+{% for category in [i for i in site.categories if i == "HTML"] %}
+  <div class="archive-group">
     {% capture category_name %}{{ category | first }}{% endcapture %}
     <div id="#{{ category_name | slugize }}"></div>
     <p></p>

From 1080f65e6ccce8e5668817fba9f34821124a80ff Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:57:29 +0900
Subject: [PATCH 033/638] Update projects.md

---
 _pages/projects.md | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 6b7f9b070..e90e6f0ae 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -5,19 +5,28 @@ title: Projects
 ---
 
 <div id="archives">
-{% for category in [i for i in site.categories if i == "HTML"] %}
-  <div class="archive-group">
-    {% capture category_name %}{{ category | first }}{% endcapture %}
-    <div id="#{{ category_name | slugize }}"></div>
-    <p></p>
-    
-    <h3 class="category-head">{{ category_name }}</h3>
-    <a name="{{ category_name | slugize }}"></a>
-    {% for post in site.categories[category_name] %}
-    <article class="archive-item">
-      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
-    </article>
-    {% endfor %}
-  </div>
-{% endfor %}
+  <section id="archive">
+     <h3>Most Recent Posts</h3>
+      {%for post in site.projects %}
+      {% unless post.next %}
+      <ul class="this">
+          {% else %}
+          {% capture month %}{{ post.date | date: '%B %Y' }}{% endcapture %}
+          {% capture nmonth %}{{ post.next.date | date: '%B %Y' }}{% endcapture %}
+          {% capture year %}{{ post.date | date: '%Y' }}{% endcapture %}
+          {% capture nyear %}{{ post.next.date | date: '%Y' }}{% endcapture %}
+          {% if year != nyear %}
+      </ul>
+      <h2 style="text-align:left;">{{ post.date | date: '%Y' }}</h2>
+      <ul class="past">
+          {% endif %}
+          {% if month != nmonth %}
+          <h3 style="text-align:left;">{{ post.date | date: '%B %Y' }}</h3>
+          {% endif %}
+          {% endunless %}
+          <p><b><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></b> - {% if post.date and post.date != "" %}{{ post.date | date: "%e %B %Y" }}{%endif%}</p>
+          {% endfor %}
+      </ul>
+    <h3>Oldest Posts</h3>
+  </section>
 </div>

From 6721a18cc9b389840c98f31ded0187607a20b02b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 21:59:13 +0900
Subject: [PATCH 034/638] Update projects.md

---
 _pages/projects.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index e90e6f0ae..40d1259c5 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -7,7 +7,7 @@ title: Projects
 <div id="archives">
   <section id="archive">
      <h3>Most Recent Posts</h3>
-      {%for post in site.projects %}
+      {%for post in site.posts %}
       {% unless post.next %}
       <ul class="this">
           {% else %}

From dca6e96ff47571470a8cbb5d041bd7e32d0b4e5d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:04:14 +0900
Subject: [PATCH 035/638] Update projects.md

---
 _pages/projects.md | 41 +++++++++++++++++------------------------
 1 file changed, 17 insertions(+), 24 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 40d1259c5..c7d80a208 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -5,28 +5,21 @@ title: Projects
 ---
 
 <div id="archives">
-  <section id="archive">
-     <h3>Most Recent Posts</h3>
-      {%for post in site.posts %}
-      {% unless post.next %}
-      <ul class="this">
-          {% else %}
-          {% capture month %}{{ post.date | date: '%B %Y' }}{% endcapture %}
-          {% capture nmonth %}{{ post.next.date | date: '%B %Y' }}{% endcapture %}
-          {% capture year %}{{ post.date | date: '%Y' }}{% endcapture %}
-          {% capture nyear %}{{ post.next.date | date: '%Y' }}{% endcapture %}
-          {% if year != nyear %}
-      </ul>
-      <h2 style="text-align:left;">{{ post.date | date: '%Y' }}</h2>
-      <ul class="past">
-          {% endif %}
-          {% if month != nmonth %}
-          <h3 style="text-align:left;">{{ post.date | date: '%B %Y' }}</h3>
-          {% endif %}
-          {% endunless %}
-          <p><b><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></b> - {% if post.date and post.date != "" %}{{ post.date | date: "%e %B %Y" }}{%endif%}</p>
-          {% endfor %}
-      </ul>
-    <h3>Oldest Posts</h3>
-  </section>
+{% for category in site.categories %}
+  <div class="archive-group">
+    {% capture category_name %}{{ category | first }}{% endcapture %}
+    {% if category_name=='HTML': %}
+    <div id="#{{ category_name | slugize }}"></div>
+    <p></p>
+    
+    <h3 class="category-head">{{ category_name }}</h3>
+    <a name="{{ category_name | slugize }}"></a>
+    {% for post in site.categories[category_name] %}
+    <article class="archive-item">
+      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
+    </article>
+    {% endfor %}
+    {% endif %}
+  </div>
+{% endfor %}
 </div>

From 649b0523c165ddbe9b7385606bcd628b0907146e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:07:23 +0900
Subject: [PATCH 036/638] Update projects.md

---
 _pages/projects.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index c7d80a208..43e3eb011 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -8,7 +8,7 @@ title: Projects
 {% for category in site.categories %}
   <div class="archive-group">
     {% capture category_name %}{{ category | first }}{% endcapture %}
-    {% if category_name=='HTML': %}
+    {% if category_name=='HTML' %}
     <div id="#{{ category_name | slugize }}"></div>
     <p></p>
     

From 4e78f2f4a96275f0ca80f51b0c63910850a96d43 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:11:54 +0900
Subject: [PATCH 037/638] Delete _projects/2024-02-02-test.md

---
 _projects/2024-02-02-test.md | 86 ------------------------------------
 1 file changed, 86 deletions(-)
 delete mode 100644 _projects/2024-02-02-test.md

diff --git a/_projects/2024-02-02-test.md b/_projects/2024-02-02-test.md
deleted file mode 100644
index d78ab7d32..000000000
--- a/_projects/2024-02-02-test.md
+++ /dev/null
@@ -1,86 +0,0 @@
----
-layout: project
-title: Test Project 
-categories: [Miscellaneous, Jekyll]
----
-
-[Reverie](https://github.com/amitmerchant1990/reverie) is a [Jekyll](https://jekyllrb.com/)-powered theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for my blog.
-
-This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
-
-![](/images/reverie-demo.png)
-
-## Features overview
-
-- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
-- Fully responsive and mobile optimized base theme
-- Sass/Coffeescript support using Jekyll 2.0
-- Free hosting on your GitHub Pages user site
-- All the SEO goodies comes in-built
-- Markdown blogging
-- Syntax highlighting using Pygments
-    - [Dracula syntax theme](https://draculatheme.com/) included
-- Disqus commenting
-- Google Analytics integration
-- Fuzzy search across blog posts
-- Pagination of posts works out-of-the-box.
-- Categorize posts out-of-the box
-- RSS Feed
-- In-built sitemap
-
-<div style="text-align: center;">
- <script async type="text/javascript" src="//cdn.carbonads.com/carbon.js?serve=CE7D6KJY&placement=wwwamitmerchantcom" id="_carbonads_js"></script>
-</div>
-
-## Using Reverie on GitHub Pages
-
-### Step 1) Fork Reverie to your User Repository
-
-Fork [this repository](https://github.com/amitmerchant1990/reverie), then rename the repository to `yourgithubusername.github.io`.
-
-Alternatively, you can use [Use this template](https://github.com/amitmerchant1990/reverie/generate) button if you want to create a repository with a clean commit history which will use Reverie as a template.
-
-Your Jekyll blog will often be viewable immediately at <https://yourgithubusername.github.io> (if it's not, you can often force it to build by completing step 2)
-
-### Step 2) Customize and view your site
-
-Enter your site name, description, avatar and many other options by editing the `_config.yml` file. You can easily turn on Google Analytics tracking, Disqus commenting and social icons here.
-
-Making a change to `_config.yml` (or any file in your repository) will force GitHub Pages to rebuild your site with jekyll. Your rebuilt site will be viewable a few seconds later at <https://yourgithubusername.github.io> - if not, give it ten minutes as GitHub suggests and it'll appear soon.
-
-### Step 3) Publish your first blog post
-
-Create a new file called `/_posts/2019-2-13-Hello-World.md` to publish your first blog post. That's all you need to do to publish your first blog post! This [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) might come in handy while writing the posts.
-
-> You can add additional posts in the browser on GitHub.com too! Just hit the <kbd>Create new file</kbd> button in `/_posts/` to create new content. Just make sure to include the [front-matter](http://jekyllrb.com/docs/frontmatter/) block at the top of each new blog post and make sure the post's filename is in this format: year-month-day-title.md
-
-## Using Categories in Reverie
-
-You can categorize your content based on `categories` in Reverie. For this, you just need to add `categories` in front matter like below:
-
-For adding single category:
-
-```md
-categories: JavaScript
-```
-
-For adding multiple categories:
-
-```md
-categories: [PHP, Laravel]
-```
-
-The contegorized content can be shown over this URL: <https://yourgithubusername.github.io/categories/>
-
-## RSS
-
-The generated [RSS feed](https://en.wikipedia.org/wiki/RSS) of your blog can be found at <https://yourgithubusername.github.io/feed>. You can see the example RSS feed over [here](https://www.amitmerchant.com/reverie/feed).
-
-## Sitemap
-
-The generated sitemap of your blog can be found at <https://yourgithubusername.github.io/sitemap>. You can see the example sitemap feed over [here](https://www.amitmerchant.com/reverie/sitemap).
-
-## License
-
-MIT
-

From 082d6dc0cfeaa75b2ca8bf1436f9bc64c6744f8f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:15:04 +0900
Subject: [PATCH 038/638] Delete _layouts/project.html

---
 _layouts/project.html | 39 ---------------------------------------
 1 file changed, 39 deletions(-)
 delete mode 100644 _layouts/project.html

diff --git a/_layouts/project.html b/_layouts/project.html
deleted file mode 100644
index 9bdbc8a40..000000000
--- a/_layouts/project.html
+++ /dev/null
@@ -1,39 +0,0 @@
----
-layout: default
----
-
-<article class="project detailed">
-  <h1>{{ page.title }}</h1>
-
-  <div>
-    <p class="author_title">{{site.author}}  ·  {{ page.date | date: "%B %e, %Y" }}</p>
-    {% if page.last_modified_at %}
-    <p class="author_title" datetime="{{ page.last_modified_at | date_to_xmlschema }}">(Updated: {{ page.last_modified_at | date: "%b %-d, %Y" }})</p>
-    {% endif %}
-    <div class="project-tags">
-      {% if project %}
-        {% assign categories = project.categories %}
-      {% else %}
-        {% assign categories = page.categories %}
-      {% endif %}
-      {% for category in categories %}
-        <a href="{{site.baseurl}}/categories/#{{category|slugize}}">{{category}}</a>
-        {% unless forloop.last %}&nbsp;{% endunless %}
-      {% endfor %}
-    </div>
-  </div>
-    
-  <div class="entry">
-    {{ content }}
-  </div>
-
-  <div>
-    <p><span class="share-box">Share:</span> <a href="http://twitter.com/share?text={{ page.title }}&url={{site.url}}{{page.url}}" target="_blank">Twitter</a>, <a href="https://www.facebook.com/sharer.php?u={{site.url}}{{page.url}}" target="_blank">Facebook</a></p>
-  </div>
-
-  <!--<div class="date">
-    Written on {{ page.date | date: "%B %e, %Y" }}
-  </div>-->
-
-  {% include disqus.html %}
-</article>

From 023026dda8b3107b92c1d2c8fa6f92e2da6f966c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:16:13 +0900
Subject: [PATCH 039/638] Update projects.md

---
 _pages/projects.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 43e3eb011..d4efbc620 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -11,12 +11,10 @@ title: Projects
     {% if category_name=='HTML' %}
     <div id="#{{ category_name | slugize }}"></div>
     <p></p>
-    
-    <h3 class="category-head">{{ category_name }}</h3>
     <a name="{{ category_name | slugize }}"></a>
     {% for post in site.categories[category_name] %}
     <article class="archive-item">
-      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
+      <h3><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h3>
     </article>
     {% endfor %}
     {% endif %}

From 6b16fa6eaae44def4cfe76dc26af8ce436976f68 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:17:13 +0900
Subject: [PATCH 040/638] Delete _posts/2019-08-08-no-title-posts.md

---
 _posts/2019-08-08-no-title-posts.md | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 _posts/2019-08-08-no-title-posts.md

diff --git a/_posts/2019-08-08-no-title-posts.md b/_posts/2019-08-08-no-title-posts.md
deleted file mode 100644
index d14509ea8..000000000
--- a/_posts/2019-08-08-no-title-posts.md
+++ /dev/null
@@ -1,6 +0,0 @@
----
-layout: post
-title: ""
-categories: Miscellaneous
----
-Sometimes, your post just stands for itself and doesn't need a title. And that's fine, too!
\ No newline at end of file

From c5afa828b992c785cb51a97c72c997181e13cb71 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:18:55 +0900
Subject: [PATCH 041/638] Delete _posts/2019-8-12-code-snippets.md

---
 _posts/2019-8-12-code-snippets.md | 42 -------------------------------
 1 file changed, 42 deletions(-)
 delete mode 100644 _posts/2019-8-12-code-snippets.md

diff --git a/_posts/2019-8-12-code-snippets.md b/_posts/2019-8-12-code-snippets.md
deleted file mode 100644
index 6ebbd2b64..000000000
--- a/_posts/2019-8-12-code-snippets.md
+++ /dev/null
@@ -1,42 +0,0 @@
----
-layout: post
-title:  Code snippets in a blog post
-categories: [HTML,Code]
----
-
-This post demonstrate the use of code snippets in the theme. The code snippets are powered by [Pygments](http://pygments.org/) and the code theme that is been used in Reverie is called [Dracula](https://draculatheme.com/).
-
-This is a raw snippet:
-
-```
-hello world
-123
-This is a text snippet
-```
-
-This is a PHP snippet:
-
-```php
-<?php
-    echo 'Hello, World!';
-?>
-```
-
-This is a JavaScript snippet:
-
-```js
-const add = (a, b) => a + b
-const minus = (a, b) => a - b
-
-console.log(add(100,200))  // 300
-console.log(minus(100,200))  // -100
-```
-
-This is a Python snippet:
-
-```python
-def say_hello():
-    print("hello world!")
-
-say_hello()   // "hello world!"
-```

From 9baea8bf1bf711aef6c3301940bfb163af39e533 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:19:06 +0900
Subject: [PATCH 042/638] Delete _posts/2019-7-28-super-short-article.md

---
 _posts/2019-7-28-super-short-article.md | 7 -------
 1 file changed, 7 deletions(-)
 delete mode 100644 _posts/2019-7-28-super-short-article.md

diff --git a/_posts/2019-7-28-super-short-article.md b/_posts/2019-7-28-super-short-article.md
deleted file mode 100644
index ddb155b7f..000000000
--- a/_posts/2019-7-28-super-short-article.md
+++ /dev/null
@@ -1,7 +0,0 @@
----
-layout: post
-title: Some articles are just so short that we've to make the footer stick
-categories: Miscellaneous
----
-
-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
\ No newline at end of file

From 0e167647c3d779ea7879f53891b50973b1fdbf51 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:19:34 +0900
Subject: [PATCH 043/638] Delete _posts/2019-7-27-super-long-article.md

---
 _posts/2019-7-27-super-long-article.md | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 _posts/2019-7-27-super-long-article.md

diff --git a/_posts/2019-7-27-super-long-article.md b/_posts/2019-7-27-super-long-article.md
deleted file mode 100644
index d148630e9..000000000
--- a/_posts/2019-7-27-super-long-article.md
+++ /dev/null
@@ -1,11 +0,0 @@
----
-layout: post
-title: Some articles are just so long they deserve a really long title to see if things will break well
-categories: Miscellaneous
----
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce bibendum neque eget nunc mattis eu sollicitudin enim tincidunt. Vestibulum lacus tortor, ultricies id dignissim ac, bibendum in velit. Proin convallis mi ac felis pharetra aliquam. Curabitur dignissim accumsan rutrum. In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris. Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc. Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce bibendum neque eget nunc mattis eu sollicitudin enim tincidunt. Vestibulum lacus tortor, ultricies id dignissim ac, bibendum in velit. Proin convallis mi ac felis pharetra aliquam. Curabitur dignissim accumsan rutrum. In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris. Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc. Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce bibendum neque eget nunc mattis eu sollicitudin enim tincidunt. Vestibulum lacus tortor, ultricies id dignissim ac, bibendum in velit. Proin convallis mi ac felis pharetra aliquam. Curabitur dignissim accumsan rutrum. In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris. Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc. Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.

From 864f1f47b2a693ad7e79acc26e7e96050baebe87 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:21:06 +0900
Subject: [PATCH 044/638] Delete
 _posts/2019-02-13-introducing-reverie-jekyll-theme.md

---
 ...-02-13-introducing-reverie-jekyll-theme.md | 87 -------------------
 1 file changed, 87 deletions(-)
 delete mode 100644 _posts/2019-02-13-introducing-reverie-jekyll-theme.md

diff --git a/_posts/2019-02-13-introducing-reverie-jekyll-theme.md b/_posts/2019-02-13-introducing-reverie-jekyll-theme.md
deleted file mode 100644
index 8b1075da4..000000000
--- a/_posts/2019-02-13-introducing-reverie-jekyll-theme.md
+++ /dev/null
@@ -1,87 +0,0 @@
----
-layout: post
-title: Introducing Reverie - A ridiculously elegant Jekyll theme
-categories: [Miscellaneous, Jekyll]
----
-
-[Reverie](https://github.com/amitmerchant1990/reverie) is a [Jekyll](https://jekyllrb.com/)-powered theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for my blog.
-
-This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
-
-![](/images/reverie-demo.png)
-
-## Features overview
-
-- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
-- Fully responsive and mobile optimized base theme
-- Sass/Coffeescript support using Jekyll 2.0
-- Free hosting on your GitHub Pages user site
-- All the SEO goodies comes in-built
-- Markdown blogging
-- Syntax highlighting using Pygments
-    - [Dracula syntax theme](https://draculatheme.com/) included
-- Disqus commenting
-- Google Analytics integration
-- Fuzzy search across blog posts
-- Pagination of posts works out-of-the-box.
-- Categorize posts out-of-the box
-- RSS Feed
-- In-built sitemap
-
-<div style="text-align: center;">
- <script async type="text/javascript" src="//cdn.carbonads.com/carbon.js?serve=CE7D6KJY&placement=wwwamitmerchantcom" id="_carbonads_js"></script>
-</div>
-
-## Using Reverie on GitHub Pages
-
-### Step 1) Fork Reverie to your User Repository
-
-Fork [this repository](https://github.com/amitmerchant1990/reverie), then rename the repository to `yourgithubusername.github.io`.
-
-Alternatively, you can use [Use this template](https://github.com/amitmerchant1990/reverie/generate) button if you want to create a repository with a clean commit history which will use Reverie as a template.
-
-Your Jekyll blog will often be viewable immediately at <https://yourgithubusername.github.io> (if it's not, you can often force it to build by completing step 2)
-
-### Step 2) Customize and view your site
-
-Enter your site name, description, avatar and many other options by editing the `_config.yml` file. You can easily turn on Google Analytics tracking, Disqus commenting and social icons here.
-
-Making a change to `_config.yml` (or any file in your repository) will force GitHub Pages to rebuild your site with jekyll. Your rebuilt site will be viewable a few seconds later at <https://yourgithubusername.github.io> - if not, give it ten minutes as GitHub suggests and it'll appear soon.
-
-### Step 3) Publish your first blog post
-
-Create a new file called `/_posts/2019-2-13-Hello-World.md` to publish your first blog post. That's all you need to do to publish your first blog post! This [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) might come in handy while writing the posts.
-
-> You can add additional posts in the browser on GitHub.com too! Just hit the <kbd>Create new file</kbd> button in `/_posts/` to create new content. Just make sure to include the [front-matter](http://jekyllrb.com/docs/frontmatter/) block at the top of each new blog post and make sure the post's filename is in this format: year-month-day-title.md
-
-## Using Categories in Reverie
-
-You can categorize your content based on `categories` in Reverie. For this, you just need to add `categories` in front matter like below:
-
-For adding single category:
-
-```md
-categories: JavaScript
-```
-
-For adding multiple categories:
-
-```md
-categories: [PHP, Laravel]
-```
-
-The contegorized content can be shown over this URL: <https://yourgithubusername.github.io/categories/>
-
-## RSS
-
-The generated [RSS feed](https://en.wikipedia.org/wiki/RSS) of your blog can be found at <https://yourgithubusername.github.io/feed>. You can see the example RSS feed over [here](https://www.amitmerchant.com/reverie/feed).
-
-## Sitemap
-
-The generated sitemap of your blog can be found at <https://yourgithubusername.github.io/sitemap>. You can see the example sitemap feed over [here](https://www.amitmerchant.com/reverie/sitemap).
-
-## License
-
-MIT
-
-

From 04e3d619a4274cd907ae8c66561a489a6dbc1660 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:35:41 +0900
Subject: [PATCH 045/638] Update about.md

---
 _pages/about.md | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/_pages/about.md b/_pages/about.md
index e8c5fd239..89fed0a1e 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -4,26 +4,24 @@ title: About
 permalink: /about/
 ---
 
-[Reverie](https://github.com/amitmerchant1990/reverie) is a Jekyll theme which is simple and opinionated. It's actually a fork of [jekyll-now](https://github.com/barryclark/jekyll-now) with some additional features and personal touches which I've implemented to suit my needs for [my blog](https://www.amitmerchant.com).
+Hello!
 
-This is a plug-and-play Jekyll theme which you can use on GitHub Pages without even setting up a local environment.
+On the 2nd of January 2024, I embarked on a, what I thought would be, 50 day journey to learn about Machine Learning. It is way past the 50 day mark, and I keep learning and blogging. 
 
-## Features
+Topics covered so far:
+* Calculus
+* Probability and statistics
+* Classical ML methods
+* Deep learning
+* Computer vision
+* Natural Language Processing
+* Graph Neural Networks
 
-- Command-line free fork-first workflow, using GitHub.com to create, customize and post to your blog
-- Fully responsive and mobile optimized base theme
-- Sass/Coffeescript support using Jekyll 2.0
-- Free hosting on your GitHub Pages user site
-- All the SEO goodies comes in-built
-- Markdown blogging
-- Syntax highlighting using Pygments
-    - [Dracula syntax theme](https://draculatheme.com/) included
-- Disqus commenting
-- Google Analytics integration
-- Fuzzy search across blog posts
-- Pagination of posts works out-of-the-box.
-- Categorize posts out-of-the box
-- RSS Feed
-- In-built sitemap
+The list is not in order, I keep going between high and low-level concepts, so that I do not fall victim to the [law of leaky abstractions](https://www.joelonsoftware.com/2002/11/11/the-law-of-leaky-abstractions/). 
+
+Courses covered:
+* aaa
+
+Books read:
+* aaa
 
-Learn more about it [here](https://github.com/amitmerchant1990/reverie) on how to get started.

From b29eff1067bd09effd4ef24ad447604bb28c25d7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 23 Apr 2024 22:39:19 +0900
Subject: [PATCH 046/638] Update about.md

---
 _pages/about.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/about.md b/_pages/about.md
index 89fed0a1e..5f1024b32 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -17,7 +17,7 @@ Topics covered so far:
 * Natural Language Processing
 * Graph Neural Networks
 
-The list is not in order, I keep going between high and low-level concepts, so that I do not fall victim to the [law of leaky abstractions](https://www.joelonsoftware.com/2002/11/11/the-law-of-leaky-abstractions/). 
+The list is not in order, I keep going between high and low-level concepts, so that I do not fall victim to the <a href="https://www.joelonsoftware.com/2002/11/11/the-law-of-leaky-abstractions/" target="_blank">law of leaky abstractions</a>
 
 Courses covered:
 * aaa

From aba7adc5a935c4c828bcd9a54d40befcd02d3916 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 18:34:18 +0900
Subject: [PATCH 047/638] Create 2023-12-30-[Pre-study] Reviewing basic
 statistics (Day 1 is 2nd of Jan)

---
 ...eviewing basic statistics (Day 1 is 2nd of Jan) | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 _posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan)

diff --git a/_posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan) b/_posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan)
new file mode 100644
index 000000000..55ec1979b
--- /dev/null
+++ b/_posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan)	
@@ -0,0 +1,14 @@
+---
+layout: post
+title:  Pullquotes
+categories: [HTML,Code]
+excerpt: In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic.
+---
+
+In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic. {% include pullquote.html quote="It is typically placed in a larger or distinctive typeface and on the same page." %} Pull quotes are often used in magazine and newspaper articles, annual reports, and brochures, as well as on the web. They can add visual interest to text-heavy pages with few images or illustrations.
+
+Placement of a pull quote on a page may be defined in a publication's or website's style guide. Such a typographic device may or may not be aligned with a column on the page. Some designers, for example, choose not to align the quote. In that case, the quotation cuts into two or more columns, as in the example shown. Because the pull quote invites the reader to read about the highlighted material, the pull quote should appear before the text it cites and, generally, fairly close to it.
+
+Pull quotes need not be a verbatim copy of the text being quoted; depending on a publication's house style, pull quotes may be abbreviated for space or paraphrased for clarity, with or without indication.
+
+A disadvantage of pull quotes as a design element is that they can disrupt the reading process of readers invested in reading the text sequentially by drawing attention to ghost fragments out of context. At the other extreme, when pull quotes are used to break up what would otherwise be a formless wall of text, pull quote can serve as visual landmarks to help the reader maintain a sense of sequence and place.

From e0f21630ec8fbc3abd3c357f4cf73ff7b17b2b08 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 18:36:42 +0900
Subject: [PATCH 048/638] Delete _posts/2023-12-30-[Pre-study] Reviewing basic
 statistics (Day 1 is 2nd of Jan)

---
 ...eviewing basic statistics (Day 1 is 2nd of Jan) | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 _posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan)

diff --git a/_posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan) b/_posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan)
deleted file mode 100644
index 55ec1979b..000000000
--- a/_posts/2023-12-30-[Pre-study] Reviewing basic statistics (Day 1 is 2nd of Jan)	
+++ /dev/null
@@ -1,14 +0,0 @@
----
-layout: post
-title:  Pullquotes
-categories: [HTML,Code]
-excerpt: In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic.
----
-
-In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic. {% include pullquote.html quote="It is typically placed in a larger or distinctive typeface and on the same page." %} Pull quotes are often used in magazine and newspaper articles, annual reports, and brochures, as well as on the web. They can add visual interest to text-heavy pages with few images or illustrations.
-
-Placement of a pull quote on a page may be defined in a publication's or website's style guide. Such a typographic device may or may not be aligned with a column on the page. Some designers, for example, choose not to align the quote. In that case, the quotation cuts into two or more columns, as in the example shown. Because the pull quote invites the reader to read about the highlighted material, the pull quote should appear before the text it cites and, generally, fairly close to it.
-
-Pull quotes need not be a verbatim copy of the text being quoted; depending on a publication's house style, pull quotes may be abbreviated for space or paraphrased for clarity, with or without indication.
-
-A disadvantage of pull quotes as a design element is that they can disrupt the reading process of readers invested in reading the text sequentially by drawing attention to ghost fragments out of context. At the other extreme, when pull quotes are used to break up what would otherwise be a formless wall of text, pull quote can serve as visual landmarks to help the reader maintain a sense of sequence and place.

From 3ad07b7deffdad1b6cc6718124a206461142bfba Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 18:38:03 +0900
Subject: [PATCH 049/638] Create 2023-12-30-pre-study

---
 _posts/2023-12-30-pre-study | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 _posts/2023-12-30-pre-study

diff --git a/_posts/2023-12-30-pre-study b/_posts/2023-12-30-pre-study
new file mode 100644
index 000000000..18c8c2836
--- /dev/null
+++ b/_posts/2023-12-30-pre-study
@@ -0,0 +1,12 @@
+---
+layout: post
+title: Pre-study
+---
+
+In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic. {% include pullquote.html quote="It is typically placed in a larger or distinctive typeface and on the same page." %} Pull quotes are often used in magazine and newspaper articles, annual reports, and brochures, as well as on the web. They can add visual interest to text-heavy pages with few images or illustrations.
+
+Placement of a pull quote on a page may be defined in a publication's or website's style guide. Such a typographic device may or may not be aligned with a column on the page. Some designers, for example, choose not to align the quote. In that case, the quotation cuts into two or more columns, as in the example shown. Because the pull quote invites the reader to read about the highlighted material, the pull quote should appear before the text it cites and, generally, fairly close to it.
+
+Pull quotes need not be a verbatim copy of the text being quoted; depending on a publication's house style, pull quotes may be abbreviated for space or paraphrased for clarity, with or without indication.
+
+A disadvantage of pull quotes as a design element is that they can disrupt the reading process of readers invested in reading the text sequentially by drawing attention to ghost fragments out of context. At the other extreme, when pull quotes are used to break up what would otherwise be a formless wall of text, pull quote can serve as visual landmarks to help the reader maintain a sense of sequence and place.

From ad8fc6deb1d68137f75214cedc5aa1fa87a4ef9e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 18:41:37 +0900
Subject: [PATCH 050/638] Rename 2023-12-30-pre-study to
 2023-12-30-pre-study.md

---
 _posts/{2023-12-30-pre-study => 2023-12-30-pre-study.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2023-12-30-pre-study => 2023-12-30-pre-study.md} (100%)

diff --git a/_posts/2023-12-30-pre-study b/_posts/2023-12-30-pre-study.md
similarity index 100%
rename from _posts/2023-12-30-pre-study
rename to _posts/2023-12-30-pre-study.md

From 463dcae433ab8e9d7076a5556574359c3e9c6bbb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:01:54 +0900
Subject: [PATCH 051/638] Update 2023-12-30-pre-study.md

---
 _posts/2023-12-30-pre-study.md | 53 ++++++++++++++++++++++++++++++----
 1 file changed, 48 insertions(+), 5 deletions(-)

diff --git a/_posts/2023-12-30-pre-study.md b/_posts/2023-12-30-pre-study.md
index 18c8c2836..e87837927 100644
--- a/_posts/2023-12-30-pre-study.md
+++ b/_posts/2023-12-30-pre-study.md
@@ -1,12 +1,55 @@
 ---
 layout: post
-title: Pre-study
+title: [Pre-study] Some basics before starting my journey
+categories: statistics
 ---
 
-In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic. {% include pullquote.html quote="It is typically placed in a larger or distinctive typeface and on the same page." %} Pull quotes are often used in magazine and newspaper articles, annual reports, and brochures, as well as on the web. They can add visual interest to text-heavy pages with few images or illustrations.
+Today, I learned about JASP because I was going to find an alternative program for SPSS. I thought it was a useful program, so I tried to give a turning ear and technical statistics, but it was fun. But before I knew more about JASP, I thought, 'It would be good to review my basic statistical knowledge'.
 
-Placement of a pull quote on a page may be defined in a publication's or website's style guide. Such a typographic device may or may not be aligned with a column on the page. Some designers, for example, choose not to align the quote. In that case, the quotation cuts into two or more columns, as in the example shown. Because the pull quote invites the reader to read about the highlighted material, the pull quote should appear before the text it cites and, generally, fairly close to it.
+Fortunately, there is a free Introduction to Statistics course at Coursera by Professor Guenther Walther of Stanford University.
 
-Pull quotes need not be a verbatim copy of the text being quoted; depending on a publication's house style, pull quotes may be abbreviated for space or paraphrased for clarity, with or without indication.
+*Homoscedasticity*:
 
-A disadvantage of pull quotes as a design element is that they can disrupt the reading process of readers invested in reading the text sequentially by drawing attention to ghost fragments out of context. At the other extreme, when pull quotes are used to break up what would otherwise be a formless wall of text, pull quote can serve as visual landmarks to help the reader maintain a sense of sequence and place.
+In a homoscedastic dataset, the variance of the errors (residuals) is constant across all levels of the independent variable(s). In simpler terms, the spread of the residuals is the same throughout the range of predictor values.
+
+*Heteroscedasticity*:
+Heteroscedasticity occurs when the variance of the errors is not constant across all levels of the independent variable(s). In other words, the spread of residuals changes as the values of the independent variable(s) change.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f17e6eec-c8e0-4c4b-9dee-b716e2f75f56)
+
+The lecture material was as follows:
+
+1. Introduction and Descriptive Statistics for Exploring Data
+2. Producing Data and Sampling
+    * Exploring sampling concepts and experimental design
+    * Topical content such as simple random sampling, bias, observation-to-experiment, and randomized controlled experiments
+    * Contents of patent applications with special industrial insights
+3. Probability
+    * Defining probabilities and learning essential probability rules to solve simple and complex challenges
+    * Topical content such as supplementation, same possible results, addition and multiplication
+4. Normal Approximation and Binomial Distribution
+    * Empirical rules for data and modules for normal approximation
+    * Basic content of normal curves, data standardization, binomial distributions, and random variables
+5. Sampling Distributions and the Central Limit Theorem
+    * Learning about the central limit theorem
+    * Topical content such as differences between parameters and statistics, expected values, and standard errors
+6. Regression
+    * Introduction to regression, correlation coefficients, regression lines, and modules for regression diagnosis
+    * predictions, mean in regression, outliers
+7. Confidence Intervals
+    * Module for creating and interpreting confidence intervals in standard situations
+    * Content topics such as finding confidence intervals using the central limit theorem and estimating standard errors using the bootstrap principle
+8. Tests of Significance
+    * Modules on hypothesis test logic and performing appropriate statistical tests for various samples and situations
+    * Topical content such as p-values, t-tests, and statistical significance versus significance for hypotheses
+9. Resampling
+    * Modules focused on Monte Carlo methods and bootstrap methods used in computer-intensive statistical inference
+    * Learning about the theoretical principles behind these methods and their applications in different contexts, such as regression and confidence interval generation
+10. Analysis of Categorical Data
+    * Modules focused on three important statistical analyses of categorical data
+    * Chi-square test content that tests the relationship between two categorical variables, the color ratio and homogeneity of M&M, and independence
+11. One-Way Analysis of Variance (ANOVA)
+    * Modules for F-tests in the Basic and One-way ANOVA Examples of ANOVA
+    * Contents such as comparing multiple means, using F distribution for ANOVA evaluation, etc
+12. Multiple Comparisons
+    * Modules focused on data snooping and multi-test misgivings in the era of big data

From e6f2710b99255ade0080d79b16f6c11645784813 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:02:51 +0900
Subject: [PATCH 052/638] Update 2023-12-30-pre-study.md

---
 _posts/2023-12-30-pre-study.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2023-12-30-pre-study.md b/_posts/2023-12-30-pre-study.md
index e87837927..ca8c07934 100644
--- a/_posts/2023-12-30-pre-study.md
+++ b/_posts/2023-12-30-pre-study.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: [Pre-study] Some basics before starting my journey
+title: (Pre-study) Some basics before starting my journey
 categories: statistics
 ---
 

From 7943bbc1063ba2c15731ce6e8356c04c069ee479 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:16:12 +0900
Subject: [PATCH 053/638] Create 2024-1-2-day-1.md

---
 _posts/2024-1-2-day-1.md | 66 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 _posts/2024-1-2-day-1.md

diff --git a/_posts/2024-1-2-day-1.md b/_posts/2024-1-2-day-1.md
new file mode 100644
index 000000000..9ee082a53
--- /dev/null
+++ b/_posts/2024-1-2-day-1.md
@@ -0,0 +1,66 @@
+---
+layout: post
+title: (Day 1) Machine learning course by Caltech
+categories: machine-learning
+---
+
+## Hello :)
+
+### Today is Day 1! My adventure begins!
+
+Today, I decided to review some machine learning knowledge, and I watched a machine learning lecture conducted by [Professor Yaser Abu-Mostafa](https://youtu.be/mbyG85GZ0PI?list=PLD63A284B7615313A) of the California Institute of Technology. The course includes 18 lectures, and the professor is Yaser Abu-Mostafa. The lectures were originally held in 2012, but I think it will be helpful to learn basic knowledge and basic concepts of machine learning. I randomly came across them, and I liked the professor from the first lecture, so I kept watching. The lectures are very interesting, so I studied all the lectures from No. 1 to No. 11 today and I think I will finish it tomorrow.
+
+Each lecture was filled with different math functions and math explanations, and the lecturer explained it all very well, step by step, and looking back at it, it seems complicated in the end, but during the lecture the explanations were very clear.
+
+The content was like below:
+
+1. The learning problem
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2f4cd129-9ad9-45e3-9566-cdc7e607235e)
+
+2. Can we learn?
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ccd61c69-7e7f-40d1-a327-bd2949a578df)
+
+3. Regression model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5deb5594-056a-4367-9461-10157c4f7182)
+
+4. Error and noise
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/46d227da-8fb2-48d4-abee-2d7c21774930)
+
+5. Training vs testing
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/01250297-f863-4705-a381-aa27de575242)
+
+6. Generalization theory
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3bc782e0-6a8b-48c7-9d87-07a096d7649b)
+
+7. The VC dimension
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/dc959ed8-0c32-43da-824a-eda94db24f5a)
+
+8. Regression model (2)
+
+`If you look at linear models, you can think of them as an economic car. They get you where you want to go and they dont consume a lot of gas. You may not be very proud of them, but they actually do the job. It is remarkable how often they succeed on their own and they are sufficient to get you the learning performance that you want. Try to use them first and see if they achieve what you want.` - an interesting quote from the professor. As I am learning, it might be useful to always at the start, just use a simple linear model for whatever I am doing, just to see its performance.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/33379d3b-a7df-4a7a-aeb4-89d28e3dc43b)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/7990bc1a-5a44-4f90-a3c8-31363936f642)
+
+10. Neural networks
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f42821ca-4c01-413a-bf18-56196f074c37)
+
+11. Overfitting
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3af02a03-f722-41d4-be4a-462ca63ff57b)
+
+I'm really looking forward to studying for the remaining lectures! The picturea above are included in the lecture videos, but rather than doing that, it would be good to take notes to post something like this.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-1-machine-learning-course-by-caltech.html)

From 24fe0ef3a36c8cb4165c05aa50aff01f3bb4b1b7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:16:40 +0900
Subject: [PATCH 054/638] Update 2023-12-30-pre-study.md

---
 _posts/2023-12-30-pre-study.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_posts/2023-12-30-pre-study.md b/_posts/2023-12-30-pre-study.md
index ca8c07934..9ecef5e1c 100644
--- a/_posts/2023-12-30-pre-study.md
+++ b/_posts/2023-12-30-pre-study.md
@@ -53,3 +53,5 @@ The lecture material was as follows:
     * Contents such as comparing multiple means, using F distribution for ANOVA evaluation, etc
 12. Multiple Comparisons
     * Modules focused on data snooping and multi-test misgivings in the era of big data
+
+[Original post in Korean](https://50daysml.blogspot.com/2023/12/day-1-12.html)

From 0ff5a7a3b027462747eef7fef44abc7b09801b85 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:18:59 +0900
Subject: [PATCH 055/638] Update 2024-1-2-day-1.md

---
 _posts/2024-1-2-day-1.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/_posts/2024-1-2-day-1.md b/_posts/2024-1-2-day-1.md
index 9ee082a53..d72f64bcc 100644
--- a/_posts/2024-1-2-day-1.md
+++ b/_posts/2024-1-2-day-1.md
@@ -4,10 +4,7 @@ title: (Day 1) Machine learning course by Caltech
 categories: machine-learning
 ---
 
-## Hello :)
-
-### Today is Day 1! My adventure begins!
-
+## Hello :) Today is Day 1! My adventure begins!
 Today, I decided to review some machine learning knowledge, and I watched a machine learning lecture conducted by [Professor Yaser Abu-Mostafa](https://youtu.be/mbyG85GZ0PI?list=PLD63A284B7615313A) of the California Institute of Technology. The course includes 18 lectures, and the professor is Yaser Abu-Mostafa. The lectures were originally held in 2012, but I think it will be helpful to learn basic knowledge and basic concepts of machine learning. I randomly came across them, and I liked the professor from the first lecture, so I kept watching. The lectures are very interesting, so I studied all the lectures from No. 1 to No. 11 today and I think I will finish it tomorrow.
 
 Each lecture was filled with different math functions and math explanations, and the lecturer explained it all very well, step by step, and looking back at it, it seems complicated in the end, but during the lecture the explanations were very clear.

From b93d41ad5776b4adec721defad224b87cab4e2d0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:27:59 +0900
Subject: [PATCH 056/638] Create 2024-1-3-day-2.md

---
 _posts/2024-1-3-day-2.md | 51 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 _posts/2024-1-3-day-2.md

diff --git a/_posts/2024-1-3-day-2.md b/_posts/2024-1-3-day-2.md
new file mode 100644
index 000000000..d51417779
--- /dev/null
+++ b/_posts/2024-1-3-day-2.md
@@ -0,0 +1,51 @@
+---
+layout: post
+title: (Day 2) Finishing the ML course by Calctech
+categories: [machine-learning, statistics]
+---
+
+## Hello :) Today is Day 2! 
+A quick summary of today:
+* Finished [Professor Yaser Abu-Mustafa's ML course](https://youtu.be/mbyG85GZ0PI?list=PLD63A284B7615313A)
+* Covered [Master statistics with Python in Codecademy](https://www.codecademy.com/learn/paths/master-statistics-with-python)
+
+**Firstly, below are some memos from Professor Abu-Mostafa's lectures**
+
+* Lecture 12 - Regularization
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/949d7c43-8345-4f4e-afdf-fea3247957bd)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f837907f-943c-470d-9d6a-e1a99dc53baa)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b32b507b-d89f-4485-83e5-67fb6dcabf11)
+
+* Lecture 13 - Validation
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/dc6ace62-4230-49bb-9671-a6cfe13c794a)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ee5d46e6-0808-4aea-b5f9-afe76c464e77)
+
+* Lecture 14 - SVM
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/889a5e96-281c-4362-acd0-cba7a2a4ae47)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8b4d27e6-df41-40eb-909f-54c0d9295124)
+
+* Lecture 15 - Kernel methods
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9f6d7639-5028-4ce7-9803-3171922cc1ad)
+
+* Lecture 16 - Radial basis functions
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/863aa45b-e056-49d5-a5f8-55b58033bdb9)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/33535ab6-3503-4aac-8796-16fef4d8fe36)
+
+* Lecture 17 - Three learning principles  
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cab38c65-bffe-4f67-b13d-c47dcd1d52dc)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9f9ff29e-5ec5-453a-887c-234efa25d895)
+
+**Secondly, regarding the Master statistics with Python**
+I wanted to learn a bit more about test statistics, and I got what I wanted
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cf621156-6e0a-448e-9adc-d0e7c129564c)
+
+Also, using chatGPT I got this nice summary
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b329ab53-168a-4cbf-b34c-21863a302588)
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-2-machine-learning-course-by-caltech.html)

From 5db0c63b53a7159beaa89fe51a272b1203db51ff Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:39:31 +0900
Subject: [PATCH 057/638] Create 2024-1-4-day-3.md

---
 _posts/2024-1-4-day-3.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 _posts/2024-1-4-day-3.md

diff --git a/_posts/2024-1-4-day-3.md b/_posts/2024-1-4-day-3.md
new file mode 100644
index 000000000..ce4438e48
--- /dev/null
+++ b/_posts/2024-1-4-day-3.md
@@ -0,0 +1,24 @@
+---
+layout: post
+title: (Day 3) Intro to time series
+categories: [machine-learning, time-series]
+---
+
+## Hello :) Today is Day 3!
+A quick summary of today:
+* learning about time series predictions
+* Beginner ML course by [Microsoft](https://github.com/microsoft/ML-For-Beginners)
+
+Time series prediction is a technique for predicting future values based on given historical data, and it is used in various applications, and is useful in various fields such as finance, production, and health.
+
+To predict time series, [Prophet](https://medium.com/illumination/understanding-fb-prophet-a-time-series-forecasting-algorithm-c998bc52ca10), developed by Facebook, is simple to use and provides strong performance.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/70818273-ed7b-4ded-8936-300a3ad3b2f1)
+
+I also found Microsoft's ML-for-beginners, and the content includes pre-electrical quiz-lecture-post-factual assistance. I couldn't find the time today that much, so I can only do 1-Introduction, but I will try to do it every week from now on.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-3.html)

From 964de12adb3a5c524ff814a0b687717efff5bf23 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:49:20 +0900
Subject: [PATCH 058/638] Create 2024-1-5-day-4.md

---
 _posts/2024-1-5-day-4.md | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 _posts/2024-1-5-day-4.md

diff --git a/_posts/2024-1-5-day-4.md b/_posts/2024-1-5-day-4.md
new file mode 100644
index 000000000..ab49d675f
--- /dev/null
+++ b/_posts/2024-1-5-day-4.md
@@ -0,0 +1,36 @@
+---
+layout: post
+title: (Day 4) Looking for a ML book and Microsoft's ML-For-Beginners course
+categories: [machine-learning, regression]
+---
+
+## Hello :) Today is Day 4!
+A quick summary of today:
+* [Financial Risk Management Using Python Machine Learning](https://book.interpark.com/product/BookDisplay.do?_method=detail&sc.shopNo=0000400000&sc.prdNo=356709401)
+* Microsoft's ML-For-Beginners lecture 2 - [regression](https://github.com/divakaivan/ML-For-Beginners/tree/main/2-Regression)
+
+**Firstly**, when I looked at the table of contents of the textbook, I liked it, but when it arrived today, it was not and I applied for a refund. In the past, I had used [Python deep learning TensorFlow](https://product.kyobobook.co.kr/detail/S000000833270) and [Python deep learning machine learning introductory](https://product.kyobobook.co.kr/detail/S000000833261), so I believe that I can quickly learn through practical tasks. I will keep looking for books. 
+
+**Secondly**, about Microsoft's regression lecture
+
+The content follows
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0ea71f6a-a1d6-44e1-9c19-c3e1fb4354a8)
+
+1. Introduced me to scikit-learn and jupyter notebooks, and
+2. Making graphs with matplotlib
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5453ad7e-b18c-4018-9417-4c65be845ace)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b21bc463-6a93-41eb-be45-022f2d56039e)
+
+3. Linear and Polynomial regression
+4. Intro to Logistic regression and how it is different from linear
+    * Linear Regression: Used when modeling the linear relationship between a dependent variable and one or more independent variables.
+    * Logistic Regression: Used for binary classification problems, where the goal is to classify instances into one of two classes.
+  
+Tomorrow ~ I will study lecture 3 of ML-For-Beginners and try to find interesting data or notebook in Kaggle and practice on my own.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-4-microsoft-ml-for-beginners.html)

From 6f54bda77a14645307b18eb15747c740fcd1e033 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 19:51:20 +0900
Subject: [PATCH 059/638] Create 2024-1-6-day-5.md

---
 _posts/2024-1-6-day-5.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 _posts/2024-1-6-day-5.md

diff --git a/_posts/2024-1-6-day-5.md b/_posts/2024-1-6-day-5.md
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/_posts/2024-1-6-day-5.md
@@ -0,0 +1 @@
+

From d09dfd9144eb6362fc6f7b1008bf8fe2585c24e7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:03:03 +0900
Subject: [PATCH 060/638] Update 2024-1-6-day-5.md

---
 _posts/2024-1-6-day-5.md | 77 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/_posts/2024-1-6-day-5.md b/_posts/2024-1-6-day-5.md
index 8b1378917..1c5c9b593 100644
--- a/_posts/2024-1-6-day-5.md
+++ b/_posts/2024-1-6-day-5.md
@@ -1 +1,78 @@
+---
+layout: post
+title: (Day 5) Kaggle's 'Learn' courses
+categories: [machine-learning]
+---
 
+## Hello :) Today is Day 5!
+A quick summary of today:
+* found extremely helpful courses for ML in Kaggle
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0ea13b70-d32f-4737-9707-f9a2c77139bd)
+
+The content of the above courses is as below
+
+**Intro to Machine Learning**
+* model creating, training and predictions
+* model validation
+* model selection - under and overfitting
+* beginner RandomForest model creation
+
+**Intermediate Machine Learning**
+* dealing with missing data
+    * deleting the column, or imputation
+* handling categorical data
+    * deleting the column, ordinal encoding, one-hot encoding
+* pipelines
+    * define preprocessor
+      ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1647ca59-f368-45e2-9bde-5cc0acf22fee)
+
+    * define model
+ ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6126bf51-35a3-41ae-b7ba-48742efb6c3c)
+
+    * create pipeline
+      ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1eb0821b-b2c7-4a01-9d0e-c17744a866e0)
+
+* cross-validation
+* XGBoost
+* Data leakage (target leakage and train-test contamination)
+
+**Data visualization**
+
+Using seaborn
+* line plot, bar graphs, heat maps, scatterplot
+
+**Feature engineering**
+
+* Mutual information - it is a lot like correlation in that it measures a relationship between two quantities. The advantage of mutual information is that it can detect any kind of relationship, while correlation only detects linear relationships. Features should be integers
+
+  ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f7315274-4f55-4d52-9989-b52e3625159c)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e267b451-f1ff-479c-b08a-4d0b0ec61455)
+
+* Clistering with K-means
+* PCA - its primary goal is to reduce the number of features (or dimensions) in a dataset while preserving as much of the original variability as possible (need to look more into this)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8a042c9c-f385-4bab-a5b7-24220a62325c)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/49d53303-4084-4b65-bc3e-517da14e73a9)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b917ce5a-c377-4b9c-b7b7-4f715b79025c)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c7fe6f85-86dc-4c16-92cc-53757b61de5c)
+
+* Target encoding with MEstimateEncoder
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/732635cb-9f21-4122-9eae-1071e642d1ca)
+
+Above, instead of mean encoding, smoothing would be better. The idea is to blend the in-category average with the overall average. Rare categories get less weight on their category average, while missing categories just get the overall average.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/33da4baf-0fff-4eb7-8d0f-da0edc0eb163)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d6101a7f-1956-4e8c-928c-4cea85c7a95c)
+
+When choosing a value for m, consider how noisy you expect the categories to be.
+Use Cases for Target Encoding:
+* High-cardinality features: A feature with a large number of categories can be troublesome to encode: a one-hot encoding would generate too many features and alternatives, like a label encoding, might not be appropriate for that feature. A target encoding derives numbers for the categories using the feature's most important property: its relationship with the target.
+* Domain-motivated features: From prior experience, you might suspect that a categorical feature should be important even if it scored poorly with a feature metric. A target encoding can help reveal a feature's true informativeness.
+
+Today I learned about PCA, pipelines and target encoding, but I have a feeling I need to look into them a bit more. 
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-5-kaggle-learn-courses.html)

From 84cb6c829fc4ab9a5801049e4e7bb846cd41c2c5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:05:08 +0900
Subject: [PATCH 061/638] Update 2024-1-6-day-5.md

---
 _posts/2024-1-6-day-5.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-6-day-5.md b/_posts/2024-1-6-day-5.md
index 1c5c9b593..dc726c650 100644
--- a/_posts/2024-1-6-day-5.md
+++ b/_posts/2024-1-6-day-5.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 5) Kaggle's 'Learn' courses
-categories: [machine-learning]
+categories: [traditional-machine-learning]
 ---
 
 ## Hello :) Today is Day 5!

From d2216921281092e3f1dfbf948ec78227562dbe6b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:05:20 +0900
Subject: [PATCH 062/638] Update 2024-1-5-day-4.md

---
 _posts/2024-1-5-day-4.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-5-day-4.md b/_posts/2024-1-5-day-4.md
index ab49d675f..7ee9d8d68 100644
--- a/_posts/2024-1-5-day-4.md
+++ b/_posts/2024-1-5-day-4.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 4) Looking for a ML book and Microsoft's ML-For-Beginners course
-categories: [machine-learning, regression]
+categories: [traditional-machine-learning, regression]
 ---
 
 ## Hello :) Today is Day 4!

From 432772da1709bc787eb79041a6d60e5d14a9c86a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:05:33 +0900
Subject: [PATCH 063/638] Update 2024-1-4-day-3.md

---
 _posts/2024-1-4-day-3.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-4-day-3.md b/_posts/2024-1-4-day-3.md
index ce4438e48..679da2a05 100644
--- a/_posts/2024-1-4-day-3.md
+++ b/_posts/2024-1-4-day-3.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 3) Intro to time series
-categories: [machine-learning, time-series]
+categories: [traditional-machine-learning, time-series]
 ---
 
 ## Hello :) Today is Day 3!

From dae855099a995366dba5d1ca4e767d1b2f819d54 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:05:46 +0900
Subject: [PATCH 064/638] Update 2024-1-3-day-2.md

---
 _posts/2024-1-3-day-2.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-3-day-2.md b/_posts/2024-1-3-day-2.md
index d51417779..49c0ef06a 100644
--- a/_posts/2024-1-3-day-2.md
+++ b/_posts/2024-1-3-day-2.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 2) Finishing the ML course by Calctech
-categories: [machine-learning, statistics]
+categories: [traditional-machine-learning, statistics]
 ---
 
 ## Hello :) Today is Day 2! 

From 4394cdbb2db63963f4e82bb80a20793c4900562b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:06:00 +0900
Subject: [PATCH 065/638] Update 2024-1-2-day-1.md

---
 _posts/2024-1-2-day-1.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-2-day-1.md b/_posts/2024-1-2-day-1.md
index d72f64bcc..06f66f544 100644
--- a/_posts/2024-1-2-day-1.md
+++ b/_posts/2024-1-2-day-1.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 1) Machine learning course by Caltech
-categories: machine-learning
+categories: traditional-machine-learning
 ---
 
 ## Hello :) Today is Day 1! My adventure begins!

From c214f0eba695294f55599bac7cc295a0dcc9800a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:08:17 +0900
Subject: [PATCH 066/638] Create 2024-1-7-day-6.md

---
 _posts/2024-1-7-day-6.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 _posts/2024-1-7-day-6.md

diff --git a/_posts/2024-1-7-day-6.md b/_posts/2024-1-7-day-6.md
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/_posts/2024-1-7-day-6.md
@@ -0,0 +1 @@
+

From 3b41699f5f71170c87b3f8cd9bee3ce0f4b43ce9 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:18:50 +0900
Subject: [PATCH 067/638] Update 2024-1-7-day-6.md

---
 _posts/2024-1-7-day-6.md | 54 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/_posts/2024-1-7-day-6.md b/_posts/2024-1-7-day-6.md
index 8b1378917..7d15e9bfc 100644
--- a/_posts/2024-1-7-day-6.md
+++ b/_posts/2024-1-7-day-6.md
@@ -1 +1,55 @@
+---
+layout: post
+title: (Day 6) Time series course - Kaggle
+categories: [time-series]
+---
 
+## Hello :) Today is Day 6!
+A quick summary of today:
+* covered Kaggle's [intro to time series course](https://www.kaggle.com/learn/time-series)
+
+It was a bit hard, but interesting.
+
+**Linear regression with time series**
+* Time-step features
+  *  It can be extracted directly from the time axis and the characteristics of year, month, day, and time measured at regular time intervals from start to end are typical. This is a useful characteristic when the observed values have periodic properties
+*  Lag features
+  * Occurs as a time difference in the observed values, and the current observed values are expressed as previous observed values. This is a useful characteristic when the observed values have autocorrelation or series correlation
+
+**Trends**
+* It refers to a pattern that increases or decreases uniformly over a long section as a whole
+* once we've identified the shape of the trend, we can attempt to model it using a time-step feature
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1caf4f47-3707-470d-9c5f-bfedba3dad2e)
+
+**Sasonality**
+* Seasonality is one of the representative factors that occur in time step characteristics, where seasonality refers to periodic fluctuations, and time series data indicate seasonality when fluctuations in observations are repeatedly observed on a daily, weekly, monthly, and yearly basis
+
+  ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/05f016e2-184e-4c9d-84bb-7a9daed185e4)
+
+* I got introduced to Fourier features here
+  * Used to capture and model periodic patterns in time series data
+  * They are features based on amplitudes and phases of sine and cosine functions with different frequencies
+  * Commonly used for tasks like forecasting to enhance the model's ability to handle cyclic variations.
+  * They enable models to understand and predict periodic structures in the data, enhancing overall predictive performance in tasks with recurring patterns
+ 
+* Periodogram - to know which Fourier feature to use, we can inspect the periodogram
+  ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/90cb00e7-022d-492e-86fd-bdd85c10dfdc)
+
+In this case, we can see annual, quarterly and weekly cycles occur
+
+**Hybrid Forecasting**
+* Linear regression can reveal trends, but complex interactions cannot be easily revealed. On the other hand, XGBoost can reveal complex interactions, but cannot reveal trends
+* "We could imagine learning the components of a time series as an iterative process: first learn the trend and subtract it out from the series, then learn the seasonality from the detrended residuals and subtract the seasons out, then learn the cycles and subtract the cycles out, and finally only the unpredictable error remains."
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6863430f-5837-4bac-8d6e-a4afc4fe8bcb)
+
+* Using linear regression and XGBoost together, will help our model's predictions to be better
+
+I did only 1 lecture today, but the content was a bit difficult. Still it was good and I hope I can actually apply the above methods in practice soon.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-6-time-series-course-kaggle.html)
+  

From b06d9baca902c378104b44deff0fec6af5820cec Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:37:31 +0900
Subject: [PATCH 068/638] Create 2024-1-8-day-7.md

---
 _posts/2024-1-8-day-7.md | 67 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 _posts/2024-1-8-day-7.md

diff --git a/_posts/2024-1-8-day-7.md b/_posts/2024-1-8-day-7.md
new file mode 100644
index 000000000..53ab7a374
--- /dev/null
+++ b/_posts/2024-1-8-day-7.md
@@ -0,0 +1,67 @@
+---
+layout: post
+title: (Day 7) Applying the time series knowledge to practice
+categories: [time-series, applying-knowledge]
+---
+
+## Hello :) Today is Day 7!
+A quick summary of today:
+* [WON/USD exchange rate prediction model](https://www.kaggle.com/code/divakaivan12/basic-linear-regression-to-predict-kr-won-to-usd)
+* [Seoul Bike Rental Demand - Linear Regression Model and Seasonal Features](https://www.kaggle.com/code/divakaivan12/beginner-linear-regression-seasonality-features)
+
+Following yesterday's Kaggle time series course, I decided to apply what I learned.
+The links go to the Kaggle notebooks for each model. Below I will try to provide an overview of both.
+
+### WON/USD exchange rate prediction model
+
+**Data**
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c27c6d69-c5a2-4dc2-b4af-3c24ee0424eb)
+
+I used simple interpolation to deal with missing data points
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0c8fe8d4-6ee9-4d3c-8968-f52848e39d25)
+
+**Feature engineering**
+lag_1 - made 1 day behind lag feature
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/37e77d37-0af4-453b-9261-8e94624692e5)
+
+**Model evaluation**
+R-squared: 0.9941
+Mean Squared Error: 63.3640
+Mean Absolute Error: 4.6090
+
+Homoscedasticity check
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ed494892-fc77-46c8-b6e8-ceaaf48de475)
+
+The model results seem good, but I know that the used features are not that good because I only use lag_1 (I just applied what I learned yesterday)
+
+### Seoul Bike Rental Demand
+Looked to do time series forecast using Seoul Bicycle Demand Data, which includes daily and hourly information and weather information on bicycles rented in Seoul. The goal was to construct a basic linear regression model that integrates seasonal features and holiday information.
+
+**Some data viz**
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6759b2a3-dc12-4ac8-a00b-341571d6467f)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5339e89c-b8ec-4304-8864-1c6cbee69e27)
+
+**Checked for a trend (seems like there is not)**
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c2032fa9-3cf0-4c89-beef-1fb04b82150a)
+
+**Seasonality check**
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/31cd6ef4-f77a-455b-93ea-c7b94d025edd)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/00d716f2-bffc-48f8-85e2-001e561e9b03)
+
+Seems there is annual, and no daily.
+
+After adding seasonality features, the model results:
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3c7c139e-7764-4a8d-a073-7f2810351155)
+
+Also made a 2nd model, that includes holiday data:
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cb6e7cee-9cdf-4471-9b55-6083ad1f3b92)
+
+**Results**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3569f17a-d612-4ac6-aa3e-8b8b7b57d0af)
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-7.html)

From 05e5b4f6c1a47419d0772b81c0cdae6d55eb5113 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:46:33 +0900
Subject: [PATCH 069/638] Create 2024-1-9-day-8.md

---
 _posts/2024-1-9-day-8.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 _posts/2024-1-9-day-8.md

diff --git a/_posts/2024-1-9-day-8.md b/_posts/2024-1-9-day-8.md
new file mode 100644
index 000000000..4b6f34455
--- /dev/null
+++ b/_posts/2024-1-9-day-8.md
@@ -0,0 +1,31 @@
+---
+layout: post
+title: (Day 8) Time series - finding trends
+categories: [time-series, applying-knowledge]
+---
+
+## Hello :) Today is Day 8!
+A quick summary of today:
+* [Basic trend model for Provision prices in Nepal](https://www.kaggle.com/code/divakaivan12/basic-trend-model-for-provision-prices-in-nepal)
+
+**Data**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8facf26e-81ed-44e6-8b65-e76dbbfa9258)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/acc2c812-6329-4dcf-b8f6-440a9cfe8473)
+
+Moving average
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ddf3ac44-1dd1-4d70-9ee7-ba517587039d)
+
+**Trend model, and forecast**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/fa8265bc-981a-4b31-a78d-abab29220ad1)
+
+Today's practice was not that extraordinary, but it helped to learn about making a trend forecast model.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-8-trend.html)

From ade480e2687f43821a4cb622d6c6ce3e055fb38d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:49:17 +0900
Subject: [PATCH 070/638] Create 2024-1-10-day-9.md

---
 _posts/2024-1-10-day-9.md | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 _posts/2024-1-10-day-9.md

diff --git a/_posts/2024-1-10-day-9.md b/_posts/2024-1-10-day-9.md
new file mode 100644
index 000000000..cf9e311e7
--- /dev/null
+++ b/_posts/2024-1-10-day-9.md
@@ -0,0 +1,8 @@
+---
+layout: post
+title: (Day 9) Simple linear regression to predict GCPA
+categories: [applying-knowledge]
+---
+
+## Hello :) Today is Day 9!
+A quick summary of today:

From f26f358cecbe76890e3286de73302ab204652c99 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:49:49 +0900
Subject: [PATCH 071/638] Update 2024-1-5-day-4.md

---
 _posts/2024-1-5-day-4.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-5-day-4.md b/_posts/2024-1-5-day-4.md
index 7ee9d8d68..0736c20fd 100644
--- a/_posts/2024-1-5-day-4.md
+++ b/_posts/2024-1-5-day-4.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 4) Looking for a ML book and Microsoft's ML-For-Beginners course
-categories: [traditional-machine-learning, regression]
+categories: [traditional-machine-learning]
 ---
 
 ## Hello :) Today is Day 4!

From 48976f7f4875a8268f3cb4dd1637ae77c49cd4e7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:57:02 +0900
Subject: [PATCH 072/638] Update 2024-1-10-day-9.md

---
 _posts/2024-1-10-day-9.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/_posts/2024-1-10-day-9.md b/_posts/2024-1-10-day-9.md
index cf9e311e7..b666f539e 100644
--- a/_posts/2024-1-10-day-9.md
+++ b/_posts/2024-1-10-day-9.md
@@ -6,3 +6,34 @@ categories: [applying-knowledge]
 
 ## Hello :) Today is Day 9!
 A quick summary of today:
+* Linear regression to predict student GCPA on [Kaggle](https://www.kaggle.com/code/divakaivan12/linear-regression-to-predict-student-cgpa?scriptVersionId=158434364)
+
+Today I tried to make a simple linear regression model.
+
+**Data**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/bff54d82-c16e-4e57-944a-bee4a8a2a68f)
+
+CGPA, IQ, Placement (whether the student got into college)
+
+**Some viz**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3541451d-3cf0-4fc2-b892-0b27a5d5d60b)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9521d188-da78-497c-a2d3-783af7cd87db)
+
+CGPA high => placement=1 degree of certainty is high too. IQ doesn't seem to matter
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/7632750d-6db9-46e7-8487-ae5e7b6ed2e3)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c05352a2-cb82-4c9c-9c85-c9854eb55fab)
+
+For fun, I selected IQ and placement as the features, and target is CGPA. 
+
+**Results**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0db86459-28bb-42ad-bd35-b3b1707b0fc2)
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-9-cgpa.html)

From 42dabe429d7eb60e7c8a5bb04ea0e760f3f34305 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 20:58:14 +0900
Subject: [PATCH 073/638] Update 2024-1-10-day-9.md

---
 _posts/2024-1-10-day-9.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_posts/2024-1-10-day-9.md b/_posts/2024-1-10-day-9.md
index b666f539e..a8b6f8154 100644
--- a/_posts/2024-1-10-day-9.md
+++ b/_posts/2024-1-10-day-9.md
@@ -32,6 +32,8 @@ For fun, I selected IQ and placement as the features, and target is CGPA.
 
 ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0db86459-28bb-42ad-bd35-b3b1707b0fc2)
 
+As expected IQ does not seem to matter. 
+
 That is all for today!
 
 See you tomorrow :)

From 2a23af805526e17fbd6404b41eb6ee9bd76f1dc2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:06:50 +0900
Subject: [PATCH 074/638] Create 2024-1-11-day-10.md

---
 _posts/2024-1-11-day-10.md | 56 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)
 create mode 100644 _posts/2024-1-11-day-10.md

diff --git a/_posts/2024-1-11-day-10.md b/_posts/2024-1-11-day-10.md
new file mode 100644
index 000000000..f282cc0bd
--- /dev/null
+++ b/_posts/2024-1-11-day-10.md
@@ -0,0 +1,56 @@
+---
+layout: post
+title: (Day 10) Simple linear regression to predict GCPA
+categories: [applying-knowledge]
+---
+
+## Hello :) Today is Day 10!
+A quick summary of today:
+* House price basic prediction model comparison on [Kaggle](https://www.kaggle.com/code/divakaivan12/house-price-basic-prediction-model-comparison)
+
+**Data**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9a73bcdd-212c-4ce6-9fe1-ce24549741af)
+
+Majority was categorical data, so I used `pd.get_dummies()` to prepare them.
+
+**Some viz**
+
+* Area and price correlation 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6580ed06-ac3d-4b15-9f3b-90e147c9c0a3)
+
+* Outliers
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/16b9cdd4-23b3-43d6-9600-5e5783800e69)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c2b194ae-dbd9-495e-903b-9365934c02cc)
+
+I used z-score or IQR to deal with them. 
+
+**Top features**
+
+I used the correlation table to find the top-10 features that correlated with price
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a4cb96af-2b4d-4968-abad-9d6a453007c2)
+
+**Model creation**
+
+I using LinearRegression, DecisionTreeRegressor, RandomForestRegressor, XGBRegressor (wanted to compare results)
+
+**Results**
+
+LinearRegression - Train MSE: 0.81978 | Test MSE: 1.37829
+
+DecisionTreeRegressor - Train MSE: 1.06803 | Test MSE: 1.64172
+
+RandomForestRegressor - Train MSE: 0.91908 | Test MSE: 1.41498
+
+XGBRegressor - Train MSE: 0.26813 | Test MSE: 1.60890
+
+Lastly I tried using some Polynomial features, and the results are:
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/201e4d49-f3e4-47d9-a172-a08d514a0dd2)
+
+I spilled boiling water on my finger, so writing this post was very hard :( 
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-10.html)

From d4f4de681a07334ae71fd5ca9f6eb745b9892596 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:25:30 +0900
Subject: [PATCH 075/638] Create 2024-1-12-day-11.md

---
 _posts/2024-1-12-day-11.md | 66 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 _posts/2024-1-12-day-11.md

diff --git a/_posts/2024-1-12-day-11.md b/_posts/2024-1-12-day-11.md
new file mode 100644
index 000000000..4e0cfc855
--- /dev/null
+++ b/_posts/2024-1-12-day-11.md
@@ -0,0 +1,66 @@
+---
+layout: post
+title: (Day 10) Joined a 'bank churn prediction' competition on Kaggle
+categories: [applying-knowledge]
+---
+
+## Hello :) Today is Day 10!
+A quick summary of today:
+* Binary Classification with a Bank Churn Dataset on [Kaggle](https://www.kaggle.com/competitions/playground-series-s4e1)
+* My [notebook](https://www.kaggle.com/code/divakaivan12/various-svm-models-for-bank-churn-classification) with various SVM models for bank churn classification
+
+**Data**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/929caab0-ae5a-410e-a9bb-9f067fbc435b)
+
+I tried various SVM models and SVM-hybrind models, and below are the results from all
+
+**Basic SVM model**
+* Accuracy: 0.8554246069015663
+* Precision: 0.7598201609086607
+* Recall: 0.4608208955223881
+* F1-Score: 0.5737001965338574
+* AUC-ROC: 0.7109204519856266
+* Confusion Matrix:
+* [[25024  1015]
+ [ 3757  3211]]
+
+**SVM and RandomForestClassifier model**
+* Accuracy: 0.7972854243039356
+* Precision: 0.5216507738002188
+* Recall: 0.4789035591274397
+* F1-Score: 0.49936401047512163
+* AUC-ROC: 0.6806937627427974
+* Confusion Matrix:
+* [[22979  3060]
+ [ 3631  3337]]
+
+**SVM and KNeighborsClassifier model**
+* Accuracy: 0.8378828733298997
+* Precision: 0.6524608712049783
+* Recall: 0.4965556831228473
+* F1-Score: 0.5639312199494744
+* AUC-ROC: 0.7128886177049008
+* Confusion Matrix:
+* [[24196  1843]
+ [ 3508  3460]]
+
+**Hyperparam tuned SVM model** 
+I did manual hyperparam tuning to see the effects of various changes, and this took me the most time, and in the end - it had the same result as the base model
+
+* Accuracy: 0.8554246069015663
+* Precision: 0.7598201609086607
+* Recall: 0.4608208955223881
+* F1-Score: 0.5737001965338574
+* AUC-ROC: 0.7109204519856266
+* Confusion Matrix:
+* [[25024  1015]
+ [ 3757  3211]]
+
+Overall, the training time for each model was very long. I guess SVM is like that.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-11-kaggle-bank-churn.html)

From ae7d227dba21cfc6b9af9df68e6ba0952024d7b4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:28:33 +0900
Subject: [PATCH 076/638] Update 2024-1-12-day-11.md

---
 _posts/2024-1-12-day-11.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-1-12-day-11.md b/_posts/2024-1-12-day-11.md
index 4e0cfc855..38b6180dd 100644
--- a/_posts/2024-1-12-day-11.md
+++ b/_posts/2024-1-12-day-11.md
@@ -1,10 +1,10 @@
 ---
 layout: post
-title: (Day 10) Joined a 'bank churn prediction' competition on Kaggle
+title: (Day 11) Joined a 'bank churn prediction' competition on Kaggle
 categories: [applying-knowledge]
 ---
 
-## Hello :) Today is Day 10!
+## Hello :) Today is Day 11!
 A quick summary of today:
 * Binary Classification with a Bank Churn Dataset on [Kaggle](https://www.kaggle.com/competitions/playground-series-s4e1)
 * My [notebook](https://www.kaggle.com/code/divakaivan12/various-svm-models-for-bank-churn-classification) with various SVM models for bank churn classification

From 8eb6694acad1ee622ad97161400199c6525dc895 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:37:09 +0900
Subject: [PATCH 077/638] Create 2024-1-13-day-12.md

---
 _posts/2024-1-13-day-12.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 _posts/2024-1-13-day-12.md

diff --git a/_posts/2024-1-13-day-12.md b/_posts/2024-1-13-day-12.md
new file mode 100644
index 000000000..7c1c0bcec
--- /dev/null
+++ b/_posts/2024-1-13-day-12.md
@@ -0,0 +1,26 @@
+---
+layout: post
+title: (Day 12) Neural network model for bank churn prediction and making a webapp for exchange rate pred
+categories: [applying-knowledge, deep-learning]
+---
+
+## Hello :) Today is Day 12!
+A quick summary of today:
+* Submitted a neural network model to the [bank churn prediction competition](https://www.kaggle.com/competitions/playground-series-s4e1)
+* Following Microsoft's tutorial on [making a simple webapp](https://github.com/divakaivan/ML-For-Beginners/tree/main/3-Web-App)
+
+The SVM model I made yesterday got the below score (not much)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e11edd8a-9af6-4a0d-b252-bed26252b172)
+
+And it got me 1100th place (at that point in time). So I decided to try to make a [NN model](https://www.kaggle.com/code/divakaivan12/nn-churn/notebook) for hopefully better results.
+
+I tried a few, and the results were way better
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b3ededca-1469-4084-b62e-8d52dfabf06a)
+
+Also ~~ on [day 7](2024-1-8-day-7.md) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-12.html)

From 34adcfcf6d91c8f2bb63f749c866f2e237a35462 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:39:06 +0900
Subject: [PATCH 078/638] Update 2024-1-13-day-12.md

---
 _posts/2024-1-13-day-12.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-13-day-12.md b/_posts/2024-1-13-day-12.md
index 7c1c0bcec..0950ab712 100644
--- a/_posts/2024-1-13-day-12.md
+++ b/_posts/2024-1-13-day-12.md
@@ -17,7 +17,7 @@ And it got me 1100th place (at that point in time). So I decided to try to make
 I tried a few, and the results were way better
 ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b3ededca-1469-4084-b62e-8d52dfabf06a)
 
-Also ~~ on [day 7](2024-1-8-day-7.md) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
+Also ~~ on [day 7](./day-7.md) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
 
 That is all for today!
 

From 4f9e27bdfc467076de122aed2af3b1e745c758c0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:39:30 +0900
Subject: [PATCH 079/638] Update 2024-1-13-day-12.md

---
 _posts/2024-1-13-day-12.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-13-day-12.md b/_posts/2024-1-13-day-12.md
index 0950ab712..f1f1be4ad 100644
--- a/_posts/2024-1-13-day-12.md
+++ b/_posts/2024-1-13-day-12.md
@@ -17,7 +17,7 @@ And it got me 1100th place (at that point in time). So I decided to try to make
 I tried a few, and the results were way better
 ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b3ededca-1469-4084-b62e-8d52dfabf06a)
 
-Also ~~ on [day 7](./day-7.md) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
+Also ~~ on [day 7](./2024-1-8-day-7.md) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
 
 That is all for today!
 

From 2f4d35ed6141a657f26f5fc0cd8255601aec8b30 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 21:49:23 +0900
Subject: [PATCH 080/638] Update 2024-1-13-day-12.md

---
 _posts/2024-1-13-day-12.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-13-day-12.md b/_posts/2024-1-13-day-12.md
index f1f1be4ad..968ec5450 100644
--- a/_posts/2024-1-13-day-12.md
+++ b/_posts/2024-1-13-day-12.md
@@ -17,7 +17,7 @@ And it got me 1100th place (at that point in time). So I decided to try to make
 I tried a few, and the results were way better
 ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b3ededca-1469-4084-b62e-8d52dfabf06a)
 
-Also ~~ on [day 7](./2024-1-8-day-7.md) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
+Also ~~ on [day 7](https://ivanstudyblog.github.io/day-7/) I created a model to predict WON/USD exchange rate. From Microsoft's ML-for-beginners tutorial I learned how to publish that simple model using pythoneverywhere. Here is the (simple) [webapp](https://divakaivan.pythonanywhere.com/).
 
 That is all for today!
 

From 4f2dad46eb9ba389330bc9ac6f2917de58156e81 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 26 Apr 2024 22:03:13 +0900
Subject: [PATCH 081/638] Create 2024-1-14-day-13.md

---
 _posts/2024-1-14-day-13.md | 45 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 _posts/2024-1-14-day-13.md

diff --git a/_posts/2024-1-14-day-13.md b/_posts/2024-1-14-day-13.md
new file mode 100644
index 000000000..ee5743172
--- /dev/null
+++ b/_posts/2024-1-14-day-13.md
@@ -0,0 +1,45 @@
+---
+layout: post
+title: (Day 13) I will be part of Korea Conference on Software Engineering 2024
+categories: [traditional-machine-learning]
+---
+
+## Hello :) Today is Day 13!
+A quick summary of today:
+* started [Machine Learning specialization](https://www.coursera.org/specializations/machine-learning-introduction) by Andrew Ng
+* got accepted to present at the [Korea Conference on Software Engineering 2024](http://sigsoft.or.kr/kcse2024/)
+
+First, I wanted to do Andrew Ng's machine learning lecture because I received a lot of recommendations. I am yet to finish it, but below is some of the content
+
+### Gradient descent
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1acee212-3d81-4db9-9f2b-905d47cd3c10)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/af759878-4871-40bc-8c27-0ea90ca9e808)
+
+We can see how the error decreases with the learning rate and w, b change. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/67cd43fe-cc27-47c9-b048-68d2d98735b9)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/bc856ebb-1578-488c-9b63-78fce1c58901)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/7003226d-38eb-4d96-85df-465799b1b7bd)
+
+If we keep updating w, b in that manner, we will reach the (global) minimum
+
+### Learning rate
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3f4e8551-ed17-4e62-8047-7f516f74f9ed)
+
+We can use the above method to try different learning rates when training NN models.
+
+### Regularization
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9fc0ef45-6d9a-4c17-9d0b-f188f978b096)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2c7c54c4-067e-4399-956b-646d9902b85f)
+
+Depending on lambda, the strength changes.
+
+**Next, about the KCSE 2024**
+During this semester, I wrote a literature review paper related to ChatGPT during my 'Software Quality Guarantee and Test' class, and my professor told me about the opportunity to submit it to the 2024 Korean Software Engineering Conference, so I submitted it and the result came out today, that it got accepted 🥳 So I will go to present it at the conference.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-13-2024.html)

From 4b3231718cc9d5a5aeb8f6ffd485908e255b4abe Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:14:49 +0900
Subject: [PATCH 082/638] Create 2024-1-15-day-14.md

---
 _posts/2024-1-15-day-14.md | 51 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 _posts/2024-1-15-day-14.md

diff --git a/_posts/2024-1-15-day-14.md b/_posts/2024-1-15-day-14.md
new file mode 100644
index 000000000..1fdaf388d
--- /dev/null
+++ b/_posts/2024-1-15-day-14.md
@@ -0,0 +1,51 @@
+---
+layout: post
+title: (Day 14) Andrew Ng's Machine learning specialization
+categories: [theory]
+---
+
+## Hello :) Today is Day 14!
+A quick summary of today:
+* Advanced learning algorithms - [Coursera](https://www.coursera.org/learn/advanced-learning-algorithms?specialization=machine-learning-introduction)
+
+Andrew Ng and Professor Yaser S. Abu-Mostafa's teaching methods are similar so I am really enjoying this!
+
+Today I learned about:
+
+* model selection
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0a5ebd88-a915-48fa-95f9-3e8016f8fdf2)
+
+When selecting a model, keep in mind the words "Do not use test data." You must select a model based on the training and verification datasets to evaluate the model with test data!
+
+* bias vs variance, and regularization
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c3da6da5-2283-4219-99ab-a2bc10aa2ebc)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/62e63592-173d-4bfb-8638-0075729df275)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6db65830-4f36-4628-a4ec-54d81e62ca7a)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/070fb5e6-525d-413d-b3c3-b795ab8d81be)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0c8edcb9-b5d5-475a-a973-799993e09cb5)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2b9eef1b-89cb-40b3-ac2b-df01b4220233)
+
+Andrew Ng said that bias and variance theory can be understood, but the difference can only be understood when creating a model. In the future, I should consider bias and variance when making a model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9a739829-eeb6-48f9-aa3d-a81133a3a1d0)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3dbf99b8-1d6f-4f33-a331-44720840d9a7)
+
+In neural nets, lambda can also be an important hyperparam
+
+* random forest
+
+  ![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/fd40a29e-c7a7-4a1a-980d-e1d92067046b)
+
+* decision trees and neural nets
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ea0b93d4-b09f-40d9-9ea1-041318620209)
+
+Today was great! Tomorrow I will learn about Unsupervised Learning, Recommenders, Reinforcement Learning
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-15-andrew-ngs-machine-learning.html)

From 16505adeffaa759d89410e4fed0c7009b49607f4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:15:07 +0900
Subject: [PATCH 083/638] Update 2024-1-2-day-1.md

---
 _posts/2024-1-2-day-1.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-2-day-1.md b/_posts/2024-1-2-day-1.md
index 06f66f544..f36453e0c 100644
--- a/_posts/2024-1-2-day-1.md
+++ b/_posts/2024-1-2-day-1.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 1) Machine learning course by Caltech
-categories: traditional-machine-learning
+categories: theory
 ---
 
 ## Hello :) Today is Day 1! My adventure begins!

From 82c94f88eb03510abf838a31c4b8a9a995b55a66 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:15:16 +0900
Subject: [PATCH 084/638] Update 2024-1-3-day-2.md

---
 _posts/2024-1-3-day-2.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-3-day-2.md b/_posts/2024-1-3-day-2.md
index 49c0ef06a..8b8550cfa 100644
--- a/_posts/2024-1-3-day-2.md
+++ b/_posts/2024-1-3-day-2.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 2) Finishing the ML course by Calctech
-categories: [traditional-machine-learning, statistics]
+categories: [theory, statistics]
 ---
 
 ## Hello :) Today is Day 2! 

From 7111111aa91554242d9123955e591963bbda6bc3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:15:28 +0900
Subject: [PATCH 085/638] Update 2024-1-4-day-3.md

---
 _posts/2024-1-4-day-3.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-4-day-3.md b/_posts/2024-1-4-day-3.md
index 679da2a05..44e249da2 100644
--- a/_posts/2024-1-4-day-3.md
+++ b/_posts/2024-1-4-day-3.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 3) Intro to time series
-categories: [traditional-machine-learning, time-series]
+categories: [theory, time-series]
 ---
 
 ## Hello :) Today is Day 3!

From 4628c83404689bb8985c4496fe0602b58d05d3db Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:15:41 +0900
Subject: [PATCH 086/638] Update 2024-1-5-day-4.md

---
 _posts/2024-1-5-day-4.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-5-day-4.md b/_posts/2024-1-5-day-4.md
index 0736c20fd..be283139a 100644
--- a/_posts/2024-1-5-day-4.md
+++ b/_posts/2024-1-5-day-4.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 4) Looking for a ML book and Microsoft's ML-For-Beginners course
-categories: [traditional-machine-learning]
+categories: [theory]
 ---
 
 ## Hello :) Today is Day 4!

From f70d5b8fbf03de0dec3215c180645bf25c42fe79 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:15:51 +0900
Subject: [PATCH 087/638] Update 2024-1-6-day-5.md

---
 _posts/2024-1-6-day-5.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-6-day-5.md b/_posts/2024-1-6-day-5.md
index dc726c650..94d6214f7 100644
--- a/_posts/2024-1-6-day-5.md
+++ b/_posts/2024-1-6-day-5.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 5) Kaggle's 'Learn' courses
-categories: [traditional-machine-learning]
+categories: [theory]
 ---
 
 ## Hello :) Today is Day 5!

From 51af6277201a9629faade10ac13a3d36f83a6699 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:16:51 +0900
Subject: [PATCH 088/638] Update 2024-1-15-day-14.md

---
 _posts/2024-1-15-day-14.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-15-day-14.md b/_posts/2024-1-15-day-14.md
index 1fdaf388d..29de1afaa 100644
--- a/_posts/2024-1-15-day-14.md
+++ b/_posts/2024-1-15-day-14.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 14) Andrew Ng's Machine learning specialization
-categories: [theory]
+categories: [traditional-machine-learning, deep-learning]
 ---
 
 ## Hello :) Today is Day 14!

From f5a796116144bf325fbfa1afac4572ab6bc1b928 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:17:00 +0900
Subject: [PATCH 089/638] Update 2024-1-2-day-1.md

---
 _posts/2024-1-2-day-1.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-2-day-1.md b/_posts/2024-1-2-day-1.md
index f36453e0c..06f66f544 100644
--- a/_posts/2024-1-2-day-1.md
+++ b/_posts/2024-1-2-day-1.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 1) Machine learning course by Caltech
-categories: theory
+categories: traditional-machine-learning
 ---
 
 ## Hello :) Today is Day 1! My adventure begins!

From 097415db30139dbce6bbcbdb966358e6f3b4c5b8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:17:07 +0900
Subject: [PATCH 090/638] Update 2024-1-3-day-2.md

---
 _posts/2024-1-3-day-2.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-3-day-2.md b/_posts/2024-1-3-day-2.md
index 8b8550cfa..49c0ef06a 100644
--- a/_posts/2024-1-3-day-2.md
+++ b/_posts/2024-1-3-day-2.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 2) Finishing the ML course by Calctech
-categories: [theory, statistics]
+categories: [traditional-machine-learning, statistics]
 ---
 
 ## Hello :) Today is Day 2! 

From 727cfcaf5a0082a354a9c371cb5cc2239d747f60 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:17:16 +0900
Subject: [PATCH 091/638] Update 2024-1-4-day-3.md

---
 _posts/2024-1-4-day-3.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-4-day-3.md b/_posts/2024-1-4-day-3.md
index 44e249da2..679da2a05 100644
--- a/_posts/2024-1-4-day-3.md
+++ b/_posts/2024-1-4-day-3.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 3) Intro to time series
-categories: [theory, time-series]
+categories: [traditional-machine-learning, time-series]
 ---
 
 ## Hello :) Today is Day 3!

From c6ddc84f905315c2cde53f673d50e36881282810 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:17:25 +0900
Subject: [PATCH 092/638] Update 2024-1-5-day-4.md

---
 _posts/2024-1-5-day-4.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-5-day-4.md b/_posts/2024-1-5-day-4.md
index be283139a..0736c20fd 100644
--- a/_posts/2024-1-5-day-4.md
+++ b/_posts/2024-1-5-day-4.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 4) Looking for a ML book and Microsoft's ML-For-Beginners course
-categories: [theory]
+categories: [traditional-machine-learning]
 ---
 
 ## Hello :) Today is Day 4!

From 9d7276153610d9b067d1961382a4d9edfcbce78f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:17:32 +0900
Subject: [PATCH 093/638] Update 2024-1-6-day-5.md

---
 _posts/2024-1-6-day-5.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-6-day-5.md b/_posts/2024-1-6-day-5.md
index 94d6214f7..dc726c650 100644
--- a/_posts/2024-1-6-day-5.md
+++ b/_posts/2024-1-6-day-5.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 5) Kaggle's 'Learn' courses
-categories: [theory]
+categories: [traditional-machine-learning]
 ---
 
 ## Hello :) Today is Day 5!

From f0c307967688f0fc479ab5e530195b1813507f93 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:25:12 +0900
Subject: [PATCH 094/638] Create 2024-1-16-day-15.md

---
 _posts/2024-1-16-day-15.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 _posts/2024-1-16-day-15.md

diff --git a/_posts/2024-1-16-day-15.md b/_posts/2024-1-16-day-15.md
new file mode 100644
index 000000000..50f2c9267
--- /dev/null
+++ b/_posts/2024-1-16-day-15.md
@@ -0,0 +1,26 @@
+---
+layout: post
+title: (Day 15) Quiz and practical lab from Andrew Ng's course
+categories: [traditional-machine-learning, deep-learning, applying-knowledge]
+---
+
+## Hello :) Today is Day 15!
+A quick summary of today:
+* I did Stanford Online's Machine Learning specialization quiz and practice labs
+
+I got the certificates for 2 of the courses
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1103cc6a-921a-474f-ad01-cce450b7149c)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2d02dbe5-9d5b-466f-a3ca-e2ee9cbc2e9f)
+
+As for the Lab - I learned how to adjust the Lambda of neural network models. And with the below code, we can find the optimal value.
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d9292980-0886-46e8-a3c4-eba887f9a12f)
+
+Tomorrow, I will finish the last Unsupervisored Learning, Recommenders, Reinforce Learning lectures and quiz and practice labs. 
+And, Machine Learning Specialization will be over.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-16-practical-lab.html)

From 3f5672afb491208b9dab0ab48f02390aa8a680cb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:34:58 +0900
Subject: [PATCH 095/638] Create 2024-1-17-day-16.md

---
 _posts/2024-1-17-day-16.md | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 _posts/2024-1-17-day-16.md

diff --git a/_posts/2024-1-17-day-16.md b/_posts/2024-1-17-day-16.md
new file mode 100644
index 000000000..e136ab910
--- /dev/null
+++ b/_posts/2024-1-17-day-16.md
@@ -0,0 +1,52 @@
+---
+layout: post
+title: (Day 16) Quiz and practical lab from Andrew Ng's course
+categories: [traditional-machine-learning, deep-learning, applying-knowledge]
+---
+
+## Hello :) Today is Day 16!
+A quick summary of today:
+* Finished Stanford/DeepLearning.AI's Machine learning specialization
+
+**Firstly**, let me share the final specialization certificate
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6239fb09-7a6d-4430-a968-f44f08f84220)
+
+**As for the learned content**
+
+Today I learned about reinforcement learning for the 1st time, and thanks to Andrew Ng it was absolutely enjoyable
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ce2d8a9a-92da-4fb3-b338-9c8df2bae255)
+
+`State` means action
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3db1b183-d2e4-4e23-bc49-7ed795a3bd38)
+
+`Return` just like in finance, is an important word
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6b1d6cf2-1761-40b5-982b-f6b3e8f2666f)
+
+`Policy` tells us which action to take given a state
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b6ff0932-afe6-4325-bfd0-49176c22c1a9)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f561aed4-e1c8-4b9d-8b7a-919bd0841964)
+
+Depending on the `discount factor` the model's impatience changes
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8f0a77a1-db5a-4824-8bd1-350d7c971a9e)
+
+Bellman equation - It is the return if you start from state s, take action a (once), then behave optimally after that
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/15e17f47-7a29-4e97-87ac-2735905dd2a2)
+
+_How should I choose the behaviour correctly while training?_ At first, Andrew Ng said that it would be good to choose a high probability of random behavioural choice and then gradually lower it while training
+
+<br/>
+
+From tomorrow, I will start preparing for the [TensorFlow developer certificate](https://www.tensorflow.org/certificate) which I found about today
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-16-ml-specialization.html)

From 37f689a119748a431b800125c0b5dbdb9751626d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:35:25 +0900
Subject: [PATCH 096/638] Update 2024-1-17-day-16.md

---
 _posts/2024-1-17-day-16.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-17-day-16.md b/_posts/2024-1-17-day-16.md
index e136ab910..3817f03a8 100644
--- a/_posts/2024-1-17-day-16.md
+++ b/_posts/2024-1-17-day-16.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 16) Quiz and practical lab from Andrew Ng's course
-categories: [traditional-machine-learning, deep-learning, applying-knowledge]
+categories: [reinforcement-learning, deep-learning]
 ---
 
 ## Hello :) Today is Day 16!

From 4d1f5defca218c58cf5df39e8c90d7eee85ab783 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:44:43 +0900
Subject: [PATCH 097/638] Create 2024-1-18-day-17.md

---
 _posts/2024-1-18-day-17.md | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 _posts/2024-1-18-day-17.md

diff --git a/_posts/2024-1-18-day-17.md b/_posts/2024-1-18-day-17.md
new file mode 100644
index 000000000..adc3e1d83
--- /dev/null
+++ b/_posts/2024-1-18-day-17.md
@@ -0,0 +1,37 @@
+---
+layout: post
+title: (Day 17) Microsoft's ML-For-Beginners - Classification and Clustering
+categories: [traditional-machine-learning, applying-knowledge]
+---
+
+## Hello :) Today is Day 17!
+A quick summary of today:
+* Classification and Clustering from [Microsoft](https://github.com/microsoft/ML-For-Beginners)
+
+I summarized the content in the photos below
+
+We can deal with imbalanced data using the `imblearn` library (before vs after using SMOTE)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/df19720c-adfa-4acd-b62e-2269569c4fc2)
+
+Lastly, I tried mading a website with a model of what food to eat today
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/385e4142-1e29-4e5d-bab0-6d71a417e508)
+
+After selecting ingredients, you will receive cooking recommendations based on your choice
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e810b128-6b13-4298-aa64-37074632ac20)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c718a4af-615b-495f-bdcc-f40015c64864)
+
+When I was covering the clustering material, I found a good way to visualize the data in such cases
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ae53941b-6b25-4bdb-b52e-36e086c87764)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6a775adc-2a37-45eb-a96f-89de65b2c908)
+
+I could not start preparation for the TensorFlow developer certificate today, but I will definitely start tomorrow!
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-17-microsoft-ml-for-beginners.html)

From 77ecff01932dd3685203e343b5030fcbfe776022 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:53:58 +0900
Subject: [PATCH 098/638] Create 2024-1-19-day-18.md

---
 _posts/2024-1-19-day-18.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 _posts/2024-1-19-day-18.md

diff --git a/_posts/2024-1-19-day-18.md b/_posts/2024-1-19-day-18.md
new file mode 100644
index 000000000..cbd1e7b3c
--- /dev/null
+++ b/_posts/2024-1-19-day-18.md
@@ -0,0 +1,38 @@
+---
+layout: post
+title: (Day 18) Step 1 to TensorFlow Developer Certificate
+categories: [traditional-machine-learning, deep-learning]
+---
+
+## Hello :) Today is Day 18!
+A quick summary of today:
+* Started Introduction to TensorFlow for Artificial Intelligence, Machine Learning, and Deep Learning by [DeepLearning.AI](https://www.coursera.org/learn/introduction-tensorflow?specialization=tensorflow-in-practice)
+
+The content follows
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/fe405ea8-4c83-42ca-bb2c-cba8e67e11c6)
+
+Some pictures of some of the content
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ef45a33d-1cce-45c0-8e42-c1ba1c047309)
+
+There was a simple model using mnist's fashion dataset, and a CNN
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/317fd3d1-554b-4079-967d-6cb825a74080)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/98e9b089-403a-4331-b4a5-6ecbaae4de56)
+
+In layer.MaxPooling2D(2,2), the image size is reduced, and if Conv2D(3,3) is used, the layer changes each 3x3 pixels to 1 pixel. Therefore, the image shape becomes 11,11,64 from 13,13,64
+
+Also, I got introduced the the `ImageDataLoader`
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d87fac4f-99ea-4710-83e5-d76561583007)
+
+<br/>
+
+Today was an introductory part, and I am looking forward to doing the rest (course 2, 3, 4)!
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-18-step-1-to-tensorflow-developer.html)

From f38504cc498dd6d7d5fb26f93e5fc70a91f7ba43 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 16:59:24 +0900
Subject: [PATCH 099/638] Create 2024-1-20-day-19.md

---
 _posts/2024-1-20-day-19.md | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 _posts/2024-1-20-day-19.md

diff --git a/_posts/2024-1-20-day-19.md b/_posts/2024-1-20-day-19.md
new file mode 100644
index 000000000..c38dff8db
--- /dev/null
+++ b/_posts/2024-1-20-day-19.md
@@ -0,0 +1,40 @@
+---
+layout: post
+title: (Day 19) TensorFlow Developer Certificate Part 2 - Convolutional Neural Networks in TensorFlow
+categories: [deep-learning]
+---
+
+## Hello :) Today is Day 19!
+A quick summary of today:
+* [CNN for TensorFlow developer certificate](https://www.coursera.org/learn/convolutional-neural-networks-tensorflow?specialization=tensorflow-in-practice)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cc668e99-e6bb-4596-bb22-c6066478174e)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/36d6063d-5976-458d-b9f2-d8f2049eb07a)
+
+Today I learned a bit more about the `ImageGenerator`
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d635754a-0be8-43d5-85da-d6e12947380f)
+
+* rescale=1./255 - pixel normalization
+* rotation_range=40 - rotates images 40 degrees
+* width_shift_range=0.2 - shifts images horizontally - image width
+* height_shift_range=0.2 - shifts images vertically within the range - image height
+* shear_range=0.2 - applies shear transformation
+* zoom_range=0.2 - zooms into images, up to a 20% zoom
+* horizontal_flip=True - flips images horizontally
+* fill_mode='nearest'	Determines the strategy for filling in newly created (or missing) pixels after transformations; 'nearest' fills with the nearest existing pixel value
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5907b5e4-8b14-420b-9e69-3633509e84a8)
+
+In the above case, overfitting can be corrected as shown in the picture below with Dropout(x) 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/52a050be-9c86-47ae-a81f-d5c41b570591)
+
+
+Tomorrow's part is Natural Language Processing, so it's going to be fun because I don't know anything about NLP.
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-19-tensorflow-developer-certificate.html)

From 7bdbc066682fdccd496cad90346abf35110afe7c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:07:32 +0900
Subject: [PATCH 100/638] Create 2024-1-21-day-20.md

---
 _posts/2024-1-21-day-20.md | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 _posts/2024-1-21-day-20.md

diff --git a/_posts/2024-1-21-day-20.md b/_posts/2024-1-21-day-20.md
new file mode 100644
index 000000000..32ab44b58
--- /dev/null
+++ b/_posts/2024-1-21-day-20.md
@@ -0,0 +1,59 @@
+---
+layout: post
+title: (Day 20) Natural Language Processing - TensorFlow Developer Certificate Part 3
+categories: [deep-learning, nlp]
+---
+
+## Hello :) Today is Day 20!
+A quick summary of today:
+* [Natural Language Processing in TensorFlow](https://www.coursera.org/learn/natural-language-processing-tensorflow/home/week/1)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/085e29c3-4c0e-4a6b-9bc6-ea977528830a)
+
+To do NLP, first, you need to tokenize the data and use pad_sequence
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9a558310-4a3b-478f-b34f-699fc5448070)
+
+Today, I learned about the sequence model. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d5a55a5d-adbc-4f0b-8671-d0b8f6450b74)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ecb23c6c-9b81-46a3-9a7e-35362162d1ac)
+
+I tried different models
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e794e3d9-bd2b-4e08-bd25-e3f365068102)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/936a7e59-8b2a-46f9-b16c-a4276f84daa7)
+
+And another one
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/96c0e865-429a-4b4a-ac9f-62dac22d8d9e)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5a584e21-7355-40b8-bace-0d5ecbc10447)
+
+### **Next**, I tried making a next-word predictor model
+
+Base model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cb2cb5d3-30b7-4241-a808-8f0664644823)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ff4aeec9-a037-4c51-8423-2b3041a8c1d6)
+
+2nd model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ea9e9969-d489-45a4-b45a-510605aec0eb)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a8de33a3-7149-4e44-af87-88d7d612d8a8)
+
+3rd model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/7c0b007e-03bb-4afa-9570-421942e0cd2c)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1d96058e-d141-49d4-a13c-820184f7f78a)
+
+
+Among them, the Base model obtained the highest acuity and lowest loss.
+
+I am still new to NLP, so I need to practice with new data on kaggle.
+
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-20-natural-language-processing.html)

From 8b02eb4acad5962d2f2ed15b6ea8a3ce4fcd96a1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:07:52 +0900
Subject: [PATCH 101/638] Update 2024-1-20-day-19.md

---
 _posts/2024-1-20-day-19.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-20-day-19.md b/_posts/2024-1-20-day-19.md
index c38dff8db..25dabaf98 100644
--- a/_posts/2024-1-20-day-19.md
+++ b/_posts/2024-1-20-day-19.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 19) TensorFlow Developer Certificate Part 2 - Convolutional Neural Networks in TensorFlow
-categories: [deep-learning]
+categories: [deep-learning. cnn]
 ---
 
 ## Hello :) Today is Day 19!

From c9f76dd78dd73684c6bb057ed5642b9b799935d8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:20:51 +0900
Subject: [PATCH 102/638] Create 2024-1-22-day-21.md

---
 _posts/2024-1-22-day-21.md | 57 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 _posts/2024-1-22-day-21.md

diff --git a/_posts/2024-1-22-day-21.md b/_posts/2024-1-22-day-21.md
new file mode 100644
index 000000000..5ded6e53f
--- /dev/null
+++ b/_posts/2024-1-22-day-21.md
@@ -0,0 +1,57 @@
+---
+layout: post
+title: (Day 21) (Almost) finished prep for TensorFlow developer certificate
+categories: [deep-learning. cnn]
+---
+
+## Hello :) Today is Day 21!
+A quick summary of today:
+* finished the [DeepLearning AI course](https://www.coursera.org/professional-certificates/tensorflow-in-practice) that prepares me for the certificate
+
+However, I don't think I can take the actual test yet. Because I think I need more practice. From tomorrow, I will apply what I learned in the course as the data I see in kaggle.
+
+Today I finished the last part - [Sequences, Time series & Prediction](https://www.coursera.org/learn/tensorflow-sequences-time-series-and-prediction?specialization=tensorflow-in-practice)
+
+I found the code for a very useful function
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/73bab40e-cfac-4ac7-8623-3c0bf10c2a39)
+
+When predicting time series with tensorflow, the dataset must be divided into windows
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/415b01e1-3619-4f4c-bd4a-9a6d60d14582)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cc420d14-803a-4ce9-acd2-187bdba8f352)
+
+You can train with this simple model, but it might be too simple
+
+I learned about the Huber loss - In statistics, the Huber loss is a loss function used in robust regression, that is less sensitive to outliers in data than the squared error loss
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f3f7436e-8605-43a4-a1de-02b880558f64)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c584f7fb-7138-4566-b161-3468e5776b1f)
+
+You have to change the data to the lambda layer before dividing it into windows and putting it into the Bidirectional LSTM layer. In addition, x*100.0 is applied to change it to a more understandable type in the last layer
+
+In the last piece of homework, I made the below model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a4435489-cab4-45d6-9e75-5418b43f9f4a)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4b6811a5-921d-4208-90d3-0d68cacd30de)
+
+**results: mse: 5.34, mae: 1.80**
+
+* Certificate
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/946448aa-43af-40ea-97f8-636d301431ca)
+
+
+### I also found a recent talk by Andrew Ng on [opporunities in AI](https://youtu.be/5p248yoa3oE)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f636b315-1210-4d18-bd97-73ddf96fe045)
+
+The full color is in the current state, but the semi-transparent color is in the state of AI after 3 years.
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-21-tensorflow-developer-certificate.html)

From b78313dad3ea1f1beb583c998b5d40450d692ce1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:21:47 +0900
Subject: [PATCH 103/638] Update 2024-1-22-day-21.md

---
 _posts/2024-1-22-day-21.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-22-day-21.md b/_posts/2024-1-22-day-21.md
index 5ded6e53f..d840406c9 100644
--- a/_posts/2024-1-22-day-21.md
+++ b/_posts/2024-1-22-day-21.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 21) (Almost) finished prep for TensorFlow developer certificate
-categories: [deep-learning. cnn]
+categories: [deep-learning, time-series]
 ---
 
 ## Hello :) Today is Day 21!

From 23f67fa65dda22646d04eec83e3b36f78f43c1a1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:22:01 +0900
Subject: [PATCH 104/638] Update 2024-1-20-day-19.md

---
 _posts/2024-1-20-day-19.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-1-20-day-19.md b/_posts/2024-1-20-day-19.md
index 25dabaf98..0df61fe94 100644
--- a/_posts/2024-1-20-day-19.md
+++ b/_posts/2024-1-20-day-19.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 19) TensorFlow Developer Certificate Part 2 - Convolutional Neural Networks in TensorFlow
-categories: [deep-learning. cnn]
+categories: [deep-learning, cnn]
 ---
 
 ## Hello :) Today is Day 19!

From 48ea6e83ca2d81ae0686c8aa81b03520ce80e33e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:37:05 +0900
Subject: [PATCH 105/638] Create 2024-1-23-day-22.md

---
 _posts/2024-1-23-day-22.md | 118 +++++++++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 _posts/2024-1-23-day-22.md

diff --git a/_posts/2024-1-23-day-22.md b/_posts/2024-1-23-day-22.md
new file mode 100644
index 000000000..9ed9cb3b1
--- /dev/null
+++ b/_posts/2024-1-23-day-22.md
@@ -0,0 +1,118 @@
+---
+layout: post
+title: (Day 22) Created a Card image classifier model using Neural networks
+categories: [deep-learning, cnn, applying-knowledge]
+---
+
+## Hello :) Today is Day 22!
+A quick summary of today:
+* Card image classifier using neural networks on [Kaggle](https://www.kaggle.com/code/divakaivan12/card-image-classifier-using-neural-networks?scriptVersionId=160107109)
+
+I learned about CNN during Course 1, 2 of the TensorFlow developer certificate preparation course and wanted to apply it to new data today.
+I searched Kaggle and found the following card image dataset.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/fd015831-1016-4042-8620-91cafed374aa)
+
+Following the course, first I used `ImageDataGenerator`
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d3027292-2f58-4627-8ab9-888dff7d526f)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5a2e82ca-7fea-4f9a-a46c-1be49719e687)
+
+Then made the model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/82467531-bb93-4db1-b8fa-abe8d99e4ec8)
+
+Wrote the below to compare train and valid results
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/efbfdd2f-3539-4e2e-aba4-0cbfa66610a0)
+
+And finally, evaluated the model on unseen data
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/da239665-f1c3-4651-a8ea-70cf577ab940)
+
+The results... not that good haha but it was very good practice overall.
+
+Various models were constructed to obtain good performance, by changing layers, learning rates, data augmentation variables, filters of layers... Still, more than 65% accuracy was not obtained with the valid dataset. The loss of some models increased significantly after 5-10 epochs, but the loss of any model did not reach 50% or more accuracy after 100 epochs.
+
+Below are pics of some of the models that I ran.
+
+* model 1
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9c424f6f-0d46-4047-a013-60e178161d83)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3c81e034-043b-42e0-89ff-855190b7fe4b)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0fe35c91-ca92-4fae-bdc2-8e5cda06b2b8)
+
+* model 2
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8c298058-5a45-4168-9ceb-c0f3a99a0fe3)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9e276a14-9ea6-4518-b2c2-bf361cc77f73)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2388db65-c8a8-4f6d-ae05-ccc9d59a0e44)
+
+* model 3
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/574078e7-87d0-421b-bacc-2247326feeba)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8354dc25-4d13-4320-9016-1d402cec1f38)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ccf365e5-47df-4670-b7be-2570cde14dae)
+
+* model 4
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8f445eca-d6e0-41a2-8237-810d481fd8de)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a2c60cfc-7cc7-4fd2-a3d0-ae8d97ef4b32)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a8e533e3-d8b8-4fa0-9eb5-e7c31465082d)
+
+* model 5
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/44ee33ab-0dab-4de7-b997-5383d5a85d87)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b30f4fa6-4488-4fec-b76c-ec125c017997)
+
+* model 6
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/de83dc12-7f90-4a81-aba5-d508eac66a77)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/326e13e0-6bad-4623-8096-18e00d6bce81)
+
+stopped the training
+
+* model 7
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8cce9c82-45b0-445b-af48-7cffd21bdb4f)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c76b8c0d-0cb4-42e0-8256-ab358312e904)
+
+stopped the training
+
+* model 8
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/06b5a6f9-fc20-429f-8e95-7e0c7f5c3151)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/fcedf5ea-dda5-4f98-b053-084a31e6f727)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1b0252ea-cada-4e0c-bbe3-352324d0c4f2)
+
+* model 9
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/7b5866fb-8148-460d-8918-481df1152cd3)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8afde0b1-8b37-47b8-a36b-7c5d69e8cb3c)
+
+* model 10
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/83c72c63-9fe0-4d05-a0a8-ad1fe2fe8b14)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/871363c2-72c7-4d83-8d49-c95ebdbb818c)
+
+* model 11
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/566b9143-370c-4ef9-95b8-eea87c4b31f2)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2c9f9090-f451-4826-91da-f1c12e09600b)
+
+* model 12
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/334d7ed0-ebef-44a3-b38f-4019885b9b9b)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a0d233cc-5d91-4136-8df4-8dc8e7615aa7)
+
+I don't include all the information in the model picture, it's Model 1 - base model, and then I just changed the variables.
+It took quite a while to train. It took about an hour per 20 epochs.
+
+Tomorrow I will try to make some kind of NLP model.
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-22.html)

From e8d3f7d088728d81be1a960d32ef3e184720a0d7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 17:55:31 +0900
Subject: [PATCH 106/638] Create 2024-1-24-day-23.md

---
 _posts/2024-1-24-day-23.md | 92 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 _posts/2024-1-24-day-23.md

diff --git a/_posts/2024-1-24-day-23.md b/_posts/2024-1-24-day-23.md
new file mode 100644
index 000000000..cc04507a1
--- /dev/null
+++ b/_posts/2024-1-24-day-23.md
@@ -0,0 +1,92 @@
+---
+layout: post
+title: (Day 23) Tried making an Natural Language Processing model
+categories: [deep-learning, nlp, applying-knowledge]
+---
+
+## Hello :) Today is Day 23!
+I wanted to apply what I learned from [DeepLearning.AI's](https://www.coursera.org/learn/natural-language-processing-tensorflow/) course:
+* [Fin statement sentiment classification](https://www.kaggle.com/code/divakaivan12/beginner-fin-statement-classification-tensorflow)
+* [Football text generator model](https://www.kaggle.com/code/divakaivan12/beginner-football-text-generator-model?scriptVersionId=172820619)
+
+### Firstly, the financial statement sentiment classification model
+
+In order to analyze text sentiment, I found data divided into text of its statement on Kaggle.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/18197bf4-4f98-48e1-8eac-fd14792ec84a)
+
+First, I removed stopwords
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5e532888-e8ec-4569-b298-f16090c8e548)
+
+Then I tokenized and padded the text
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/164f1c34-2ee9-4d7d-a55b-24fc4d99a289)
+
+Then created the model (below is the one I ended up publishing in the Kaggle notebook)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/02dc09d8-d662-4ecf-a714-7b2b515a8d81)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b9604b66-f0cd-4097-a6b5-a93f1253e357)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b37a38a4-f646-4827-b8c1-6210cd208a30)
+
+
+### Secondly, the football text generator model
+
+For this one, I gathered [data](https://www.kaggle.com/datasets/divakaivan12/description-of-2019-2020-premier-league) from Wikipedia, and some from ChatGPT about the 2019-2020 Premier League season.
+
+Firstly, I tokenized the text
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a9266d8d-27f0-47d5-b03a-bf1207c8b808)
+
+Then I created subphrases based on each sentence
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b9c36650-3181-44ec-b21a-34e8aa183b21)
+
+For example, `I like dogs` is `I - 1, like - 2, dogs - 3`
+```
+[[0,0,1],
+
+[0,1,2],
+
+[1,2,3]]
+```
+
+Here I also tried many models, but the final one is below
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/947d8408-dbd6-43b7-bc18-fc53d563edef)
+
+Accuracy:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9607fabe-10d8-45ef-90ff-21091613e206)
+
+Loss:
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3fd14976-976a-4949-a1e5-c002cb89281d)
+
+Lastly, I gave it some key words to start from
+`['Chelsea', 'Manchester United', 'Arsenal', 'Manchester City', 'Tottenham', 'Best team was', 'Against the wall', 'The FA cup was hard but', 'The season was filled with hardships and', 'In the end, the best team was', 'Some teams performed bad, so they were'] `
+
+and the output for each one:
+
+```
+1. Chelsea amidst a season of transition under the management of club legend frank lampard demonstrated an evolving form in the league
+2. Manchester United with flashes of brilliance and frustrating dips in form in the lower ranks added an extra layer of intensity showcasing
+3. Arsenal reached the final of the fa cup overcoming formidable opponents including manchester city and chelsea secure a chance to free
+4. Manchester City led by the tactical genius of pep guardiola sought to defend their premier league crown with the same possession based
+5. Tottenham amazon made a significant impact by broadcasting two rounds of fixtures in december featuring the highly anticipated merseyside derby occasionally
+6. Best team was ensure an organized viewing experience games played on the same day were assigned separate time slots preventing overlap and ensuring
+7. Against the wall season was characterized by moments of brilliance on the pitch as well as challenges that ultimately impacted city's quest for
+8. The FA cup was hard but and son heung min and kane continuing to be standout contributors in attack maintained city's trademark attacking prowess led by
+9. The season was filled with hardships and brighton exemplified city's dominance in disarray tournaments success in other competitions and wolverhampton while arteta southampton their modern history of
+10. In the end, the best team was a mere sporting event securing eight consecutive wins 24 of which 7 were carried over from the previous season of
+11. Some teams performed bad, so they were threats to opposition recognizing that klopp and ultimately brighter days lay ahead for the blues under their legendary manager that
+```
+
+Tomorrow, I will try to make some kind of time-series model
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-23-natural-language-processing.html)

From 5e5cd249c7e5b643a9718457b7a58a49116f1ad1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 18:13:12 +0900
Subject: [PATCH 107/638] Create 2024-1-25-day-24.md

---
 _posts/2024-1-25-day-24.md | 134 +++++++++++++++++++++++++++++++++++++
 1 file changed, 134 insertions(+)
 create mode 100644 _posts/2024-1-25-day-24.md

diff --git a/_posts/2024-1-25-day-24.md b/_posts/2024-1-25-day-24.md
new file mode 100644
index 000000000..d2df8d7d6
--- /dev/null
+++ b/_posts/2024-1-25-day-24.md
@@ -0,0 +1,134 @@
+---
+layout: post
+title: (Day 24) Using neural nets for time series predictions
+categories: [deep-learning, time-series, applying-knowledge]
+---
+
+## Hello :) Today is Day 24!
+A quick summary of today:
+* Using neural nets to predict ₩/$ exchange rate on [Kaggle](https://www.kaggle.com/code/divakaivan12/using-neural-nets-to-predict-exchange-rate?scriptVersionId=160564254)
+
+The data I used today was also used when I made the model I made on Day 7. But then I made a regression model, and today I made a neural network model.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/546f8e71-b827-4eac-8e1b-2538e7e65d24)
+
+First of all, to predict time series, I learned that it would be good to divide the data into windows
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f283aae4-4bff-4a60-8b78-fe64cb3623d0)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c3b72161-bfc7-4cd7-b8a6-82ae994ddc00)
+
+Tuning the parameters above greatly affects the performance. It was fun to tune this parameter and model layer parameter a lot throughout the day and see how the performance was different
+
+The final model is:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0b5ca33b-225d-4196-af4f-e1db05abfa42)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/fdc2aa72-19af-40ac-bcd8-1a036f138e2f)
+
+Results:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5abb04af-9b84-4dfb-a229-596ccdb7e39e)
+
+Test data results:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4c5da68d-3fc7-4671-ba55-ee65d2637109)
+
+
+Below, are various models, and modifications where I wanted to see the impact of hyperparameters, so I tuned the models manually.
+
+* model 1
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/84da28f4-0496-40c7-9f2c-151bfa4c54e0)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e7ca0c1e-4ee7-499d-9be0-1f9c88dbbeda)
+
+MAE: 0.02755
+
+* model 2
+window_size = 183
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/40e5648a-445c-4e4e-a130-24177d495893)
+
+MAE: 0.03141
+
+* model 3
+window_size = 60
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/adc07f23-0dd4-40da-9fce-9b2da0346a1b)
+MAE: 0.02798
+
+* model 4
+shuffle_buffer_size = 5000
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/10da09b4-13bd-488a-ac59-caa4f93dd8a0)
+MAE: 0.02503
+
+* model 5
+shuffle_buffer_size = 10000
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/551e9d8b-883b-4477-98c6-94fa13aa3c69)
+MAE: 0.01939
+
+* model 6
+window_size=45
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/79a43e4b-a647-4f3c-aaf4-53f591dcb818)
+MAE: 0.03332
+
+* model 7
+window_size=60;
+batch_size=16
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/31e5f4ce-c689-4b19-9ac6-aa22236b768c)
+MAE: 0.02999
+
+* model 8
+batch_size=32; added another LSTM layer
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/dc98f072-90ef-4c1e-bd02-be9d9858b148)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/40e8b56c-076b-48b8-8b41-e2cb6bfd829b)
+MAE: 0.01627
+
+* model 9
+shuffle_buffer_size = 1000 (as it was originally)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5c8a519d-0932-47d0-856e-d61401ae16b6)
+MAE: 0.02009
+
+* model 10
+added another LSTM layer
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/bd8c3535-b128-42c2-bb50-e35a6c890b44)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1baebe6a-8d7b-4ae0-9aa7-2d267a9037c0)
+MAE: 0.01756
+
+* model 11
+shuffle_buffer_size = 5000
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/76b7878a-35e7-4d1d-ba57-ec234320f610)
+MAE: 0.04496
+
+* model 12
+shuffle_buffer_size=1000 + change of model architecture
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/11cc16bb-a47a-46b4-9e5b-aac339233f53)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9ecfccf3-afda-4c80-a43f-e6e4002dff2f)
+MAE: 0.02415
+
+* model 13
+changed model architecture again
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0657ea95-64f0-4f14-b494-e81420ba0fde)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1cdf5b0a-22e7-4228-96f1-6faa75e4ce76)
+MAE: 0.02625
+
+There were other models too, but I did not take picture
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-24.html)
+
+
+

From 5e0eecc0f2de761745a39f042470ecfbbfcac209 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 18:32:58 +0900
Subject: [PATCH 108/638] Create 2024-1-26-day-25.md

---
 _posts/2024-1-26-day-25.md | 53 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 _posts/2024-1-26-day-25.md

diff --git a/_posts/2024-1-26-day-25.md b/_posts/2024-1-26-day-25.md
new file mode 100644
index 000000000..9f176333f
--- /dev/null
+++ b/_posts/2024-1-26-day-25.md
@@ -0,0 +1,53 @@
+---
+layout: post
+title: (Day 25) Using neural nets for time series predictions
+categories: [deep-learning, applying-knowledge]
+---
+
+## Hello :) Today is Day 25!
+A quick summary of today:
+* randomly found [Andrej Karpathy's](https://www.youtube.com/watch?v=VMj-3S1tku0&list=PLAqhIrjkxbuWI23v9cThsA9GvCAUhRvKZ&ab_channel=AndrejKarpathy) neural network series and started it
+
+It has 8 videos
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/200f6794-8c1d-4487-9d3a-4738b38bcefa)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6dfc717a-e9f3-4aef-8c68-a5e479b558a5)
+
+Today I managed to watch 4 of them. WOW amazing!
+
+Some of the content is below
+
+Firstly, I learned a bit about forward and backward propagation. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/bcb0eb9d-2fb3-485d-b622-c39b9559512b)
+
+Simply put, forward prop is prediction and loss calculation, and backprop is through calculus(chain rule) the gradient for every neuron is calculated, and then with SGD, Adam or another optimizer, we do optimization.
+
+Then, a simple character text prediction model was constructed and it is similar to DeepLearning.AI's text generation course.
+
+**However**
+In the fourth video, was about a NNs loss function. Andrej said that the loss visualization should not be similar to the hockey-stick (he said it was not good). I think the loss of the model I did yesterday was like that.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/526b4798-e382-4030-9f39-60c9767f2d40)
+
+So, I decided to change the model a bit, and Andrej said that it is a good solution to put the bias directly on the last layer.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0daf5f49-067a-4e15-a286-002fe54fd756)
+
+Resulting loss graph:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4cfae444-ba93-45d6-8b93-bb52c3fa284e)
+
+And test prediction:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e08ccde7-30b9-4776-b4b8-3cb531874995)
+
+It still looks like a hockey-stick a bit, but it is definitely better than before. Definitely need to rewatch today's videos in the future.
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-25-andrej-karpathy.html)

From de13db473b7b3fe56efd343c120f6c2d22605b50 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 18:46:09 +0900
Subject: [PATCH 109/638] Create 2024-1-27-day-26.md

---
 _posts/2024-1-27-day-26.md | 42 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 _posts/2024-1-27-day-26.md

diff --git a/_posts/2024-1-27-day-26.md b/_posts/2024-1-27-day-26.md
new file mode 100644
index 000000000..4e4ba9722
--- /dev/null
+++ b/_posts/2024-1-27-day-26.md
@@ -0,0 +1,42 @@
+---
+layout: post
+title: (Day 26) Deep Learning Specialization course by Andrew Ng
+categories: [deep-learning]
+---
+
+## Hello :) Today is Day 26!
+A quick summary of today:
+* started DeepLearning.AI's specialization course and covered [NNs and DL](https://www.coursera.org/specializations/deep-learning)
+
+The specialization covers 5 courses
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/51cc4f0d-25f0-4546-88ea-5e91a8a030b5)
+
+I decided to learn a bit more about deep learning.
+
+To summarize the contents, it was about making neural networks which includes forward pass, back pass, and activation function calculation.
+
+* Learned about sigmoid (+wrote the code)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/18e2fca0-9e28-44d1-b491-6faf5d09ca90)
+
+* sigmoid derivative
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9a367155-d628-4b34-bd94-65c08e51665a)
+
+* how forward and back prop work
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5064049b-b4e9-40ef-a1b3-143734100121)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2cb413f1-fedc-44a5-a7ec-cc391b52ca54)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0b036457-2299-4335-a711-a55d60e33999)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/60a9ce34-2619-464a-975b-5674d252b5ff)
+
+This was a bit difficult, but it's very fun. The final programming assignment of the 1st course is to create a neural network model according to the picture above from start to finish! So I will refer to those pics tomorrow when I attempt it.
+
+_Fundamental knowledge is key!_
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-26-deep-learning-specialization.html)
+

From b99bb634e4c536e5d0764a33df68d457e39386ad Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 19:04:21 +0900
Subject: [PATCH 110/638] Create 2024-1-28-day-27.md

---
 _posts/2024-1-28-day-27.md | 101 +++++++++++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 _posts/2024-1-28-day-27.md

diff --git a/_posts/2024-1-28-day-27.md b/_posts/2024-1-28-day-27.md
new file mode 100644
index 000000000..5a014acdb
--- /dev/null
+++ b/_posts/2024-1-28-day-27.md
@@ -0,0 +1,101 @@
+---
+layout: post
+title: (Day 27) Improving deep learning models
+categories: [deep-learning]
+---
+
+## Hello :) Today is Day 27!
+A quick summary of today:
+* covered the 2nd course from [DeepLeaning.AI](https://www.coursera.org/learn/deep-neural-network?specialization=deep-learning)'s DL specialization
+
+### First of all, I finished what I started yesterday
+I made a simple deep learning model from scratch.
+
+1. Init params
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6091f8ad-a3a8-423a-ad5a-b37d04be9512)
+
+2. do forward prop, compute cost, do backward prop, update params
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b78e80f0-63db-41a7-8a7d-0347f774de92)
+
+I don't know if people do this in practice, but I think it helped me to understand the neural network model more deeply.
+
+### Now for the Improving Deep Neural Networks: Hyperparameter Tuning, Regularization and Optimization course
+
+It introduced this basic recipe for ML
+
+<img width="631" alt="image" src="https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8aeff876-e2cf-4d85-afe8-ad39cb62b36c">
+
+To overcome bias or variance there are many methods but one of them is to normalize the data
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d4f1b823-fedf-410d-a503-0a3db3b2fa28)
+
+After normalizing, finding the minimum is way easier.
+
+We can also do different initializations of the Weights and Biases of the layers
+
+1. In this case W, b are set to 0
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5c0beaf5-680a-47ed-8423-44e5f6312fa5)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3312043f-7730-47b8-96bc-47a930033010)
+
+2. In this case, W is a large number, b is 0
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e274f7b4-1354-4fe3-a9d5-b3eef7a999ab)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c3a45e84-3116-46fd-ae06-335f5ab06efb)
+
+3. There is this He initialization
+```python
+parameters['W' + str(l)] = np.random.randn(layers_dims[l], layers_dims[l-1])*np.sqrt(2./layers_dims[l-1])
+        parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))*np.sqrt(2./layers_dims[l-1])
+```
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d4c0ecdc-ee6e-4219-a638-6c46be14b710)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/09ccb982-03bd-4058-b67b-5350b9b07f7d)
+
+Different inits lead to different results. Random inits are good for breaking symmetry and make sure different hidden layers learn different things.
+
+Next, **regularization**
+
+1. A model without
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4f238133-c60b-418a-bec3-6ec6c2b66638)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6c7b6cad-a846-44ae-a39e-5eb16c63b1c1)
+
+2. Using L2 norm
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4180ff2f-f2d2-4067-ae5e-ac71f2889c92)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/ed447454-a489-4ecb-aca4-6bdfc695428b)
+
+3. Using a dropout layer
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a363d195-f255-47a6-8d56-e7770131cf35)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9cf34030-679e-4c06-b932-676964709c42)
+
+Next, **optimization**
+
+The Adam optimizer has 3 hyperparams that can affect its effects
+
+* Beta1: typically set close to 1 (but less than 1), controls the exponential decay rate for the first moment estimates (the mean of the gradients).
+* Beta2: Similar to beta1, beta2 is another exponential decay rate parameter, typically also set close to 1. It controls the decay rate for the second moment estimates (the uncentered variance of the gradients). A common default value for beta2 is 0.999.
+* Epsilon: This parameter is a small constant added to the denominator to prevent division by zero and to improve numerical stability. It ensures that the optimizer's calculations don't explode when the denominator approaches zero. A typical value for epsilon is around 1e-8.
+
+Also, I was given a learning rate optimization method
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/29abfe12-d504-4a47-9dfe-d0f537f40f50)
+
+Among these, the next one is supposed to be the most effective
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e272c05d-83e6-480f-b962-97ba56fe9ce1)
+
+Another regularization technique is **batch normalization**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/7a980336-f9a2-4f73-8292-fcf5fcc6fd80)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d9a17c27-2b90-4347-9230-e72aaabeb166)
+
+It is a method used to normalize the pre-activation values (z) in neural networks, allowing for more stable training. It employs parameters, beta and gamma, to adjust the mean and standard deviation of the normalized values.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/42bd26ed-48b0-4c14-838c-3a8ed52f4bc6)
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-27.html)
+

From 34bd91bc039c8fe8fe0d4c1064b4daf866ab5e44 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 19:19:27 +0900
Subject: [PATCH 111/638] Create 2024-1-29-day-28.md

---
 _posts/2024-1-29-day-28.md | 79 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 _posts/2024-1-29-day-28.md

diff --git a/_posts/2024-1-29-day-28.md b/_posts/2024-1-29-day-28.md
new file mode 100644
index 000000000..6c954fbb6
--- /dev/null
+++ b/_posts/2024-1-29-day-28.md
@@ -0,0 +1,79 @@
+---
+layout: post
+title: (Day 28) Diving deeper into Convolutional Neural Networks
+categories: [deep-learning, cnn]
+---
+
+## Hello :) Today is Day 28!
+A quick summary of today:
+* covered [CNN by Andrew Ng](https://www.coursera.org/learn/convolutional-neural-networks/)
+
+Today's material was a bit hard, but still fun ^^ 
+
+I learned about widely used CNN architectures (LeNet, AlexNet, VGG-16, ResNet, MobileNets)
+
+### 1. Basics
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/693d7c8f-1e2f-49b1-b739-774a0ad3ceb7)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cb20a0e4-f8ec-4e7c-938d-24eacec2ea62)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2270e89c-acb9-4d0b-961c-bf9813415681)
+
+depending on the filter hyperparam, we can learn different features
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/941b82d2-4a9b-478b-a4bd-57225ad4498e)
+
+During training, the w1....w9 parameters can be learned, allowing us to know the different features of the image
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f2c4b504-9377-4568-b132-5538cf3330b0)
+
+If you look at this picture, the model is composed of a conv layer and a pool layer.
+Among the pool layers, there are average and max layers, and the calculation is different.
+
+If I want to learn more about architectures, I will need to read some research papers. 
+
+### 2. Widely used CNN architectures
+
+* LeNet
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2692b5b9-8e96-4269-a3d7-7f2417d6626d)
+7 layers: 3 conv, 2 pooling, 2 dense. Used for handwritten digit recognition
+
+* AlexNet
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/855ded16-5456-4e70-9f9b-e9a0eb95ed48)
+8 layers: 5 conv,  max-pooling layer, 3 dense
+
+* VGG-16
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0da8f37a-6ff1-4694-b438-020e0a42da5e)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c09104d1-6cd7-429b-9f3b-0a69dcc74c6d)
+conv + pool + dense layers
+
+* ResNet
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b87e8fa8-c0f2-4d0c-b908-468b3f7c2d39)
+different architectures: ResNet-18, ResNet-50, ResNet-101, and ResNet-152
+
+ResNet's 'skip connections' allowed us to train deeper learning neural network model. 
+
+### 3. Other
+
+* 1x1 convlutions
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e64d5fe9-98b0-42fd-9e5b-453db5a52f0c)
+I need to read the paper for this one - [https://arxiv.org/abs/1409.4842](.)
+
+* MobileNets
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8e71410a-a024-4359-b144-4de7e0dcae3f)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/62e652b2-79c1-4f74-98ce-23b4c4ee1dee)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3bfdb4fe-bc2d-4182-89a3-19924b36934d)
+
+There is big cost difference.
+
+
+I need to apply what I learned today in Kaggle some day.
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-28-convolutional-neural-networks.html)
+
+

From b32afbfa61cfe8814c7b9e2ff2361686a859b757 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 19:35:28 +0900
Subject: [PATCH 112/638] Create 2024-1-30-day-29.md

---
 _posts/2024-1-30-day-29.md | 99 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 _posts/2024-1-30-day-29.md

diff --git a/_posts/2024-1-30-day-29.md b/_posts/2024-1-30-day-29.md
new file mode 100644
index 000000000..59154fbe4
--- /dev/null
+++ b/_posts/2024-1-30-day-29.md
@@ -0,0 +1,99 @@
+---
+layout: post
+title: (Day 29) How to do object detection?
+categories: [deep-learning, cnn]
+---
+
+## Hello :) Today is Day 29!
+A quick summary of today:
+* Object detection from DeepLearning.AI's DL course
+
+Today's material is still part of the CNN course.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/663a8f35-52d5-48bd-8a7a-13d677591c8e)
+
+### 1. Classification with localization
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b3576f0a-80b8-4974-8a9c-f0df17a00567)
+
+If you give a picture to our model, output also includes the object you looked for and the 'bounding box' coordinates of that object
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/09430688-1368-4175-b23a-a8cb9cd184e1)
+
+For example, y includes [pc, bx, by, bh, bw, c1, c2, c3] and pc indicates whether there is an object or not, and the coordinate diagram of the bx, by, bh, bw bounding box is also a variable of multiple classifications (c1 = pedestrian, c2 = car, c3 = motorcycle).
+
+**How do we train such a model? **
+
+First of all, we can train a classification model to see if there is a car or not, and then we can find out whether there is a car or not using the Sliding Windows detection method.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0b6faccd-302e-4719-9c55-96f7fff270cf)
+
+However, it was found that the cost of this method was too high, and a method that could be done with convolution was created.
+In the above case, we move the window one by one, one by one, but we can do it all at once with a convolution.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3c32cfbd-54f9-4d5d-868a-d706ecc1957f)
+
+### 2. YOLO
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/17b388cb-54e9-4ab7-814c-fb0a1e9a425a)
+
+The You Only Look Once algorithm is one of the most widely used algorithms. It is a fast method that can be processed at once if you give an image.
+Using the 'Grid based approach', the picture is divided into several windows and calculates the bounding box and pc of the object in that window.
+
+### 3. Object localization evaluation
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d224b2c7-804f-4ba1-8731-7de7bda6aae7)
+
+It can be evaluated by the Intersection over Union (IoU) method.
+If there is a prediction bounding box and a real bounding box, the IoU can be calculated with area of overlap (yellow) and area of union (blue).
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/74dd5637-7634-4ad9-a390-ef84f707fb2b)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6674ae54-314e-41c0-bef0-86e6e89dce85)
+
+In fact, when using the YOLO model, 19x19 grid is often used, so the object may be in various windows, so if the pc (probability of object presence) of the bounding box is not 0.6, it can be discarded.
+
+### 4. Anchor boxes
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f32d6ad9-1846-47ba-a479-5834d922eabe)
+
+The anchor box concept is, for example, a car is usually horizontal. A person is vertically standing. So if you define anchor box for 2 things, you will include anchor box information in the output.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e02fc88d-6faf-492e-b52e-89d3bdfc4622)
+
+And the output is not 3x3x8, but it will be 3x3x(8x2) (there are 2 anchor boxes)
+
+### 5. Regional proposal (R-CNN)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/728efba1-06d7-43bd-8f49-fb9a70234c8b)
+
+R-CNN is a method of defining and showing different classes as different colours (numbers) rather than dividing them into windows.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/88221c12-93e9-4fd8-82f3-f42fda9f6bdc)
+
+Below are algorithms that have been found to be faster than R-CNN
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/248a99f9-ba25-4114-81f7-9b435d32a696)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/43a434b0-992d-4e15-b808-43123bc620d1)
+
+### 6. Semantic segmentation
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/15756b61-7084-4378-bda5-159f4a39904d)
+
+Semantic segmented images can be received by a transpose convolution method without the Dense layer in the normal model (hence it is removed in the pic above)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/47a3473c-c9c7-4fd7-89d1-2c90db58b790)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5880dfd1-e03b-4d09-a743-366c5f1f1577)
+
+This method is called U-Net.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/75313e3b-2e0e-41f1-9a64-e5ba0376f44f)
+
+The U-Net implementation seems a bit hard right now, so I will take a note to do it later. 
+
+### KCSE 2024
+
+I don't think I can learn much on my own for three days starting from tomorrow at the KCSE 2024 conference, but I will summarize what will be presented at the conference.
+My presentation is on February 3rd and I'm looking forward to it!
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-29-object-detection.html)

From a45da451a4feffa68e48a1c4985f9419e487aab7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 19:49:18 +0900
Subject: [PATCH 113/638] Create 2024-1-31-day-30.md

---
 _posts/2024-1-31-day-30.md | 79 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 _posts/2024-1-31-day-30.md

diff --git a/_posts/2024-1-31-day-30.md b/_posts/2024-1-31-day-30.md
new file mode 100644
index 000000000..3aa2ff536
--- /dev/null
+++ b/_posts/2024-1-31-day-30.md
@@ -0,0 +1,79 @@
+---
+layout: post
+title: (Day 30) 
+categories: [deep-learning, cnn]
+---
+
+## Hello :) Today is Day 30!
+A quick summary of today:
+* covered the last bit of the [CNN course](https://www.coursera.org/learn/convolutional-neural-networks/home/week/4)
+* first day of [KCSE 2024](http://sigsoft.or.kr/kcse2024/)
+
+### Firstly, the last bit of the CNN course
+
+It was about face recognition and neural style transfers.
+
+_Face recognition_
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/41999682-7d92-4ae5-9acc-9d9d835c2e48)
+
+You need to be aware of the difference between recognition and verification.
+Verification is the process of showing a face and verifying that it is the person by comparing it with the image of the person in the system.
+Recognition is the process of showing a face and verifying that a person's face is among all the pictures in the system.
+
+**How do we evaluate a face recognition model?**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/efc3369d-8ac1-405d-9317-923b220dfd17)
+
+**How do we compute similarity?**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/56490cfb-bc6a-4669-af7f-9b4a2a0b6597)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4f119864-6118-4b57-9385-6ff47e05c0b7)
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6fba6ad6-ec65-4733-a8e6-10aef7a14517)
+
+You can give all the pictures to the same model, compare the data of the last Dense layer, and calculate the loss function in that way.
+There is also a variable alpha. This is to use three pictures at a time and anchor A (self image), positive P (other self image), and negative N (fake image). The accuracy of the model depends on the size of max between 0 and the difference between the absolute values of A-P, and A-N, and adding alpha. High means that the model is more certain when the model recognizes or rejects the face.
+
+We can also use logistic regression
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8edd2c21-e2f8-4929-8481-ba8b5a36b236)
+
+_Neural style transfer_
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c8af9d88-bc28-446e-9e7b-43d0ac5d830a)
+
+We give images C and S to a model, and the output is a image C with the style of S applied to it.
+The loss function is to calculate the difference between C and S image features.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/c8142091-3a48-4344-8703-773857ab638e)
+
+And finally ~
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3ce6ea08-fa1b-48b4-afdf-b787195948fe)
+
+### Secondly, KCSE 2024
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d1df2ad8-e96d-45bd-90c7-f55c7f497468)
+
+I couldn't take notes a lot today, but the seminar I listened to was
+ChatGPT and LLM
+- LLM looking back from a software engineering perspective
+- Troubleshooting Use Cases in Software Engineering Using LLM
+- The Power of LLMs Across the Entire Section of Software Engineering
+- Presenter Park Jae-ho (Rainbow Brain Co., Ltd.)
+
+- - LLMs and Software Engineering - History, Landscape, Outlook
+- Presenter Shin Yoo - KAIST
+
+I got a lot of materials to read, but the most interesting things are:
+* [On the naturalness of software](https://people.inf.ethz.ch/suz/publications/natural.pdf)
+* [Large Language Models for Software Engineering: Survey and Open Problems](https://arxiv.org/abs/2310.03533)
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/01/day-30-day-1-face-recognition-neural.html)
+

From 830d5ca3404fe4431afef3d74aedd5d2ee854112 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sat, 27 Apr 2024 19:51:42 +0900
Subject: [PATCH 114/638] Update 2024-1-31-day-30.md

---
 _posts/2024-1-31-day-30.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-1-31-day-30.md b/_posts/2024-1-31-day-30.md
index 3aa2ff536..cba7c8308 100644
--- a/_posts/2024-1-31-day-30.md
+++ b/_posts/2024-1-31-day-30.md
@@ -1,7 +1,7 @@
 ---
 layout: post
-title: (Day 30) 
-categories: [deep-learning, cnn]
+title: (Day 30) KCSE 2024 day 1 and Face recognition & Neural Style transfer
+categories: [deep-learning, cnn, reading-research]
 ---
 
 ## Hello :) Today is Day 30!

From 8d0e02ba2a95c8f6ed47e3559fdf7867e3345181 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 22:37:16 +0900
Subject: [PATCH 115/638] Create 2024-2-1-day-31.md

---
 _posts/2024-2-1-day-31.md | 72 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 _posts/2024-2-1-day-31.md

diff --git a/_posts/2024-2-1-day-31.md b/_posts/2024-2-1-day-31.md
new file mode 100644
index 000000000..2d5b977e7
--- /dev/null
+++ b/_posts/2024-2-1-day-31.md
@@ -0,0 +1,72 @@
+---
+layout: post
+title: (Day 31) Natural Language Processing and KCSE day 2
+categories: [deep-learning, nlp]
+---
+
+## Hello :) Today is Day 31!
+A quick summary of today:
+* covered [sequence models](https://www.coursera.org/learn/nlp-sequence-models/home/week/2) from DeepLearning.AI's DL course
+* KCSE day 2
+
+### **Firstly, about the NLP part of the deep learning specialization**
+
+Today's part was about natural language processing models.
+The RNN model can process natural language well, but the Gated Recurring Unit (GRU) model can do it more effectively. The RNN model does not remember long-distance relationships well, so the GRU model overcomes its shortcomings. In addition, the RNN model also has the problem of vanishing gradients, so the GRU can overcome it. Vanishing gradient is a problem in which the gradient becomes too small and disappears if the distance is long when doing backprop.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/004f89e7-b1fd-4cbd-9b5a-be11df9d2a35)
+
+And, I learned a little about BRNN. BRNN is a neural network structure that considers both past and future information by processing input sequential data in both directions. Through this, it is possible to more effectively grasp a long range of dependencies and patterns and is usefully used for sequential data processing such as natural language processing and speech recognition. In general, BRNN is composed of two RNN components, each of which provides a comprehensive input expression by processing sequences in forward and reverse directions.
+
+Next ~ word embeddings.
+Whereas other methods may use indexing of words, or characters when making text generation models, we can use analogies and word vectors. With the first methods, we might end up with 50,000 classes (if we have 50,000 unique words), instead of that, we can use analogies to group words together. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5219072d-9e6b-4fed-864b-2a9f5c7c81e6)
+
+analogy with gender/royalty/age/food, and each value repesents how close it is to that 'group' - like -1 and 1 for male and female, and king (-0.95) - queen (0.97), because the difference in the words comes from the gender, whereas apple/orange are 0, because it is not related to gender at all
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1256a041-9a4f-45a9-b012-844176e8a738)
+
+Word embeddings encode semantic similarity between words. Words that are semantically similar tend to have similar vector representations. For example, in a well-trained word embedding model, the vectors for "king" and "queen" are expected to be closer to each other than the vectors for "king" and "car".
+For example, man to woman is the same as king to {x}. man is -1, woman is 1>  (-1) - (1) = -2 > so we look what value with king, gives a value very close to -2. To calculate this similarity, we can use 'cosine similarity'. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2c182626-3735-4b0e-954d-d39d7597f7a5)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/203e0d9a-c1bb-4152-98e6-5fddc9079919)
+
+Going back to softmax being used for classification in NLP models (i.e. text generation). 
+
+If there are too many classes, the computational cost may be very high, and since the probabilities for all classes are calculated, the computational cost may be high and the learning speed may be slow.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6e05dc37-6eff-46d6-b711-fc8c9df9efe5)
+
+So the hierarchical softmax method was created. Like a tree, if we have 10,000 classes, it tasks, is the class in the first or second 5000, then it splits by 2, and tasks again
+
+You can still use softmax, but you should think of it as a binary classification problem
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d09b90b5-0f5c-44a0-885c-4702ec06b2e3)
+
+Give pairs of words to the model, for example, 1 correct pair (orange + juice) and 4 incorrect pairs (orange king, orange book, etc), and the labels.
+
+### **As for KCSE 2024 day 2**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/17018213-cd71-4b12-9238-7c9475615561)
+
+[link to paper](https://dl.acm.org/doi/10.1145/3635706)
+
+The paper introduces SINVAD - a method for creating images to test DNNs with images that very closely resample the actual image. For example, the presentor showed images of numbers, 7 and 9, where both numbers could be seen in the image (a confusing image even for humans), and the point was to create images that try to deceive the machine.
+
+I think this is especially memorable. It was kind of fun
+"Enhanced Transformer Variant Autoencoder for Generating Various Appropriate Facial Responses". What got stuck in my head was that the authors want to create natural face videos that listen to a call, conversation. When we communicate, one person talks, the other listens, and the listener usually does a little smile, or nod, other small gestures with their face, and the authors looked into making a model that generates such faces, when given an input 'talker', so the result of the model is the 'listender'. In the ppt, they showed some funny show videos of the generated face, but I did not record it, and neither it is in the paper above. 
+
+Tomorrow is the last day and I'll present. I'll put the presentation materials in tomorrow's article. I prepared in Korean, but I'll do it in English to present a longer content a little faster tomorrow. Still, it seems like the participants speak English , so I hope they present it without any problems. If there is anything I don't understand well, I'll explain it in Korean.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/07b63014-476a-4faf-a676-584d88caffe4)
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/02/day-31-day-2.html)

From 7bbce502b5ed58c7860a89f10263026bfc7890d7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 22:49:54 +0900
Subject: [PATCH 116/638] Create 2024-2-2-day-32.md

---
 _posts/2024-2-2-day-32.md | 54 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 _posts/2024-2-2-day-32.md

diff --git a/_posts/2024-2-2-day-32.md b/_posts/2024-2-2-day-32.md
new file mode 100644
index 000000000..ec1ef6de2
--- /dev/null
+++ b/_posts/2024-2-2-day-32.md
@@ -0,0 +1,54 @@
+---
+layout: post
+title: (Day 32) Natural Language Processing and KCSE day 2
+categories: [deep-learning, nlp]
+---
+
+## Hello :) Today is Day 32!
+A quick summary of today:
+* covered the last part of the [deep learning specialization](https://www.coursera.org/specializations/deep-learning)
+* presented at the KCSE 2024
+
+### **Firstly, regarding the DL specialization**
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9ea6b7c9-2058-4b75-844f-3044629e0c43)
+
+The last part of Sequence models is Language transformers, which I haven't learned so far, but I found out that it includes a variety of features for NLP.
+
+Transformers excel at capturing context through a self-attention mechanism, which can add to the importance of other words within the sequence depending on contextual relevance, effectively modeling long range dependencies and finding detailed relationships within the text.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/4d33abd5-0308-4280-8b47-96c707488039)
+
+The self-attention mechanism relates each word to the query, key, and value vector, so key helps identify the word most relevant to the query for the word. Value contributes to the final expression. By dynamically adjusting the expression according to the context, the self-attention mechanism enables a richer understanding of word meanings and relationships in sequential columns, thereby improving the ability of the transformer network in natural language processing tasks.
+
+You can stack the self-attention "layers", and you get multi-head attention. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d0a83c29-d278-4ebf-b377-345adb8ab31a)
+
+The multi-head attention mechanism can be briefly described as for loop over the self-attention mechanism. Each calculation step is called a head, and each head calculates a self-attention for the input sequence. These calculations are performed in parallel and the results are combined to create a multi-head stack. Through this mechanism, multiple questions can be asked for each word, and much richer and more effective expressions of each word can be learned. For example, in the case of a Jane want to France in September sentence, when can understand the French word as a question, and when - September, who - Jane, etc - can be understood as context.
+
+This was just just an introduction to transformers, but I think I need to study transformers well to understand NLP more deeply
+
+### **As for my presentation**
+
+My slides can be found [here](https://docs.google.com/presentation/d/1XcA4xo-84KGQ1FuE_BfhIRaw_aqGzq-nZ4ZYtmjxqhc/edit#slide=id.p)
+
+And the short paper - [here on page 115](https://drive.google.com/file/d/17sTTsxcg896LtaU08NPfXEyoDA4HpV4o/view)
+
+My topic is a review of ChatGPT utilization: An analysis of prompt engineering strategies applied to various domains, but I just wrote a review of papers about prompt engineering.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e17a893a-3c6a-4c4c-80e1-f2b86e7fdb95)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/74be7ff5-96aa-427a-927e-1f74755e6cae)
+
+You can see the prompt strategy of this study in the table above, and my favorite strategy is Oh, 2023 paper. Creating a prompt strategy to solve math problems over there, the strategy was 91% accurate by solving the problems of nine Korean math textbooks with the "Role-Rule-Example-Process" prompt. Usually, ChatGPT is not good at math, but it does well with the proposed prompt. 
+
+I think today's presentation went well. I was a bit nervous at the start but became find after 2-3 mins. 
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/02/day-32-language-transformers-day-3-my.html)

From d6c8d67bb08d0f5b473a02c1993e1f745a29150c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 22:50:23 +0900
Subject: [PATCH 117/638] Update 2024-2-1-day-31.md

---
 _posts/2024-2-1-day-31.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-1-day-31.md b/_posts/2024-2-1-day-31.md
index 2d5b977e7..7a6fb07d3 100644
--- a/_posts/2024-2-1-day-31.md
+++ b/_posts/2024-2-1-day-31.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 31) Natural Language Processing and KCSE day 2
-categories: [deep-learning, nlp]
+categories: [deep-learning, nlp, reading-research]
 ---
 
 ## Hello :) Today is Day 31!

From 92bacb3d9330be0c5883f74cfd3b6a89bc7051eb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 22:51:28 +0900
Subject: [PATCH 118/638] Create 2024-2-3-day-33.md

---
 _posts/2024-2-3-day-33.md | 40 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 _posts/2024-2-3-day-33.md

diff --git a/_posts/2024-2-3-day-33.md b/_posts/2024-2-3-day-33.md
new file mode 100644
index 000000000..881c438f6
--- /dev/null
+++ b/_posts/2024-2-3-day-33.md
@@ -0,0 +1,40 @@
+---
+layout: post
+title: (Day 33) Natural Language Processing and KCSE day 2
+categories: [deep-learning, nlp]
+---
+
+## Hello :) Today is Day 33!
+A quick summary of today:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From 5940c7ab54c4084e20cbfa368c895522d9a9c2aa Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 22:52:01 +0900
Subject: [PATCH 119/638] Update 2024-2-2-day-32.md

---
 _posts/2024-2-2-day-32.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-2-day-32.md b/_posts/2024-2-2-day-32.md
index ec1ef6de2..c85725900 100644
--- a/_posts/2024-2-2-day-32.md
+++ b/_posts/2024-2-2-day-32.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 32) Natural Language Processing and KCSE day 2
+title: (Day 32) Language Transformers and KCSE day 3
 categories: [deep-learning, nlp]
 ---
 

From 1e18c46b0d0e3cc2074dbef5367f1b05d6122c10 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 23:03:35 +0900
Subject: [PATCH 120/638] Update 2024-2-3-day-33.md

---
 _posts/2024-2-3-day-33.md | 40 +++++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/_posts/2024-2-3-day-33.md b/_posts/2024-2-3-day-33.md
index 881c438f6..45a70f337 100644
--- a/_posts/2024-2-3-day-33.md
+++ b/_posts/2024-2-3-day-33.md
@@ -1,40 +1,64 @@
 ---
 layout: post
-title: (Day 33) Natural Language Processing and KCSE day 2
-categories: [deep-learning, nlp]
+title: (Day 33) Tensorflow deployment specialization and a webapp for recognizing the Korean alphabet
+categories: [deep-learning, cnn, model-deployment]
 ---
 
 ## Hello :) Today is Day 33!
 A quick summary of today:
+* covered [TensorFlow: Data and Deployment Specialization](https://www.coursera.org/specializations/tensorflow-data-and-deployment)
+* made a simple webapp for recognizing Korean alphabet characters
 
+### Firstly - learning how to deploy tensorflow models
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/eb74ea8c-6fe4-4b2c-a416-2276123ae7cf)
 
+I learned how to train a machine learning model in the browser. I can do it right away with tensorflowjs.
 
+You can train it directly in the browser
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/58fa4de2-9cdf-477e-a719-8cce74d3c08e)
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d3900f2b-0fe3-4762-9226-975bae1a1e5b)
 
+You can load a model developed in a jupyter notebook too.
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/b6ba93ef-1ce5-4099-bc7b-97543a8ebd14)
 
+One of the assignments was to create a rock-paper-scissors model using my laptop's camera.
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/43816772-719b-450c-a567-3a5071a6db52)
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/23801196-8c25-4559-a7d3-4621d38a5049)
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d7182aa0-e4fb-4413-9b2d-3bd1c154e150)
 
+(no idea why balloons appeared on that last one)
 
+### Secondly, deployed a simple Hangul classification model
 
+I found out if I was going up the model with js today, so I wondered if I could actually come up with an idea I've made before. 
+[Hangul classification model](https://www.kaggle.com/code/divakaivan12/beginner-hangul-classification-using-tensorflow) < I made it when I first learned machine learning (Summer 2023~) and I wanted to make a simple Hangul character classification app or website based on this model even if I didn't understand 100% what I did at that time. But I didn't have much time, so I gave up. Today, this idea was born again.
 
+After constructing a simple model, I made a website with html and js
+The accuracy and loss of the model are as follows
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/be4004eb-dfb0-4984-bc16-42a83303cbc3)
 
+I had to deal with some problem with canvas from my browser which was frustrating, but I finally managed to get it work.
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/9b9502b1-040a-4810-a614-d71a956c1e50)
 
+The accuracy and loss of the model were not so good, so I tested it with the characters I wrote.
 
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/05beaa4c-dbf3-4979-a8ea-da5bb2d4774e)
 
+It got 20 out of 30. Not too bad haha. 
 
+<br/>
 
+That is all for today!
 
+See you tomorrow :)
 
-
-
-
-
-
-
+[Original post in Korean](https://50daysml.blogspot.com/2024/02/day-33-tensorflow-deployment.html)

From 03f55eab30fb0079e8f8ecae87d2faa7ee14a89f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 23:13:50 +0900
Subject: [PATCH 121/638] Create 2024-2-4-day-34.md

---
 _posts/2024-2-4-day-34.md | 67 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 _posts/2024-2-4-day-34.md

diff --git a/_posts/2024-2-4-day-34.md b/_posts/2024-2-4-day-34.md
new file mode 100644
index 000000000..19affa3cc
--- /dev/null
+++ b/_posts/2024-2-4-day-34.md
@@ -0,0 +1,67 @@
+---
+layout: post
+title: (Day 34) Deploying ML models and TensorBoard
+categories: [model-deployment]
+---
+
+## Hello :) Today is Day 34!
+A quick summary of today:
+* today I covered the final part of [TensorFlow: Data and deployment specialization](https://www.coursera.org/specializations/tensorflow-data-and-deployment)
+
+When creating an app for iOS/Android, it would be nice to follow the following step-by-step guide.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e86a663d-273a-4532-bc24-c576f41ab64c)
+
+ Set up the model in the desired platform > process input provided by the user > pass the input to the model > get results and show them
+
+Then, I learned about tensorflow datasets (tfds).
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/283e19e4-0627-4151-81ab-a18f9b3a763f)
+
+Cache can be used to improve performance
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/befe1f66-27b1-4d5a-9329-cef992514364)
+
+Memory cache provides faster access time and fits well with small to medium-sized datasets. Disk cache provides persistence and scalability and fits well with large datasets
+
+Next, I learned a bit about TensorBoard
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0d462def-ba15-4a5a-bf4c-945845763f8e)
+
+You can see the mean and standard deviation for each epoch on the TensorBoard (Cool!)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/1e9519e3-7955-4d11-be98-04c3b93cef9a)
+
+In addition, if you create a classification model, you can also see the confluence matrix of each epoch
+
+For example, epoch 1:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0375d3a9-86b8-4b5a-a608-da25539d78b5)
+
+Epoch 3:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2b9ce4f7-b5db-4c7f-878a-eafd9c7fdd72)
+
+Lastly, learned about _Federated learning_
+
+Federated learning is a technology that learns a global model in a decentralized situation in which a central server and local clients cooperate with each other. It has advantages in terms of data privacy and communication efficiency, and it requires solutions to problems such as completely decentralized learning, and there are several challenges such as model learning in situations where data is unevenly distributed.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a19c5ab9-2179-42b4-905a-4fa889813c56)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/aa42d8d2-cd91-417b-b7a7-77470eaa0524)
+
+Actually, I was going to certify [TF developer](https://www.tensorflow.org/certificate) today, but my card kept getting rejected :/ (I emailed Google)
+
+If I cannot do that, my next options are:
+* [TensorFlow: Advanced Techniques Specialization](https://www.coursera.org/specializations/tensorflow-advanced-techniques)
+* [opencv university course (computer vision)](https://opencv.org/university/)
+
+I have to decide tomorrow
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/02/day-34-deploying-ml-models-tensorboard.html)

From acb1b98cd91c917caf6c6768f92fd871fe6c4596 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 23:26:04 +0900
Subject: [PATCH 122/638] Create 2024-2-5-day-35.md

---
 _posts/2024-2-5-day-35.md | 98 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100644 _posts/2024-2-5-day-35.md

diff --git a/_posts/2024-2-5-day-35.md b/_posts/2024-2-5-day-35.md
new file mode 100644
index 000000000..ba4015523
--- /dev/null
+++ b/_posts/2024-2-5-day-35.md
@@ -0,0 +1,98 @@
+---
+layout: post
+title: (Day 35) TensorFlow Advanced techniques
+categories: [deep-learning]
+---
+
+## Hello :) Today is Day 35!
+A quick summary of today:
+* [covered Tensorflow Advanced techniques specialization](https://www.coursera.org/specializations/tensorflow-advanced-techniques)
+
+Some of the material is summarised below ^^
+
+### Sequential vs Functional API
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8fef0b0d-76b8-4eb0-96c0-f3c5ff9cc6b1)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/34683701-0388-4bf3-87f3-5ae84423359d)
+
+When using a functional API, the input layer must be defined directly and each layer must be attached at the end of the next layer. And the input and output of the model must also be defined. Because a model composed of functional APIs can have various inputs and outputs
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6f16e507-3075-4645-83de-9f76cecf0df5)
+
+Functional APIs allow complex models to be configured
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/2047cec9-b0d8-4103-8110-40da089758c2)
+
+Rather than that model, I'll describe it as a simpler model
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/84b2d7f3-cc17-463b-887c-4210a4af7a4d)
+
+In addition, the performance and loss functions of each output can be defined separately
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/45de1ddc-cbaa-4262-b9a2-d85d9b27ac21)
+
+### Siamese network model
+
+The Simesen network is a model with two inputs and one output
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/089bb742-79a7-4d2d-a322-c9471ae70b0e)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3ac8cb86-2623-49d7-a6d7-fbd8ccf6d38e)
+
+For example, you can make a model that can compare clothes
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/23ee0158-c24e-4659-9602-e2d2acf7df94)
+
+The T-shirts were similar, so I classified them as 1.0 (conversely, it would have classified them as 0).
+
+### Custom loss function
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/0ace2766-180b-4606-8553-5ced5b1f75ed)
+
+If the absolute value of a is less than the threshold we defined, 1/2 * a**2, conversely, threshold * (abs(a) - 1/2 * threshold)
+Simple version can be used as follows
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/f5dc1aa3-8cd0-4aac-98d4-7857bb3abf70)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/af761bc9-81cc-4957-93af-35e42b5da08b)
+
+Still, there is a better way to organize threshold values (with Class)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/8ee9f041-e1d2-45a3-a418-fdceb0c8a0c6)
+
+### Custom layers
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a0059b93-fbf3-4d3e-9a44-2373c2131b56)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/547c505c-d6d4-4da2-a83f-c6d94eb17b0b)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/69b6a530-83c8-4ea9-8174-03143341792b)
+
+### Custom models
+
+We can make custom/complex models using the functional API
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/70853cff-3d0e-4169-9894-1edee5504491)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/3186d821-f26c-4a43-bfbe-0bbd6a80c266)
+
+But we can do more complicated models than that
+
+This is an example of ResNet. 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/5239703b-ff2f-4e2d-91ff-8829c8ab4a40)
+
+I first learned about ResNet on Day 28, but I didn't understand how to do it. But today I found out how functional api can be used.
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/cf609946-baed-4e09-913d-c007cb261e21)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a3b9e199-5ba5-433d-afc2-f9fdf6080baf)
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/02/day-35-tensorflow-advanced-techniques.html)

From 00b4f25394f64044bb3350457393d499aa99f3f2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Tue, 30 Apr 2024 23:35:51 +0900
Subject: [PATCH 123/638] Create 2024-2-6-day-36.md

---
 _posts/2024-2-6-day-36.md | 83 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 _posts/2024-2-6-day-36.md

diff --git a/_posts/2024-2-6-day-36.md b/_posts/2024-2-6-day-36.md
new file mode 100644
index 000000000..23a6dcfca
--- /dev/null
+++ b/_posts/2024-2-6-day-36.md
@@ -0,0 +1,83 @@
+---
+layout: post
+title: (Day 36) Intro to PyTorch
+categories: [deep-learning,cnn]
+---
+
+## Hello :) Today is Day 36!
+A quick summary of today:
+* [IBM's DL with PyTorch](https://www.coursera.org/learn/deep-neural-networks-with-pytorch)
+* [PyTorch Project: Handwritten Digit Recognition](https://www.youtube.com/watch?v=vBlO87ZAiiw&ab_channel=NeuralNine)
+* [Facial Expression Recognition with PyTorch](https://www.coursera.org/projects/facial-expression-recognition-with-pytorch) and [Deep Learning with PyTorch : Object Localization](https://www.coursera.org/projects/deep-learning-with-pytorch--object-localization)
+
+Tesla autopilot and ChatGPT apparently use PyTorch, so for some time I have been wanting to see what PyTorch is like. Given that I wanna go deeper into computer vision and self-driving cars, I wanted to "get my hands dirty". 
+
+### IBM's DL with PyTorch
+
+I did IBM's DL with PyTorch and it was good, but not very practical. It assumes I have no DL knowledge, but it was not that bad, just a refresher on loss, activations, back and forward prop, gradient descent, etc. 
+
+PyTorch's library is something that needs to be defined more directly than TensorFlow. I thought I should apply it to something simple.
+
+### PyTorch Project: Handwritten Digit Recognition
+
+I decided to do this tutorial project.
+
+In fact, when I first studied machine learning in the summer of 2023, this person's channel was very helpful. I love tutorial projects for beginners such as kn, linear regression, logistic regression, and decision tree.
+
+Difference with TensorFlow
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/64b316b6-4b4b-4fff-b2e4-2e7b13e42321)
+
+You have to load data with the DataLoader 
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d5fadd00-3526-4f70-88a9-7a4516d6e11b)
+
+Explicitly define the model, and each layer. (forward prop)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a812e30f-5f60-4892-9aec-414981aca4b1)
+
+When training, backprop should be defined directly. Also, the model should change to the training state with model.train() (when testing/eval later, model.eval())
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d6362ec6-807d-49d4-aacf-272887d3ad3a)
+
+You have to calculate the loss and express the loss and accuracy in print
+
+### Facial Expression Recognition with PyTorch and Deep Learning with PyTorch : Object Localization
+
+Here, I made a model using transfer learning. The second project consisted of a bounding box model of vegetables or apples. The first model was more interesting. These two projects used an already constructed model, and I understood a little more deeply how to do training and validation with Pytorch
+
+The facial expression model used the following data
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/6fe18f0c-5514-4bef-b0c8-c1dd382b270f)
+
+also did data augmentation here (with transforms.Compose)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/a4adebf0-939e-41ac-aba0-42ef57788109)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d1132bed-72db-4c0d-8312-2d2d6f6ea5e2)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/79d7088c-c960-4495-a94a-60f9cbb73c93)
+
+Must include init and forward functions in model class structure
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/e41c6f1d-e368-497c-b068-fe5eb8924717)
+
+Training:
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/59776215-ee97-4788-9364-5e40c86a63a5)
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/26b545ca-332c-4c6b-99e1-c7fa00cf3d84)
+
+If you look at the results, you can see the probability of class of each image
+
+![image](https://github.com/ivanstudyblog/ivanstudyblog.github.io/assets/167014511/d3e8acb6-fd22-48dc-9883-0079fff6274a)
+
+Today's study was interesting, but tomorrow, I'll make my own Pytorch model with different data!
+
+<br/>
+
+That is all for today!
+
+See you tomorrow :)
+
+[Original post in Korean](https://50daysml.blogspot.com/2024/02/day-36-intro-to-pytorch.html)

From 1fd6b20b9d57aa6baf6022bc1aea58478c56e1fd Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 19:30:01 +0900
Subject: [PATCH 124/638] Create 2024-2-7-day-37.md

---
 _posts/2024-2-7-day-37.md | 85 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 _posts/2024-2-7-day-37.md

diff --git a/_posts/2024-2-7-day-37.md b/_posts/2024-2-7-day-37.md
new file mode 100644
index 000000000..d0677b648
--- /dev/null
+++ b/_posts/2024-2-7-day-37.md
@@ -0,0 +1,85 @@
+--- 
+layout: post 
+title: (Day 37) PyTorch traffic sign classification and detection model
+categories: [deep-learning,cnn] 
+--- 
+
+## Hello :) Today is Day 36! 
+A quick summary of today:
+* [Simple PyTorch animal classification](https://www.kaggle.com/code/divakaivan12/simple-pytorch-animal-classification)
+* Traffic sign classification (and localisation)
+
+**Firstly**
+
+to put knowledge into practice I decided to try to make an animal classification model
+
+There are some limitations with Kaggle's GPU so I had to tone down the picture sizes, the augmentations and model complexity.
+
+So ~ 
+data augmentation
+
+![image](https://github.com/user-attachments/assets/786406e4-44d5-4949-9a13-05a0890f1e65)
+
+Image example:
+
+![image](https://github.com/user-attachments/assets/12f72eba-592d-48ec-834c-b2c1b36dc97b)
+
+The categories: cat, dog, elephant, horse, lion
+
+I divided the data into batches:
+
+![image](https://github.com/user-attachments/assets/d174fdc4-f251-42b7-9ee4-739784ee9b61)
+
+The model is:
+
+![image](https://github.com/user-attachments/assets/6ef88088-efd6-4901-aa26-5c4aa91a28ae)
+![image](https://github.com/user-attachments/assets/e26bf73d-4401-4a27-b4c8-e36ce0c6c5e9)
+
+The training took a lot of time; below you can see the last epoch's results
+
+![image](https://github.com/user-attachments/assets/6cdd3091-54f7-4cfa-bd8d-1cfd4fe27ec2)
+
+Example classification:
+
+![image](https://github.com/user-attachments/assets/cb2aa10d-d4b3-4b01-a37f-b21e95df8ae7)
+![image](https://github.com/user-attachments/assets/886c769c-7079-4115-b8a7-c0dd20b5a675)
+![image](https://github.com/user-attachments/assets/b723efe2-a542-42db-8c89-1ed70a3e5c8d)
+
+(there are some weird classifications too haha)
+
+**Next ~ **
+
+About the traffic sign classification and localisation model
+
+[Here](https://www.kaggle.com/code/divakaivan12/traffic-sign-classification-and-bbox-model-pytorch?scriptVersionId=162144835) is a link to the notebook on Kaggle.
+
+The model's dataset has 43 classes (here are 21 of them)
+
+![image](https://github.com/user-attachments/assets/8a1bdbe9-e1e3-4b76-841e-5f14fff2242d)
+![image](https://github.com/user-attachments/assets/589c7158-4a2d-46e6-961d-fe72d3eefb43)
+
+The image size varied a lot so I standardised it
+
+![image](https://github.com/user-attachments/assets/b18f6921-8afc-4a96-9add-b4295db3d01b)
+
+Model architecture:
+
+![image](https://github.com/user-attachments/assets/e6b9b5e5-bc67-40a4-8838-e3658278af30)
+![image](https://github.com/user-attachments/assets/11869595-922c-4482-865c-fc5597d754df)
+![image](https://github.com/user-attachments/assets/93b9e74c-9ae7-49bf-a3ca-f059baac053e)
+
+I tried various architectures, but the accuracy was not that good, so online I found *AlexNet* and wow ~
+
+![image](https://github.com/user-attachments/assets/15b8f9ce-bde5-4868-9c17-6cd3be215081)
+
+Example:
+
+![image](https://github.com/user-attachments/assets/985708d2-be7c-466c-901c-aa69287bdfbf)
+
+I applied what I learned yesterday to this model according to the vegetable/apple bounding box model from yesterday, but the accuracy and loss won't decrease even after training, and I'll try again tomorrow!!!
+
+Now it's 2.37am and I will go to bed ~
+
+That is all for today!
+
+See you tomorrow :)

From 38cc17fd3dc58056b7de87debe85aed498c718fb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 19:40:29 +0900
Subject: [PATCH 125/638] Create 2024-2-8-day-38.md

---
 _posts/2024-2-8-day-38.md | 128 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 _posts/2024-2-8-day-38.md

diff --git a/_posts/2024-2-8-day-38.md b/_posts/2024-2-8-day-38.md
new file mode 100644
index 000000000..5d1000002
--- /dev/null
+++ b/_posts/2024-2-8-day-38.md
@@ -0,0 +1,128 @@
+---
+layout: post
+title: (Day 38) 
+categories: [cnn,applying-knowledge]
+---
+
+## Hello :) Today is Day 38!
+A quick summary of today:
+* [Traffic sign detection model with PyTorch](https://www.kaggle.com/code/divakaivan12/traffic-sign-classification-and-bbox-model-pytorch)
+* Learned about how Telsa's autopilot works [[1](https://slideslive.com/38917690/multitask-learning-in-the-wilderness), [2](https://youtu.be/hx7BXih7zx8), [3](https://youtu.be/j0z4FweCy4M)]
+
+
+These days I stay till late, studying and then writing these blog posts ~ 
+
+Firstly, the model that I struggled to make yesterday - a model where I give it a picture of a sign, and what I get back is - what kind of sign it is + the sign's bounding box. And today, I am happy to say that I got it :) 
+
+The dataset I used looks like this:
+
+![image](https://github.com/user-attachments/assets/07596144-dc29-461b-9b50-e145265b565d)
+
+For the processed version, I took the bounding box (bbox) data, the ClassId and the path. 
+Augmentations - only resize to 112x112 (3 channels by default). Used the albumenations library since it rescales the bounding boxes alongside the image, so no extra resizing for the bbox size was needed
+
+![image](https://github.com/user-attachments/assets/f72e60c3-04ce-44b7-b77d-b259291f8c44)
+
+Final train and val lengths were 31367 and 7842, after which I batched with size 32.
+An example image from the data:
+
+Class 14 = Stop sign![image](https://github.com/user-attachments/assets/8630facd-37ef-4a96-b939-c885d2fd1213)
+
+The classification model was the same as yesterday, a variation of AlexNet. 
+
+![image](https://github.com/user-attachments/assets/6d36e3a6-b8f5-4591-8485-06cac772aa7b)
+
+For the bbox regression model, I used a pretrained efficientnet_b0, and made the final "CombinedModel"
+
+![image](https://github.com/user-attachments/assets/92837be4-5546-46de-a051-fa86f2f19c22)
+
+For the losses:
+- MSE for bbox
+- cross entropy for the classification
+- Adam optimizer with 0.001 learning rate
+
+![image](https://github.com/user-attachments/assets/04b18eee-a536-4537-a49b-1a355d69793b)
+
+Trained for 10 epochs
+
+![image](https://github.com/user-attachments/assets/c12cbd66-13c8-4ff7-b960-da2c0295b09e)
+
+Seems like good results
+
+Finally, showing some examples from the val data:
+
+![image](https://github.com/user-attachments/assets/af7c900a-9bcf-4b2f-b48f-be51ce99368d)
+![image](https://github.com/user-attachments/assets/f0953aa8-3596-49c3-8dde-fe21b0890515)
+![image](https://github.com/user-attachments/assets/4f9df138-207c-4730-adff-71dc0c741125)
+
+Maybe the data is not good, or because of the resizing, some of the images are very hard even for humans to distinguish.
+
+In the end, I am glad, this is my first more complex model. And I also used transfer learning for the 1st time too. Of course this is a very simple, give an image of a sign, get back its classification and bounding box, model but going forward making it more complex, making multi-object classification + bbox models is a good idea. 
+
+For the second interesting thing today, I saw Andrej Karpathy's name again and decided to search for some of his talks - turns out there are quite a few (nice!).
+
+[1st one](https://slideslive.com/38917690/multitask-learning-in-the-wilderness) - At ICLM 2019 about Tesla's AI, and [2nd one](https://youtu.be/hx7BXih7zx8) - AI for Full-Self Driving at Tesla
+
+Both around 30 minutes, and covered similar materials, which I will try to summarise below through some screenshots I took
+
+![image](https://github.com/user-attachments/assets/6d972f99-9130-4083-bf8f-0f54b623810f)
+
+Some other companies use a 'lidar' that beams waves from its top, and then maps the area around it, so that the vehicle can see. But at Tesla - they use just camera vision. More specifically - 8 cameras. Each camera registering new information every 27ms.
+
+![image](https://github.com/user-attachments/assets/f6b59a61-8dcb-40d9-bb0f-68e87504685a)
+
+At the start, their model was very simple - draw rectangles around cars:
+
+![image](https://github.com/user-attachments/assets/051592e8-5505-44fc-97f0-9ff52392ccb7)
+
+But even with cars, this quickly became impractical, as cars can be in many different shapes:
+
+![image](https://github.com/user-attachments/assets/59f6e2dc-8ba3-415d-bdc3-061bf5628e55)
+
+So their training process became: get data from cars -> train -> deploy -> get new inaccuracies -> re-evaluate -> deploy again etc.
+
+![image](https://github.com/user-attachments/assets/51a4e5af-f9df-4860-a7fa-5f1c5b522b08)
+
+So, next the model started to have a variety of inputs and classes (other vehicles, road markings, people, signs, traffic lights, environment, crosswalks, lane lines, etc.)
+
+![image](https://github.com/user-attachments/assets/351ea300-27e5-4b47-8d38-24eee06ab666)
+
+And identifying each task has additional sub-tasks. For examples, vehicles can come in many shapes and forms
+
+![image](https://github.com/user-attachments/assets/5644ce32-3f00-47b4-823b-f4fd1b545dca)
+
+Apparently this is 2020, they used to have 48 networks in total. 
+
+![image](https://github.com/user-attachments/assets/124b0ad6-75e7-47ed-9991-670f25aeeda8)
+
+Given the complex nature of the problem, calculating loss, proper training time, regularization is hard because some tasks have a lot of info (i.e. seeing cars) but other might have very little data. 
+
+He gave a funny example, early stopping is fine when we have 1 task:
+
+![image](https://github.com/user-attachments/assets/e8e8630d-1045-4ba9-9775-8e7907b0d6e7)
+
+But what happens when you have *many* ?
+
+![image](https://github.com/user-attachments/assets/7400dcef-9429-448f-b619-b9a176797e02)
+
+Some tasks will overfit faster than others because of more/less data, or are easier. The problem becomes how to manipulate the oversampling rates, task weights or reguralizations so that these lines' minima is at the same spot. 
+
+His [3rd](https://youtu.be/j0z4FweCy4M) talk - Tesla AI day 2021
+
+He mentioned a little about the structure. The take 1280x960 sized data from the cameras.
+
+![image](https://github.com/user-attachments/assets/2d90f7ee-ef53-4563-a7e8-b58d2c8aeb10)
+
+Going from more bigger picture with less channels, to a more detail and abstract picture with 512 channels. 
+
+A high level explanation would be, these pictures are put afterwards to each task, and afterwards the model tries to understand the area it is in. They use 'multi-camera' fusion. For example, 1 long truck may appear on multiple cameras, and instead of each camera trying to identify the truck on its own, the cameras can pass info so that the whole truck is identified. He gave examples with cars passing by, because when that happens, multiple cameras will see the car and we will get multiple recognitions of that car (orange) from each single camera, but by using multi-camera, the vehicle can have 1 smooth represetation.
+
+![image](https://github.com/user-attachments/assets/359af8b7-2cda-417b-b418-bc203034cc1d)
+
+Also, to remember what the car saw earlier, they started using some kinds of RNN so that the car can know its surroundings even if it does not see everything at the moment. For example, there is a bike 2 lanes to the right, but currently a truck is passing by and the cameras are not seeing the bike, but thanks to the 'memory' the car will know that the bike is there (or could be moving alongside). 
+
+I am not able to understand completely such complex structures yet, but seeing the path ahead is exciting! I need to start reading research papers to broaden my knowledge on models and what is happening in academia. 
+
+That is all for today!
+
+See you tomorrow :) 

From 4c8939855582093b0403dfdc77b82316398ca822 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 19:40:48 +0900
Subject: [PATCH 126/638] Update 2024-2-7-day-37.md

---
 _posts/2024-2-7-day-37.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-7-day-37.md b/_posts/2024-2-7-day-37.md
index d0677b648..91081da8e 100644
--- a/_posts/2024-2-7-day-37.md
+++ b/_posts/2024-2-7-day-37.md
@@ -4,7 +4,7 @@ title: (Day 37) PyTorch traffic sign classification and detection model
 categories: [deep-learning,cnn] 
 --- 
 
-## Hello :) Today is Day 36! 
+## Hello :) Today is Day 37! 
 A quick summary of today:
 * [Simple PyTorch animal classification](https://www.kaggle.com/code/divakaivan12/simple-pytorch-animal-classification)
 * Traffic sign classification (and localisation)

From 41906f4220a0dad7b1762601ec9cac646cb536a8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 19:48:49 +0900
Subject: [PATCH 127/638] Create 2024-2-9-day-39.md

---
 _posts/2024-2-9-day-39.md | 66 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 _posts/2024-2-9-day-39.md

diff --git a/_posts/2024-2-9-day-39.md b/_posts/2024-2-9-day-39.md
new file mode 100644
index 000000000..2da623f6b
--- /dev/null
+++ b/_posts/2024-2-9-day-39.md
@@ -0,0 +1,66 @@
+---
+layout: post
+title: (Day 39) Reading papers of powerful CNN models + going back to basics (+ some more Andrej Karpathy lectures)
+categories: [applying-knowledge, reading-research]
+---
+
+## Hello :) Today is Day 39!
+A quick summary of today:
+* Read a few research papers
+* Watched Stanford's CS231n Winter 2016 lectures delivered by [Andrej Karpathy](https://youtu.be/NfnWJUyUJYU?list=PLkt2uSq6rBVctENoVBg1TpCC7OQi31AlC)
+* Drew a nice "poster" for my wall thanks to [Samson Zhang](https://youtu.be/w8yWXqWQYmU)
+
+First I read about some regularization technique that I saw on paperswithcode called - [Label Smoothing](https://paperswithcode.com/method/label-smoothing). I was curious what other options are there besides Dropout and L2, and that was amongst the top in the regularization category. 
+
+It involves smoothing the one-hot encoded target labels by replacing some of the elements with values slightly different from 0 and 1. 
+* For example, in a classification task with 5 classes, a sample belonging to class 3 would have the target label [0, 0, 1, 0, 0]. 
+* Instead of using hard 0s and 1s in the one-hot encoded vectors, label smoothing replaces them with slightly adjusted values. For example, instead of [0, 0, 1, 0, 0], the smoothed label for the same sample might be [0.05, 0.05, 0.85, 0.05, 0.05]. 
+
+It helps with overfitting and does not allow our model to get overconfident when it classifies. 
+
+After that I read about AlexNet, ResNet and VGG which luckily (a nice coincidence) appeared in Andrej Karpathy's lecture about popular CNN models. 
+
+He showed a comparison of the three models on the ImageNet dataset
+
+![image](https://github.com/user-attachments/assets/de42b5f5-57e0-4c1a-8384-597d6faa29cb)
+
+In terms of depth: AlexNet: 8 layers, VGG: 16 or 19 layers, ResNet: 50 to 152 layers
+
+Parameters: AlexNet: 60 million, VGG: 138 million, ResNet: 25.6 million (ResNet-50)
+With deeper models the idea was that the computational complexity will increase (as seen when comparing AlexNet and afterwards VGG), but when ResNets arrived they incorporated 'skip-connections', even tho there are way more layers than VGG for example, (amongst other reasons) help with gradient flow which allows for smoother backprop and battles vanishing gradients. 
+
+Side note! In the papers I saw how models get compared and I saw top-1 and top-5 error, not knowing what it meant I just accepted it at face value. But in his lecutres, Dr. Karpathy mentioned that top-5 is, if the 1st guess is not correct, is the ground truth class in the top 5 (and here we are talking about ImageNet which has 1000 classes).
+
+Some other interesting notes I took from some of the lectures:
+
+![image](https://github.com/user-attachments/assets/55dfd103-fd4d-45a2-9d82-704dc7b0ee87)
+
+there is some kind of multi-class SVM loss, but these days cross entropy might be more powerful because during my studies around multi-class classification, i have not heard about the SVM loss. 
+
+I wanted to go over backprop basics and derivatives. Andrej Karpathy has a nice way of explaining what happens.
+
+![image](https://github.com/user-attachments/assets/f99f4dab-dc6f-4e58-a54b-3022d7b7f2b9)
+![image](https://github.com/user-attachments/assets/3a544a3b-b594-435c-b7df-613b6253cc03)
+
+It is a long series of chain rule calculations. 
+
+Another cool thing. What happens throughout a CNN.
+
+![image](https://github.com/user-attachments/assets/43037eb1-6c92-4b7b-bcf7-e6bcf080819b)
+
+For example along the network, in this part of the conv6 layer - shirt wrinkles are recognised. You can see in the block under the camera.
+
+![image](https://github.com/user-attachments/assets/eb25ff89-56b3-42d9-9825-576da97575b6)
+
+In this part - faces (kind of). 
+
+Finally, as for my nice "poster". On my wall, in front of my desk, I had some Hanja letters from when I studied for a Hanja exam a while ago. But I wanted to change with something more relevant to my current studies. 
+
+![image](https://github.com/user-attachments/assets/31873740-cce8-4211-9b7e-34aa36f2ce94)
+
+It goes over a basic MNIST 2 layer NN, forward prop > backward prop > optimization.
+It does not include detailed info and misses different loss functions (for examples), also regularization, batch norm, but I will look to update it :)
+
+That is all for today!
+
+See you tomorrow :) 

From 34e1129b61a5fe48cceab9ae01dc27c1b425f567 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 19:49:17 +0900
Subject: [PATCH 128/638] Update 2024-2-8-day-38.md

---
 _posts/2024-2-8-day-38.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-8-day-38.md b/_posts/2024-2-8-day-38.md
index 5d1000002..2fcb317d3 100644
--- a/_posts/2024-2-8-day-38.md
+++ b/_posts/2024-2-8-day-38.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 38) 
+title: (Day 38) Traffic sign classification and bbox model-PyTorch and more of Andrej Karpathy's talks
 categories: [cnn,applying-knowledge]
 ---
 

From b895156f1cabb29ad4eded9b62650e67a266e600 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 19:55:02 +0900
Subject: [PATCH 129/638] Create 2024-2-10-day-40.md

---
 _posts/2024-2-10-day-40.md | 70 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 _posts/2024-2-10-day-40.md

diff --git a/_posts/2024-2-10-day-40.md b/_posts/2024-2-10-day-40.md
new file mode 100644
index 000000000..ed30edb49
--- /dev/null
+++ b/_posts/2024-2-10-day-40.md
@@ -0,0 +1,70 @@
+---
+layout: post
+title: (Day 40) Starting a Self-driving cars course by the University of Toronto
+categories: [theory]
+---
+
+## Hello :) Today is Day 40!
+A quick summary of today:
+* Read more research papers (more like *tried* to read)
+* Found an interesting Self-driving cars course by the [University of Toronto](https://www.coursera.org/specializations/self-driving-cars)
+
+Before anything, I asked for pictures of the Korean Computer Science Conference (KCSE 2024) and they sent me.
+
+![image](https://github.com/user-attachments/assets/0f2fe69f-3f11-4cd7-9b6e-52261b31069b)
+
+Now, as for the research papers.
+Here is a list:
+[DELVING DEEPER INTO CONVOLUTIONAL NETWORKS FOR LEARNING VIDEO REPRESENTATIONS](https://arxiv.org/pdf/1511.06432.pdf)
+
+[Long-term Recurrent Convolutional Networks for Visual Recognition and Description](https://arxiv.org/abs/1411.4389)
+
+[Convolutional LSTM Network: A Machine Learning Approach for Precipitation Nowcasting](https://proceedings.neurips.cc/paper_files/paper/2015/file/07563a3fe3bbe7e3ba84431ad9d055af-Paper.pdf)
+
+[Learning Video Representations from Correspondence Proposals](https://arxiv.org/abs/1905.07853)
+
+Maybe it is a bit early for me to read papers, but I am interested. I felt like after images, videos is a natural progression (and also my interest), but if anything, the papers made me think what I can do. I felt like after the basics, the more complicated projects like multi object localization/recognition just require loading pretrained models because they are deeper than what my machine can handle and trained on way more data (and established as good models). Nevertheless, there is definitely benefit to writing models (as complicated as free ML platforms/my laptop allow me). 
+I think doing a simpe literature review at this stage would be too hard and I need a bit more understanding and practice (but that does not stop me tho, if I decide to do it). 
+
+After the Tesla and Andrej Karpathy talks, I got excited about self-driving cars. And as I learned - Tesla's autopilot is understadably extremely complicated. And I wanted to dip my toes in the field too. I found out that robotics is involved a lot. And understandably, self-driving cars are a massive projects that encapsulate many different aspects. 
+
+So on the topic of self-driving cars, I found a popular course offered by Univerity of Toronto on Coursera. And I signed up. From my searches online, I saw that Tesla might not even be number 1 when it comes to self-driving tech - there are Waymo and Cruise, which might even be considered better than Tesla. But I'm glad I found them, because it just broadens my awareness in the field.
+
+The first thing I found was a definitions of the levels of self-driving
+
+![image](https://github.com/user-attachments/assets/27f42bae-1179-486e-9bd2-9e77a5e3d2aa)
+
+Apparently Tesla is level 2, and Waymo is Level 4 at the moment. Level 3*,4,5 is where a person is not driving, and 1,2,3 is where a driver is required. 
+
+I also did a bit of the University of Toronto course and here is a summary of some interesting things that I learned.
+
+Some history:
+The concept of self-driving cars dates back to the 1920s, with various experiments and predictions. GM even envisioned self-driving cars by 1976 in a promotional video which is kind of funny, and they soft-predicted these cars will be available in the 70s/80s.
+
+In the 80s and 90s autonomous driving at low speeds and on highways was achieved.
+
+Waymo and Tesla made the field more popular. 
+
+Perception in autonomous driving:
+It involves identifying static and dynamic elements in the environment, predicting their movements, and localizing them through cameras, sensors, radars, lidar. Tesla that relies heavily on cameras could face different problems because cameras' vision might get challenged in adverse weather conditions, wheras lidar methods that use beams, overcome that particular challenge.
+Other challenges in perception include sensor uncertainties, occlusions, reflections and data loss  which require robust systems capable of handling ambiguous information and utilizing redundant sensor data.
+
+![image](https://github.com/user-attachments/assets/6f80efe0-71db-4c07-9ad7-c675ecb3d17f)
+
+Decision-making:
+There are long and short term planning. 
+* Long-term planning in autonomous driving involves setting high-level goals and trajectories, such as determining the route from one destination to another.
+* Short-term planning focuses on immediate driving decisions, like lane changes and intersection maneuvers, based on real-time environmental factors and driving conditions.
+
+Also there are strategies like reactive planning based on current conditions and predictive planning relying on accurate predictions of other vehicles/people/participants in traffic's actions.
+
+This is an overall summary. The content is very new, interesting and different from my previous studies. 
+
+OH! I almost forgot! The course requires the use of Carla (some simulation tool for self-driving research) It looks like GTA haha but I am excited to use of for the practical tasks that come later. 
+
+![image](https://github.com/user-attachments/assets/ef81ea32-4b2e-499a-8b40-8123adbe54ee)
+![image](https://github.com/user-attachments/assets/36af9bee-479d-4c7b-ad02-6579e5036403)
+
+That is all for today!
+
+See you tomorrow :) 

From 220f627c8e0c3935bd28768c4924a37dfdb23a63 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:00:35 +0900
Subject: [PATCH 130/638] Create 2024-2-11-day-41.md

---
 _posts/2024-2-11-day-41.md | 107 +++++++++++++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 _posts/2024-2-11-day-41.md

diff --git a/_posts/2024-2-11-day-41.md b/_posts/2024-2-11-day-41.md
new file mode 100644
index 000000000..53e021960
--- /dev/null
+++ b/_posts/2024-2-11-day-41.md
@@ -0,0 +1,107 @@
+---
+layout: post
+title: (Day 41) A bit advanced computer vision concept review
+categories: [applying-knowledge,cnn,theory]
+---
+
+## Hello :) Today is Day 41!
+A quick summary of today:
+* Review some computer vision concepts that are labeled as advanced - image segmentation
+* Saliency/class activation maps (CAMs)
+
+To begin with - image segmentation, aka the process of partitioning an image into multiple segments or regions based on certain characteristics such as color, intensity, texture, or semantic information with the goal of simplifying the image for potentially better analysis. 
+
+Some popular models are - SegNet, UNet, Fully convolutional network (FCNs).
+
+![image](https://github.com/user-attachments/assets/2edfd2c7-b2cd-453e-9cd9-3292ffc5d477)
+
+SegNets are a DL architecture for semantic segmentation that uses an encoder-decoder structure with pooling for efficient pixel-wise classification. It's known for its ability to produce accurate segmentation maps while being computationally efficient. like the others, it first encodes a picture (learns its details) and then a decoder is used that takes the high-level pixels and reconstructs the image into different 'segments'.
+
+Another one - which I learned about before, but did not have the chance to implement is UNet.
+
+![image](https://github.com/user-attachments/assets/bbc01c7d-21ba-40e2-a0ad-c3b5c2389070)
+
+It's a DL architecture originally designed for biomedical image segmentation (as per the original paper haha). It features a symmetric encoder-decoder architecture with skip connections that help preserve spatial information, making it effective for tasks where precise localization is crucial. (Later there are pictures where I made a UNet using TF) 
+
+As for FCNs, there are three that I saw - FCN-32, FCN-16, FNC-8.
+
+![image](https://github.com/user-attachments/assets/fdca28c4-11d4-49c8-82dc-5131fdc6fe33)
+![image](https://github.com/user-attachments/assets/a3786059-a5e2-4055-822a-25c006c9d372)
+![image](https://github.com/user-attachments/assets/1796a634-30ba-4104-834e-8af1ab755802)
+
+the number at the 8, comes from how we will upsample before the last layer (as seen in the pics) 
+
+The type of image segmentation I tried is basic, on single objects in pics.
+
+![image](https://github.com/user-attachments/assets/eeb7bb43-06e7-4ecc-aec1-b7b8e034fbaa)
+
+Putting the UNet architecture again for reference:
+
+![image](https://github.com/user-attachments/assets/1162480f-d249-400f-9c09-64dd561a019c)
+
+First construct conv2d blocks:
+
+![image](https://github.com/user-attachments/assets/9f55959d-735f-4aa1-b726-9d3b2dc2eb3c)
+
+With those construct each individual envoder blocks which will return the item required by the next encoder block, but also what will be passed to the decoder block
+
+![image](https://github.com/user-attachments/assets/c897d342-8ea8-4117-b2df-fc23b2f78171)
+
+Construct encoder:
+
+![image](https://github.com/user-attachments/assets/df4efaa9-e2e4-4237-a28c-2bd6d6a5c0a4)
+
+Do the bottom of the U-shape
+
+![image](https://github.com/user-attachments/assets/f3158cdf-bbb7-4e71-8e7b-16331a6a533c)
+
+Then construct the decoder blocks:
+
+![image](https://github.com/user-attachments/assets/db125d10-9c37-4b76-8e77-44e2c9808375)
+
+The decoder:
+
+![image](https://github.com/user-attachments/assets/f97f08ec-7258-4953-ad72-9c354ecdf9b7)
+
+And finally, stitch it all up:
+
+![image](https://github.com/user-attachments/assets/e373d0d2-0853-4ecc-ac15-ed9043ce5853)
+
+Using the pet images an example prediction vs actual is the not too bad picture below
+
+![image](https://github.com/user-attachments/assets/36ba67df-e124-4a08-b989-5783fa7bc7d5)
+
+I saw the Mask R-CNN paper, but to understand it properly, I need to see if I can write a simpler version of its code. Using these ready pretrained models is great, and they are definitely powerful, but my goal is not some project with which I will be done with DL, my goal is to understand the basics of the popular networks, and even if I cant train them myself, writing a simpler (and similar) structure is helpful!
+Mask R-CNN is used for not only, segmentation, but also it does detection and localization like in the image below:
+
+![image](https://github.com/user-attachments/assets/4341d244-63c8-4e9b-b2ed-9be94d960ea1)
+
+Going to my final point of today: Saliency/class activation maps (CAMs)
+
+To better understand how CNN models work, how the layers see images, what pixels are important, there are some established methods to 'look under the hood' (besides the ones I saw today, there are others too)
+
+1) Class activation map (CAM)
+
+From a CNN model, we take the last pre flattened layer and the output dense layer (like in the img)
+
+![image](https://github.com/user-attachments/assets/570f738a-fcbd-4fe6-8c8a-097c890ac1de)
+
+we feed them to a model
+
+![image](https://github.com/user-attachments/assets/217a9bba-ceee-4fd9-bb94-e32915fdda36)
+
+And we can get images like this (the train data is fashion MNIST)
+
+![image](https://github.com/user-attachments/assets/4a5abc2e-95da-4f19-a3d4-98ff9fdaa0e7)
+
+The darker the pixel is, the more the model 'likes' that pixel and consders it important for the prediction of that certain class. Why the green and red ? The red is where the confidence is below 90%. Maybe its because looking at what sneakers are above 90% confidence, the below 2 boots are a bit different, so maybe the data has a lot of sneaker type shoes. 
+
+As for saliency,
+
+![image](https://github.com/user-attachments/assets/8a47d378-ec3d-421e-bbdb-894c68747269)
+
+it refers to a method used to understand and visualize which parts of an input image are most influential in the network's predictions. More specifically it involves analyzing the gradients of the network's output with respect to the input image pixels. In the pic we can see that this particular model looks at the animals eyes (also the noses) to determine whether its a cat or dog (the dataset contains images of cats and dogs).
+
+That is all for today!
+
+See you tomorrow :)

From e3c5a8313d0e855b797ed546427d9ded216d8c27 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:05:07 +0900
Subject: [PATCH 131/638] Create 2024-2-11-day-42.md

---
 _posts/2024-2-11-day-42.md | 81 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 _posts/2024-2-11-day-42.md

diff --git a/_posts/2024-2-11-day-42.md b/_posts/2024-2-11-day-42.md
new file mode 100644
index 000000000..f0343bc86
--- /dev/null
+++ b/_posts/2024-2-11-day-42.md
@@ -0,0 +1,81 @@
+---
+layout: post
+title: (Day 42) Creating a UNet with PyTorch
+categories: [applying-knowledge,cnn]
+---
+
+## Hello :) Today is Day 42!
+A quick summary of today:
+* Road segmentation with UNet
+* Forest area segmentation with UNet
+* Human segmentation with UNet
+
+But why? I can easily load an UNet (well-made UNet, even pretrained) and it will work great on the images I found. Well, I tried to follow the model from yesterday, that was written with TensorFlow, and I just wanted to translate it to PyTorch and use it on new data. 
+
+Was my attempt successful? - Not exactly haha 
+Why ? - Training takes a lot of time. I ran out of free GPU resources and started using CPU (super slow ㅜㅜ). Adjusting the learning rate helped, but adjusting other hyper params can help too, but training time is too long without free GPU hours ㅜㅜ
+
+I had 3 attempts on 3 different datasets
+
+![image](https://github.com/user-attachments/assets/5a212bfc-6782-44bc-92f4-b108352399f1)
+
+This is the format that I tried to follow.
+
+Model structure:
+
+![image](https://github.com/user-attachments/assets/ed1bf54d-9a56-468f-be7c-edccb1c32af0)
+![image](https://github.com/user-attachments/assets/6ab0c63a-5ac2-44c7-a95e-7279bc2d94fa)
+![image](https://github.com/user-attachments/assets/1e1440a4-fb0b-44ba-b131-abdbead0a8b9)
+![image](https://github.com/user-attachments/assets/83734375-2336-48e2-af94-b87f2d7cbdac)
+![image](https://github.com/user-attachments/assets/3b4ab348-de26-4f54-a425-56afe2c1268f)
+![image](https://github.com/user-attachments/assets/4e0b2d1a-8ed9-4f39-ae8d-66c3096bdce3)
+![image](https://github.com/user-attachments/assets/4c90191a-76e5-4426-ab5a-a63551a63e46)
+
+Actually, I followed a TF model and translated it, so a potential issue is that I might have missed or did something in the wrong order. But given that the results are not terrible, such possibility is low (but still exists). 
+
+For the loss, I used DiceLoss (also learned about it yesterday) and also binary crossentropy.
+As for data augmentations - used albumentations and did horizontal and vertical flip with 50% probability. And of course resizing to 128 (or 256).
+
+
+Below are summaries and info about the 3 models.
+
+1) Road segmentation with UNet
+
+![image](https://github.com/user-attachments/assets/e53ba7d7-c77f-4183-8a7e-b9e36f897a47)
+
+Actually this dataset was a bit problematic, because some of the pictures had these white spots, which ruined the models' performance, but if we look in the ground truth, there are actually roads there.
+Some results that were not too bad ~ around 0.5 loss. 
+
+![image](https://github.com/user-attachments/assets/ee0e0319-f7f5-4def-8e80-014e9aaf8c59)
+
+And here is an example of a picture that is not very good and just hurts the model (probably can just delete those from the training to improve performance)
+
+![image](https://github.com/user-attachments/assets/e0b4b765-43d8-4d1a-88f4-3bc55ff8e1c9)
+
+2) Forest area segmentation with UNet
+
+![image](https://github.com/user-attachments/assets/1df4a4fb-37ca-4a67-ad20-0587cbec209b)
+
+Example of this particular dataset ^
+The valid loss after 25 epochs was 0.65 and a sample result. Definitely not great. The model needs more adjustment for these images. 
+
+![image](https://github.com/user-attachments/assets/07eb8fbb-6890-4163-8de4-9c57dca60b02)
+
+3) Human segmentation with UNet
+
+Sample image (flipped in this case haha)
+
+![image](https://github.com/user-attachments/assets/fa93abd1-0e69-4708-8182-66be03f8954a)
+
+After 100 epochs, with learning rate 0.0001 (last attempt) - 0.6 loss
+And a sample result
+
+![image](https://github.com/user-attachments/assets/b3661837-8227-4c3b-a3a9-4e0becdcbf11)
+
+Not great. :(
+
+Because the training was taking long, I ran notebooks at the same time on Kaggle, google colab and my laptop. Tried adjusting learning rate, image size, batch size. I tried to avoid adjusting the filters layers of the UNet to see if the same structure can work for different images, but unfortunately today I could not get good performance. I am thinking of concentrating on 1 and just let it run in the background while learning/trying something else tomorrow.
+
+That is all for today!
+
+See you tomorrow :) 

From d9a3782b9b77d81d956a9307f8133703759e9b14 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:09:51 +0900
Subject: [PATCH 132/638] Create 2024-2-11-day-43.md

---
 _posts/2024-2-11-day-43.md | 79 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 _posts/2024-2-11-day-43.md

diff --git a/_posts/2024-2-11-day-43.md b/_posts/2024-2-11-day-43.md
new file mode 100644
index 000000000..dd0cfaede
--- /dev/null
+++ b/_posts/2024-2-11-day-43.md
@@ -0,0 +1,79 @@
+---
+layout: post
+title: (Day 43) Coding up LeNet, VGG, InceptionNet, UNet from scratch
+categories: [applying-knowledge]
+---
+
+## Hello :) Today is Day 43!
+A quick summary of today:
+* write LeNet from scratch
+* write VGG from scratch
+* write InceptioNet from scratch
+* write UNet from scratch (again)
+
+Wanting to understand the popular models a bit more, I decided to do the above. 
+
+1) Let's begin with LeNet. 
+
+A basic framework developed in the 1990s, basic but set the groundwork for networks like AlexNet, VGG and ResNet.
+
+![image](https://github.com/user-attachments/assets/b0e2ac81-a149-4fbb-bbcd-9169c2bc49bd)
+![image](https://github.com/user-attachments/assets/3cc6d578-aa70-4513-8794-9e96618e0ef5)
+
+consists of 2 conv layers, each followed by a maxpool, and then ending with 2 fully connected (linear) layer. 
+
+2) Next is VGG
+
+![image](https://github.com/user-attachments/assets/fe475b09-8f45-4334-967f-f487220279b6)
+
+The paper proposes numerous versions, VGG11, VGG13, VGG16, VGG19 but from a google search VGG16 seems most popular (version D in the pic). It is deeper than the earlier LeNet, consisting of multiple conv+maxpool layers, each increasing the amount of filters, and decreasing the size of the image. 
+
+![image](https://github.com/user-attachments/assets/4f7a6039-f755-40aa-9b3f-e494cee18456)
+
+instead of 1 version, a general model was created so that it can adapt to the desired VGG architecture
+
+below is the implementation. I think this is a nice set-up for testing the 4 versions on 1 dataset of my choosing. 
+
+![image](https://github.com/user-attachments/assets/bfa1a582-4180-45eb-aa91-c1eb0a99b678)
+![image](https://github.com/user-attachments/assets/e20d6ee4-b678-4631-9c68-0d3f77d4d34c)
+
+3) GoogLeNet / InceptionNet
+
+This is a long one... haha. Funny that a research paper references a meme (and it got the name inception from it)
+
+![image](https://github.com/user-attachments/assets/a5c2884b-5449-4a9d-b61d-01a4abcc7eb7)
+
+GoogleNet features inception modules - which consist of parallel conv branches of different receptive field sizes.Also 1x1 convolutions reduce dimensionality and improve efficiency.Towards the end, global avgpool replaces fully connected layers for fixed-length feature vectors.
+
+![image](https://github.com/user-attachments/assets/14f358c7-a65a-4b9e-8986-ab6a83b60ff8)
+
+and the implementation:
+
+individual convolution block
+
+![image](https://github.com/user-attachments/assets/8d4f7d79-5020-41b2-9c93-3034f073db39)
+
+individual inception block
+
+![image](https://github.com/user-attachments/assets/8736a06e-fc05-4fdc-86d4-1e01e7ad0dc6)
+
+And the final GoogLeNet
+
+![image](https://github.com/user-attachments/assets/62fb3ed7-df77-42ea-9238-e28f0d871e50)
+![image](https://github.com/user-attachments/assets/24254935-cee1-48fd-91b7-3eea3fb10795)
+
+4) UNet
+
+Yesterday I attemped it and I think I got a good kind-of working model, but I was not sure if it is correct or not because I translated it from tensorflow to pytorch. Today I found another version of UNet online and pasted it into my records (thank you to [this youtuber](https://www.youtube.com/watch?v=IHq1t7NxS8k&t=122s&pp=ygURVW5ldCBmcm9tIHNjcmF0Y2g%3D))
+
+![image](https://github.com/user-attachments/assets/92852315-97ff-48a6-89d3-cb69aa7bdb7a)
+
+Pretty similar to yesterday. Here is the double conv part, and the init of the Unet itself.
+
+
+
+
+
+
+
+

From a3919c3d67923701737fe5c7e84609ff9515c0f7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:12:40 +0900
Subject: [PATCH 133/638] Update 2024-2-11-day-43.md

---
 _posts/2024-2-11-day-43.md | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/_posts/2024-2-11-day-43.md b/_posts/2024-2-11-day-43.md
index dd0cfaede..d1aebeee3 100644
--- a/_posts/2024-2-11-day-43.md
+++ b/_posts/2024-2-11-day-43.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 43) Coding up LeNet, VGG, InceptionNet, UNet from scratch
-categories: [applying-knowledge]
+categories: [applying-knowledge,cnn]
 ---
 
 ## Hello :) Today is Day 43!
@@ -70,10 +70,26 @@ Yesterday I attemped it and I think I got a good kind-of working model, but I wa
 
 Pretty similar to yesterday. Here is the double conv part, and the init of the Unet itself.
 
+![image](https://blogger.googleusercontent.com/img/a/AVvXsEiL3ekAbZ5rXaKiZxRYSsuWZcg5osBGIq70tGB97Sh2aS1oK0UulmtB4K-o3Q9NiRhTHZ5XcalTdQE0NwBvc_RX_5jIG7lCl9JaROXSDCMArOIQZTyo9tZw2dkNE8e0pI-DhJWTHMX8jWJEf1I_h73S9erjOI9oE1SXwXTKWh6QZc9pJeX9W76lygJOxoYk)
 
+and then the forward
 
+![image](https://blogger.googleusercontent.com/img/a/AVvXsEjm7tK-ANSyOlAo5UrCJLnjSTlVMp32zWtZipIwpZPSAvKfn0J1JuBsEU60ckcrvRn9nfx8CT96R5I_ff9QAToiyG1Rbkn4hwjvz_nuu6Q0pRECALOuMf2j7FO5z3fCfE-KbxYlHqHMoTGih0JVBN3_3hlP3-shJLoFvrxD5RBulWzhzHwS2KNwIScdmXiZ)
 
+Actually I decided to even test this UNet on a human image dataset in kaggle
 
+![image](https://blogger.googleusercontent.com/img/a/AVvXsEiEZeLTYDrH0xysuzuwifs9QUS285q_esZ2DxxEVlzqcEozmIhFNh7UD-YF7woPyfcx3aWG_72BKtowcNWHGnTRXMURK2M1jqTPaZvm6Sy-kvDa1H8mArgKmWrACKAxlPIW9q5lyJ_2tK534SH1rvUEqUAOzbnrJmit9Uh7lKUIZBlSMxNSX3TlBNkOZSyv)
 
+After 50 epochs, with learning_rate 0.01 and batch_size of 16, using DiceLoss and BCEWithLogitsLoss, the best model I got is: epoch: 42, train loss: 0.2380, valid loss: 0.3435. I am running a 100 epoch one with a learning rate scheduler. It will take long but the results (hopefully good) can be seen [here](https://www.kaggle.com/code/divakaivan12/human-image-segmentation-with-unet/notebook?scriptVersionId=162731893) in Kaggle. 
 
+Sample outputs from the 50 epoch model are:
 
+![image](https://blogger.googleusercontent.com/img/a/AVvXsEjxemkzGfjJ5UP5-oLJVtNOhP5jAprhNEzX3pdyHy5n4Y27DXnLbShbxWapJH5CvAbAC4eFpQ2MU8kzAlxFKHglm9rfXsg4cOo-VaYcpLL3Rnd3I49LRXT2vmZ01g-JBfuYayq2Wg4NQMqWuNHngXzQk9j8LlnFPp4jW5zga9mnW-VgQp0VQ9JM0pBsMe4B)
+
+I should try IoU, or maybe another combination of losses too. 
+
+Side note! Turns out I did not properly install pytorch on my laptop and was running things on cpu, which is a disaster, but today I set it up properly. (but still using kaggle is a bit faster haha).
+
+That is all for today!
+
+See you tomorrow :) 

From 42317e59db9b915ac81e4eae697c84eac25e4183 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:13:08 +0900
Subject: [PATCH 134/638] Rename 2024-2-11-day-43.md to 2024-2-13-day-43.md

---
 _posts/{2024-2-11-day-43.md => 2024-2-13-day-43.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2024-2-11-day-43.md => 2024-2-13-day-43.md} (100%)

diff --git a/_posts/2024-2-11-day-43.md b/_posts/2024-2-13-day-43.md
similarity index 100%
rename from _posts/2024-2-11-day-43.md
rename to _posts/2024-2-13-day-43.md

From 2527f9f542cc3cf0eddd6c9a3c98d208d13d6581 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:13:26 +0900
Subject: [PATCH 135/638] Rename 2024-2-11-day-42.md to 2024-2-12-day-42.md

---
 _posts/{2024-2-11-day-42.md => 2024-2-12-day-42.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2024-2-11-day-42.md => 2024-2-12-day-42.md} (100%)

diff --git a/_posts/2024-2-11-day-42.md b/_posts/2024-2-12-day-42.md
similarity index 100%
rename from _posts/2024-2-11-day-42.md
rename to _posts/2024-2-12-day-42.md

From 8cc746405e6d5f2908ae57e0b59a593c315710d8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:21:36 +0900
Subject: [PATCH 136/638] Create 2024-2-14-day-44.md

---
 _posts/2024-2-14-day-44.md | 112 +++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 _posts/2024-2-14-day-44.md

diff --git a/_posts/2024-2-14-day-44.md b/_posts/2024-2-14-day-44.md
new file mode 100644
index 000000000..e0c3c5906
--- /dev/null
+++ b/_posts/2024-2-14-day-44.md
@@ -0,0 +1,112 @@
+---
+layout: post
+title: (Day 44) Batch vs Layer vs Group Normalization and GANs (+ found a free KAIST AI course)
+categories: [applying-knowledge.theory]
+---
+
+## Hello :) Today is Day 44!
+A quick summary of today:
+* Found KAIST Professor Choi's Programming for AI lectures [link](https://youtu.be/TG3OJQjN-l4?list=PLLENHvsRRLjAmAjc8mV0f9C6i8Gh308SS)
+* Discovered that there is Layer and Group norm (not only Batch norm)
+* Learned about GANs
+
+1) Batch vs Layer vs Group normalization methods
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEj9R6IO0Pwu-ZtrRo8lOASNx4F0YYViutpHal9cC0rPX_kZy_Et49ToBKmI7LqllwB9bFILbor9X2pRlWlP7R7qQfT0x_Kyr6-QJhiY0HeFOT01dxrDbcrEjr1sNi6oxxI2yeeg2X3qMZxDT7BVrLtV6rBlXap2dvvE5kszBLtXNAbsmQ7tq7f4bi0F_Wrw)
+
+In one of Professor Choi's lectures, he explained about the above three norm layers (I have not heard of the 2nd 3rd 4th), plus some searches online, I found the difference.
+
+Firstly, batch norm: given a batch of activations for a specific layer, the mean and std for the batch is calculated. Then, it subtracts the mean and divides by the std to normalize the values. (+ an epsilon is added to the standard deviation for numerical stability) following that, a scale factor "gamma" and shift factor "beta" which are learnable parameters are applied.
+
+Secondly, layer norm: proposed in 2016, layer norm operates over the feature dimension (i.e., it calculates the mean and variance for each instance separately, over all the features) but unlike batch norm, it doesn't depend on the batch size so it's often used in recurrent models where batch norm performs poorly. Layer norm computes the mean and std across each individual observation instead (over all channels in case of images) rather than across the batch. This makes it batch-size independent and can therefore be used in models like RNNs or in transformer models.
+
+Thirdly, instance norm: here, mean and variance are calculated for each individual channel for each individual sample across both spatial dimensions.
+
+
+2) Generative adversarial network (GAN)
+I finally got to meet GANs. 
+What are GANs? 
+GANs are networks that aim to understand the given data and recreate it. It's 'learning' process is interesting in that, it 'fights with itself'. There are 2 networks - a generator and a discriminator. The generator tries to create images as if they came from the training set and trick the discriminator, while the discriminator tries to distinguish between the actual training images and the ones coming from the generator.
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEjiBia7mos9_778ICi05in9JwSoIJNQlNakEJWNt7Z8XypDbTQZ4WPO3rdDSMjXAh6eW4xRYxjfU5-fck1J9vJvuungV2YfpLTUzkE6z7pG-KCMGSiFLmM7STrToWUeoGNQF9Sjum0OC4lQ0T_8r4vP_I5AEAdtTCt4tX4qlqOBcjfYMhGmridsWwD5YOWq)
+
+A simple Generator:
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEgPeUySyF6kWg0XBVfilka7uFQydVHNgXD_0mpHCxdOSl6SQ7KRC7IWtFCj1lA-YYf2u1UtOb3mOEvrAlW9Cpf2WEI8VaZW_Opv94Pyv0l8aepFo-JfvVM-od7nCCJ__1FgjDOHufFTpqnz3MNGG_bEF3GgY6gk1z8OeEB5Ip4JF4zXpNmFTR5X-IoCfexm)
+
+And a discriminator:
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEiiF_vm5TXe3RUa4_0uon5lr0rrwiNcyKUNOyEvZ3OVamAqL1TR8ZxgHNImVLMz38rCnxhBanbxeDGIV6DT8oHnn6WrauN6bxC5wr7N6OwbESqYfVGrYGJ99tJ_vOA7NfieVoPTwaikGchVXZnczmp0Jgt5f6PT2zDPJZT3myD3yi-qkT60tUG5gryz9n04)
+
+They both use their own optimizer:
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEigbctX8Ms6nUVmqKeRRhcpfp9Wyc-NudRFrYSbQRBrX2Z_0AKoJqGW0utWID-4UbAFaFNdHRzdENzOkCf8_gae2E-gFNrUEAm9K7FIvdPy2zJMEes8Ho9geRmymOkh3q6XU-zIITFQAWTRjWHkG9_FyXLRRa0doYVXO4bfhUhbMAAQETiJ3OcB2lBijmWT)
+
+First you train with real images and then with fake.
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEjbUIgtTlXZ8LazoEmgipC1y9VZKbVNBqnHcjEu3BR2qZKbhQB3qklpHtFEUXGnWwhQgNi0bDmfkKG4hZ4CrSRJefUpAkDjBoWWPYlna4hKyhMhkgwAVW01hQ4QvGDv3wXLPFT7VSJfoWQROcbsHvFZWM0-ERVAHb5P3yfPaLFX0XSM0Uct1H8g1-Uq8Wlx)
+
+And then you update the generator
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEjIk8Mn0zzUZAAnfQdwCoEZznLg4M1Eb-2OAj66BVL2_MhOhpU6_k_S-y-nvge0P2wP7c59igv6Giv71XnSe1h1hvQjI0uf3MqDH5J7FGdPBnrKPZbAGakNlf-25zj2YQFwUvyyNLZV78gq5St5ldiUqeFQyRrSOwJA8zzzj5B8-kFlHCER2zyxBQAdSyU4)
+
+To get a different perspective, I went to PyTorch's [tutorial](https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html) page for GAN (more specifically - DCGAN)
+
+A DCGAN is a more complex version of GAN that uses more techniques like batch norm, leakyReLU, and conv layers. 
+
+Their GAN makes use of people faces dataset
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEiGYNwbB7pwHMAqvRnc3Cg0aaRXbzMoQuO_GrDU6aqG2nrsfMljr1Q8pzUF2lJQTNUtLffz3hFm_GG36c1QiO8I7wr5J-dHsz2BxcNmUn8BiV_VSc7LS-G5b-6Ft8EjNvvsLJzqW6395gqimzx7pNXof0E_nWrzEYBwf0jUzK8RQcnmI_qahKJxlYtjUpbf)
+
+The generator and discriminator work against each other
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEiRdmnW_iiSi5_6p5LjdssTOjKaS3VAuMBEXT3_a7FsSnh0uY0gLjC_zn3EiTb1p8eSmaNlYolpxcU4KnafQQceQK7Vel-KTf_dEIuB1Y9QDVDfEH_Ruma0UXZ7UjyEZAXRtkFyJs-y9gCAtbXeXaM9baMwS0InC7hjXe4dVOpJpYb10qCSVXPY5PIGLG3Q)
+
+The generator takes noise initially and turns it into an image to trick the discriminator:
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEgb34vsDkekipOJrAHvulAOl9HcdUsb063E_u29NJMLJ_NfbC_uhLchBcgCI-6qiKteFGM26w0hSwXYt42mG4yHgrDyZBRjrR5jB5hJslPr3xP8MizEaSRuxjoaToxbCmOHY39VaT-FnVaEErRGSopsVM1o5maHjUki6bKu-n8-4oq6W2CbyhnmbX7JEz1u)
+
+While the discriminator takes an image and outputs whether it is fake or real
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEhgMh6G3pGcT9sGeLI1UDJpdsuG6N5vf4nfc9chKqVAuxQcWh3yyKSf-33edrpkMHzNaInFntg-nGe5a0CT8aYAXlunt2YQ8Zi_6Ux6PwFT7m4PpytJVcufh7WGD_3WhHIWCq3PQJwpecLJaztD932VMsYPaA5IgB7js_ue5o-PPiBDbLpnffsaiHG75XuG)
+
+I ran their code on my laptop. Initially 5 epochs ran for 4 hours, and then I realised the device is 'cuda' but that does not work on mac, as it should be 'mps', so I was running it on cpu, wasting time :/ thinking "wow this must be computationally expensive" hahaha (model complexity is something to learn on my list)
+
+After the 5 long epochs: the resulting faces are:
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEjtcg0SLOh-7m4PW9_z19tcVklqqceM4wsQDBqWUgEcthg-O2fKMZSJuiuH8b8mScN05v511n6JSYDBoBn6QZh9plLp9C7RQQ9oVkbsGIQUlkVLH1rOxDnnjygwYgHwximLQJr1C8YTKEMy8dETpxG9clWhGflTL5rVazFShsxp1L1-LfaRmq3kshMzpowA)
+
+And this is after 19 epochs (I am going to bed now so I stopped it)
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEhXd4uVWRcEwTJdj8mijRuK4fEZhi-ZcXEL9EWgL06X7OGV5knJrBZtQOXg0sAcBfryw0pyFg1vIePVNeoak7sp4gJW_gfd7ELzAbSQJezSLby1XhT_k-evcOFH7q0c2aex0SPfk6kVNnFDpQoxKJOlIBs3TL-zNhe-PBxqUTDWa4GiDoznBJUKjomhFWwT=w382-h179)
+
+It is kinda cool. But ... 
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEjyKtoLWHIxFcwqKItkxJtcYnIB8aHannzE-7n03gWyd3KfiWirmdddIW_vvi7w1drvr7V1VpsDylXiGrTlL53W0t3Wj2MGWiFQAczezsDmh95zkyuQmVoZpnA-0uNeffIk7X8X96RcYBHvPUAnjJIbnMe8Vy8L1CKpwsyGIOiICFaUGEvYF70pjjcsf3M1)
+
+Obviously, DALLE-2 and midjourney can create way better fake images than my laptop :D but knowing the concept is key here. 
+
+Thank you to Professor Choi from KAIST for sharing his lectures to the world.
+
+Side note!
+While I was looking for GAN info, I found a tweet from 2016 from Andrej Karpathy
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEifa15q1p4J26Dk7yLVHZ8Strc_o_hjVgTiNaNLl7nbhiiZ8wNvuc7gyMvYxJomfsk1RKh6SGwR95nWl7wmwLBA2dFgr6noxdxbdFq3LLGIssy7PTUpvCZqk652AA73e5ZAwdN3IFu3IgJQx8U-T9zlhL3BKb_pjqeZVw8AMtzTNni8n4aeXKCJrSznv3h1)
+
+A joke or not, in 2021 it was the recommended by google haha
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEjOfR79TbWDcKugV19WX-cchpHgqALKRzqEsnIP78fRJw69Bg3w2Z5Z1vyRx9yFkW2gcjf58GVOB1nPmATucv4MzI_hphH3CpBd0birs3_zKuYhIK-7ur4j13hln8kpCIJzKapSsZCmCxUy2Hz_7rAVCVCJ-le_mrl0ytzMZLL6NK1jrt9qnj4BYxabUEuY)
+
+And under those tweets, I saw that he left OpenAI and wants to focus on his personal projects
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEh7dhn_gO-ZkoZUu5VWpqGPS3lXUwl-ATfWKvZsuPBjHsPG8AyfSLameeM7ZDoZKg8Ups3jbNrFbJGV_eH6-1Pf7baB4zdFvqDszN4GQiAb9EMXI4EbXpJB9Gfdeta5DhKTgDMDuMPK2cthb-hH2GYNvsyi6VKmqwtUmTWmrxyUsQjLrQUhDfwj8cetYKgX)
+
+Such as youtube videos
+
+[image](https://blogger.googleusercontent.com/img/a/AVvXsEhDSyaZx1FVfOq5e7p5SpnHtaxiemHsZyi3FfB2ibfI40bw5kMMlq_t6UbAEq3DyqMMkCZJphfq9ofmJQTDzOdPlIluTBW4SCePU9jxCfuFsPl7IF1NB6Jz3_umTRaylAI6nxUlqBSzLKAnXUTWRlKjp2APJ7Gfci_cxgiiW2DxOvmSN3h_pE5Xi3xL26wY)
+
+More Andrej Karpathy youtube videos - looking forward to them!
+
+That is all for today!
+
+See you tomorrow :)

From c7138dbdd9c5c82396b09ee6b80112a34fec9d93 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:26:18 +0900
Subject: [PATCH 137/638] Create 2024-2-15-day-45.md

---
 _posts/2024-2-15-day-45.md | 96 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 _posts/2024-2-15-day-45.md

diff --git a/_posts/2024-2-15-day-45.md b/_posts/2024-2-15-day-45.md
new file mode 100644
index 000000000..a2a410395
--- /dev/null
+++ b/_posts/2024-2-15-day-45.md
@@ -0,0 +1,96 @@
+---
+layout: post
+title: (Day 45) Trying to understand VAEs with Professor Choi from KAIST
+categories: [applying-knowledge,theory]
+---
+
+## Hello :) Today is Day 45!
+A quick summary of today:
+* Learning the theory behind VAEs with Professor Choi from [KAIST](https://youtu.be/GNOwv8bY1YM?list=PLLENHvsRRLjAmAjc8mV0f9C6i8Gh308SS)
+* Implementing a VAE from scratch with [Aladdin Persson](https://youtu.be/VELQT1-hILo) (now probably my 2nd favourite DL youtuber after Andrej Karpathy)
+
+1) Theory behind Variational Auto Encoders (VAEs)
+
+we begin with autoencoders
+
+![image](https://github.com/user-attachments/assets/74670535-7537-409c-9a8f-844e6fdabaa8)
+
+given an input, they compress it into a lower-dimensional representation, and there is a decoder, that reconstructs the original input from this compressed representation.
+
+What VAEs want to do is with that space in the middle, sample from it and generate new samples. 
+But that comes with its challenges. 
+The idea behind VAEs is that we want to estimate the posterior distribution.
+
+![image](https://github.com/user-attachments/assets/20dd6b13-08c3-4c08-876f-f46a2d4c5003)
+
+We have X, and we want to know the distribution of Z
+
+![image](https://github.com/user-attachments/assets/b2f81031-c189-4ace-9b7e-82bff038f3be)
+
+in reality, estimating the posterio dist is extremely hard, so what we do instead is we approximate P(Z|X) to be pretty similar to a Q(Z) which follows a gaus distribution. But P(Z|X) doesnt always follow a gaus dist, so in those cases doing a Q(Z) like that is not a good choice. 
+So, if we assume that Q is gaussian, we can get the mean and std of that distribution and tune those so we can more closely approximate P(Z|X)
+
+So how do we learn Q(Z) ? We can minimize KL-divergence 
+Using some log magic we end up with the below.
+
+![image](https://github.com/user-attachments/assets/dbb00ff1-94f4-4def-8079-72249091a3f5)
+
+In the below formula, logP(X) does not change - that is our data, it is fixed, so what we can do is maximize the ELBO term, this will lead to lowering the KL-divergence. So our job now is to increase the ELBO. In the ELBO term, we know P(Z,X), and from the Q(Z) (which has a gaus dist) we get the mean and std, and we can tune those so that we can maximize the ELBO. 
+
+So, VAE
+
+![image](https://github.com/user-attachments/assets/4bac7eb2-a9c4-49f7-b5bb-33b02ddcfd44)
+
+with VAE with still encode, but now we want the latent space to follow a certain dist (i.e. gaus distribution) so that we can sample and create new samples from it. 
+
+![image](https://github.com/user-attachments/assets/a2019c50-ab15-4d20-a162-0bb488ac20d6)
+
+next we can denote the encoder and decoder as above. 
+encoder gets X, and generates Z, while the decoder gets Z and generates X
+
+But why do we need the latent space to follow a certain distribution?
+
+![image](https://github.com/user-attachments/assets/03b000c0-9b7d-4a97-9626-78d42b8e9632)
+
+because of our goal - wanting to generate new samples. In the case of VAE, when we sample we can just take the decoder, give it a sample and the generated sample will resamble a realistic one. If we sample from a gaus dist, we are more likely to get a sample that will resamble a real one. Whereas in the AE space case, we cannot be sure of that. 
+
+How does Loss work ?
+In AE, the the X produces Z and then it produces X' and what we do is maximize the ELBO term (tune mu and sigma) to end up with better results. By maximizing the ELBO we minimize the loss. (The ELBO itself can be decomposed into two terms - the reconstruction term and the regularization term (KL-divergence). The reconstruction term measures the reconstruction error of the input data, while the regularization term encourages the latent space distribution to match a predefined prior distribution. By maximizing the ELBO, VAEs encourage the model to learn meaningful latent representations while ensuring that the approximate posterior distribution is close to a predefined prior distribution. The regularization term, enforced by the KL divergence, encourages the latent space distribution to match the predefined prior distribution (typically a gaus dist).
+
+![image](https://github.com/user-attachments/assets/00dc5389-4835-4b1a-82a8-2f83eb121e75)
+
+But in VAE the ELBO term is a bit different (with theta and phi).
+Side note! it becomes a function of theta and phi (recall earlier 2 pics above that thats what we called the encoder and decoder). In AE, the mean and std are given from X, but here, theta (the encoder) represents a neural network itself and it produces mean and std for each particular sample and using those, we then construct the distribution. 
+
+To maximize the ELBO here we can learn theta or phi, but seems that its best to do both. 
+
+![image](https://github.com/user-attachments/assets/daf83d39-aa8d-4d49-b8b1-49b6fc332107)
+
+How do we train VAEs? Turns out there is a problem
+
+![image](https://github.com/user-attachments/assets/bab035ef-1faf-43f0-8006-d314520318b2)
+
+We get to the right side, get the Loss and now we want to do backprop. We do that, but we end up at Z (that is sampled from the gaus dist). And we cannot differentiate further and we cannot pass this 'stochastic sampling phase' because sampling is not differentiable. 
+How do we do it then ? By using a so-called 'reparametrization trick'.
+
+![image](https://github.com/user-attachments/assets/cf1540be-5c3a-4759-8df1-f70d493d7b52)
+
+We have mu and sigma, but in order to 'pass' the sampling from a gaus distribution, we multiply the sigma by an epsilon term which itself is sampled from a gaus distribution. Z follows a mean,std gaus dist, and we can rewrite that Z to be equal to mu + sigma * noise (epsilon is that noise). 
+
+ I had to double check my notes multiple times and watch the lecture again, so that I can understand the fundamentals. Hopefully I did not make any big mistakes.
+
+
+2) Implementing a simple VAE
+
+This was a handful, so just to try to understand it a bit more, put the theory to practice I went on youtube and saw VAE from scratch. 
+
+![image](https://github.com/user-attachments/assets/e85938b6-82d5-435d-a096-8730d62490be)
+
+Just like in the lecture, we take the image, put it into a hidden layer (theta), then that produces mu and sigma (this is the encoder part). Next, we reparametrize z using mu and sigma and epsilon, and that z goes into the decoder, which turns it to phi, and then turns that back into an image. 
+The implementation was way easier than the theory behind what is happening haha (as for the loss function, BCELoss was used). But looking at the code, understanding the theory becomes a bit easier. 
+
+Now writing this at 1.51am, I realise that the lecture from Professor Choi has practice notebooks, and I checked and it even implements the proper loss (BCE + KLD), but I will go over that quickly tomorrow. I will go to sleep now (zzz)
+
+That is all for today!
+
+See you tomorrow :) 

From da9d92b2889de1737f5e79eba9ac4a8eac7cb781 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:31:23 +0900
Subject: [PATCH 138/638] Create 2024-2-16-day-46.md

---
 _posts/2024-2-16-day-46.md | 106 +++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)
 create mode 100644 _posts/2024-2-16-day-46.md

diff --git a/_posts/2024-2-16-day-46.md b/_posts/2024-2-16-day-46.md
new file mode 100644
index 000000000..2f4e56fbf
--- /dev/null
+++ b/_posts/2024-2-16-day-46.md
@@ -0,0 +1,106 @@
+---
+layout: post
+title: (Day 46) Meeting Transformers again and their implementation
+categories: [applying-knowledge,nlp,theory]
+---
+
+## Hello :) Today is Day 46!
+A quick summary of today:
+* Understanding Transformers with Professor Choi from [KAIST](https://youtu.be/RAEt4Yppxjo?list=PLLENHvsRRLjAmAjc8mV0f9C6i8Gh308SS)
+
+The first time I learned about transformer was Day 32, it was a simple intro, but I did not understand exactly what is happening. I felt like, I was just made aware of their existance in the NLP world.
+
+![image](https://github.com/user-attachments/assets/a15ee072-ebff-487e-b262-492c56be18b2)
+
+This img is from Andrew Ng's Deep learning course. 
+In a transformer, the data goes through encoder-decoder network.
+In the encoder: for each token its attention is calculated according to the other tokens. And This attention mechanism allows the model to weigh the importance of each token in the context of the entire sequence. This information is put through a feed forward network that extracts deeper features. 
+In the decoder, we start to predict words. For example we start with an start of sentence token, then we pass that at the bottom, then from the encoder we take the K(key) and V(value) and with the Q(query) from the decoder input, we try to predict the next item in the sentence (what word can come after the start of sentence token) - and this happens N amount of times. Also when we do testing, the input to the decoder is masked, we don't give it the whole sentence, because if we do, the model will see the future tokens (words) and 'cheat'.
+
+ 
+So... going back and exploring this step by step. (pictures are from Professor Choi's lecture on youtube)
+
+Attention
+
+![image](https://github.com/user-attachments/assets/1402d400-cf39-446c-b439-304ca0efc564)
+
+if we imagine the the sentence 'I like going to movies', for each word we calculate its attention to the rest of the words in a sentence. 
+At the core of doing self-attention is a QKV calculation 
+
+![image](https://github.com/user-attachments/assets/af27e08c-2658-4129-92a8-5fa497a1b5fb)
+![image](https://github.com/user-attachments/assets/d09dfb6a-1f87-4a61-9b9e-978ef01be2f1)
+
+where Wq, Wk, Wv are learnable params of the model. For each token, to calcualte its attention to itself and others. In a sentence each word (token) gets a turn. Each word, gets its Q, K and V, we calculate the softmax, multiply by the V and we get the sum - Z, which is the attenion (self+to other tokens/words), and we do that computation for each token (word) in the sentence.
+
+So... we get the Z values, now the structure of a sample encoder block is:
+
+![image](https://github.com/user-attachments/assets/c15f1043-0cda-4551-9ecf-313fa417151c)
+
+We give input tokens, in the self-attention QKV is done, we get the Z ouput, which we put through a FFN that is basically linear layers that learn information, and then that output is passed into the next encoder block, which does the same. 
+
+![image](https://github.com/user-attachments/assets/a41b62d7-31d1-4ba5-b1de-3b53aaec8e1f)
+
+A sample Encoder layout ^
+
+Inside the encoder block we have a few more calculations. But before that, there is an important point - positional encoding. 
+
+![image](https://github.com/user-attachments/assets/8b305a67-8c39-48bc-bd3b-3744e9c63f69)
+
+encoders do not have idea of the order. If we take an example sentence: you are a boy vs are you a boy - the order matters and it changes the sentence. Since the encoder doesn't know what is the order, if a sentence was "I like banana and like" - the word like will get the same attention scores, but that would be wrong, because the 2nd like - we can tell it is a mistake, so the encoder should learn and know how to order a sentence.
+
+So,
+
+![image](https://github.com/user-attachments/assets/a0c23c18-eee1-44c4-b8b2-ea23dc9aafe8)
+
+We pass the tokens to the self-attention computation (QKV), but before that we make sure to add positional encoding, so the encoder can know the order of a sentence. Then the outpuz of the self-attention is added with the pre-self-attention vector, and then is normalized, AND THEN it is passed to the FFN, which again is normalized and has residuals added. 
+Next, the decoder takes info from the encoder and tries to generate output. The decoder's structure is as below.
+
+![image](https://github.com/user-attachments/assets/603213cb-96ec-4e60-bfbd-f6da6e81aae0)
+
+When the decoder works, it generates the Q(query) from the current structure that it has (image we have a stard of sentence and I tokens), so it generates the Q based on that, next - it takes the K, V we learned in the encoder, so that the decoder can learn what is the best next word to generate. 
+
+In the decoder there is also masking. But why ? Why masked self-attention?
+
+![image](https://github.com/user-attachments/assets/6ddc20a7-d97c-47a4-b603-d0880947a334)
+
+The decoder generates word by word, so in order to do that we want to mask the future tokens of a sentence, so that the generator does not cheat and look at them. 
+
+![image](https://github.com/user-attachments/assets/b42f9257-434d-45ff-a8a1-76f0641de1ce)
+
+So if we have <sos>, I and am tokens, we will hide the next tokens (in this example a and student), so that the decoder does not cheat. We ensures that the model learns to predict tokens based only on the information available at the time of prediction. 
+We do this masking by a triangular matrix multiplication
+
+(below, we do -inf there to ensure that the output of the softmax will sum to 1 and also that the values are actually 0. If the upper right corner is 0 and not -inf, what we will get in the output masked QKV operation is not 0s but small numbers instead)
+
+![image](https://github.com/user-attachments/assets/2ac5dc5c-a789-4529-b008-87fc7fb666bd)
+![image](https://github.com/user-attachments/assets/32106050-5ad0-43fa-8af5-9fb25120799c)
+
+To sum up, this slide nicely says what is happening when we generate text. 
+We start with the <sos/start> token, get the Q - which represents the decoder's current state and is used to attend to the encoder's output. Then using the K and V with the encoder, we get the attention scores at the current stage, and we end up with a list of values, and based on softmax, we have a list of probabilities of which word may follow.
+
+![image](https://github.com/user-attachments/assets/32db255e-f79a-4d3b-bdc5-493902f31344)
+
+So, now onto the implementation (provided by Professor Choi)
+If we look at the overall structure of a transformer
+
+![image](https://github.com/user-attachments/assets/06bf9b41-101b-4f54-96de-eaadac432665)
+
+We do positional auto encoding to tell the encoder the order
+
+![image](https://github.com/user-attachments/assets/ff50deca-2d85-4df4-87d5-34ac9eee1c3e)
+
+Then we do the multi-head attention (QKV calculation). We can add masking for the decoder part
+
+![image](https://github.com/user-attachments/assets/07306d14-9b02-4fee-a098-f93a96623943)
+
+And then we pass that onto the FFN
+
+![image](https://github.com/user-attachments/assets/bf1969de-8df5-4734-a13b-5a857af67ff8)
+
+Using these building blocks we can then construct a basic transformer. 
+
+Today was a handul as well haha. But this knowledge is crucial for my general understanding of what is popular in the field these days (i.e GPT models).
+
+That is all for today!
+
+See you tomorrow :)

From 3d64453db098bbde0c0360971ccca61b7995ef3f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:37:34 +0900
Subject: [PATCH 139/638] Create 2024-2-17-day-47.md

---
 _posts/2024-2-17-day-47.md | 103 +++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 _posts/2024-2-17-day-47.md

diff --git a/_posts/2024-2-17-day-47.md b/_posts/2024-2-17-day-47.md
new file mode 100644
index 000000000..be044e642
--- /dev/null
+++ b/_posts/2024-2-17-day-47.md
@@ -0,0 +1,103 @@
+![image](https://github.com/user-attachments/assets/89152884-599e-461f-a8bd-4a6a40ab5159)---
+layout: post
+title: (Day 47) Learning a bit more about GANs and finding more KAIST courses
+categories: [applying-knowledge,theory]
+---
+
+## Hello :) Today is Day 47!
+A quick summary of today:
+* Looking at GANs from DeepLearning.AI's perspective
+* Trying to find more KAIST courses
+
+1) GANs with DeepLearning.AI
+
+GANs... a war between a generator and a discriminator
+
+![image](https://github.com/user-attachments/assets/2686dde5-fade-410b-8172-f837f66cfdbd)
+
+Given X, the disc tries to discriminate between dogs and cats, and the generator, given random noise and a class, it tries to create the X and trick the discriminator
+How do we train GANs?
+We start with the discriminator
+
+![image](https://github.com/user-attachments/assets/ab650e26-b8d6-49e7-9bc0-d1aeeaa618fa)
+![image](https://github.com/user-attachments/assets/2dda79b6-3106-4ce1-9c15-09587f484ba5)
+
+We give it real and fake examples. It gives its prediction on the fake images, calculate the loss on the fakes, then give it the real images, and calculate the loss on the real images. And the disc loss in this case is the avg of both. 
+
+Next,
+
+![image](https://github.com/user-attachments/assets/ff1cb7ff-4a88-4c33-a44e-596d17c62890)
+![image](https://github.com/user-attachments/assets/a061f172-45c0-4360-a090-141b9a89ef87)
+
+Then we train the generator,  we make random noise again, make fake images, get the discriminator to evaluate the fake images, and we get the generator loss. 
+Also the reason we use random noise is so that we dont get the same image each time. 
+
+Next - problems with traditional GANs
+Mode collapse - that is when the generator gets stuck one of the modes (on a two or multi-modal data) - i.e. if the discriminator gets tricked by a certain type of data x1 that is amongst x1...x100, then the generator will keep producing x1s because x1s trick the discriminator (which is the goal of the generator).
+
+BCELoss
+
+![image](https://github.com/user-attachments/assets/f70d18cf-0bef-4752-b55d-47ad745cb1c7)
+
+The objective of the generator is to make the generated distribution be as close as possible to the actual
+
+![image](https://github.com/user-attachments/assets/bd4c762f-0ced-48b6-a8bf-e9f2683ab211)
+
+The generator's role is to produce images that trick the discriminator while the discriminator's job is just to discriminate (judge real or not). So many times the discriminator can start outperforming the generator very quickly. At the start the disc is not perfect so it can be harder to discriminate fake and real and will give back a non-zero gradient back to the generator. 
+
+![image](https://github.com/user-attachments/assets/073a9929-04d0-4daf-beec-9f9abc820aaa)
+
+But with some 'practice' the discriminator can improve faster than the generator
+
+![image](https://github.com/user-attachments/assets/098a5448-3191-4bab-ba37-7d48d7bd0afe)
+
+and the 2 distributions will be separated and the discriminator will start giving close to zero grads to the generator. This will stop the generator from improving. 
+
+To overcome this problem, a new type of GAN arrived - Wasserstein GAN (WGANs).
+
+![image](https://github.com/user-attachments/assets/da96a047-4e5c-46b7-a570-662ffdbc7802)
+
+it uses the so-called W-Loss which offers stable training dynamics, meaningful gradients and better mode preservation. 
+
+![image](https://github.com/user-attachments/assets/f72bb869-e119-4b17-a463-95dff7763bf9)
+
+How does it do that ? It makes sure that at any point, the norm of the gradient is at most 1.
+
+![image](https://github.com/user-attachments/assets/e689a562-b144-4dea-ada8-d4e8b771fa52)
+![image](https://github.com/user-attachments/assets/0b61d0ce-60ca-4648-bba5-2405707322de)
+
+It checks at every point and it assures that the w-loss is continuous, differentiable, and also it doesnt grow too much and maintains some stability during training.
+
+But how does it enforce this ?
+with weight-clipping
+
+![image](https://github.com/user-attachments/assets/349d0cb8-b986-4323-bcb5-936d5e60bc01)
+
+I learned about Conditional GANs too. 
+
+![image](https://github.com/user-attachments/assets/9c475038-d8f6-4c5f-802d-606bfa44254c)
+
+Along the random noise, you give it a OHE vector, so that the generator can know what class its making. Also the discriminator knows which class it should be discriminating (whether that image is of the given class or not)
+
+![image](https://github.com/user-attachments/assets/239534cd-0d63-4b67-8e14-e08b29353403)
+
+What are pros and cons?
+pros: better and crafting synthetic examples, can generate more labeled examples
+cons: can be limited based on the training data, can overfit the training data.
+
+I saw there are other GANs but I guess this is where my journey with GANs ended for today.
+
+
+2) Finding more KAIST courses (like Professor Choi's)
+
+Motivated from Professor Choi's great course I decided to look at KAIST AI school's curriculum and google the course whether there are slides, videos, assignments from any of the courses uploaded online. I found a few - even from KU(<happy noises>)
+
+* [Maths for AI](https://alinlab.kaist.ac.kr/ai503_2021.html)
+* [DL for CV](https://github.com/suulkyy/ai604-s21/tree/main)
+* Recent advances in DL [here](https://alinlab.kaist.ac.kr/ai602_2023.html)
+
+I am glad, because I found what I am going to do next ^^
+
+That is all for today!
+
+See you tomorrow :) 

From 5df1e6180b3d56307b103d0542f952954f39416e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:42:40 +0900
Subject: [PATCH 140/638] Create 2024-2-18-day-48.md

---
 _posts/2024-2-18-day-48.md | 66 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 _posts/2024-2-18-day-48.md

diff --git a/_posts/2024-2-18-day-48.md b/_posts/2024-2-18-day-48.md
new file mode 100644
index 000000000..9488862c1
--- /dev/null
+++ b/_posts/2024-2-18-day-48.md
@@ -0,0 +1,66 @@
+---
+layout: post
+title: (Day 48) KAIST's AI503 Mathematics for AI (Matrix Decompositions)
+categories: [theory,math]
+---
+
+## Hello :) Today is Day 48!
+A quick summary of today:
+* Started Week 1 - Matrix decompositions of [AI503 Math for AI](https://alinlab.kaist.ac.kr/ai503_2021.html)
+
+Following the book, I read the [chapter](https://mml-book.github.io/book/mml-book.pdf) and took notes on my tablet.
+
+I hope the pics are legible. But the content is:
+- Determinant and Trace
+- Eigenvalues and eigenvectors
+- Cholesky decomposition
+- Eigendecomposition
+- Singular Value Decomposition
+- Exercises
+
+Before the exerises it is mostly the theory. But the real deal was doing the exercises. In particular finding how to do SVD. Firstly I tried to learn how to calculate eigen values and eigenvectors, and then I moved onto SVD. 
+Finding the proper way to solve SVDs... it took me a while. Firstly I followed some medium post, but then it turned out that the person made mistakes which were pointed out in the comments.
+
+SVD formula is A = E*V*U. Some of the problem came from the fact that I was not sure the exact application of this, but I just wanted to learn how to do it now, because down the line I am sure it will lead to something bigger. They key to understanding it all was this [video](https://www.youtube.com/watch?v=cOUTpqlX-Xs) from an MIT professor.
+
+To do SVD, the steps are:
+1) multiply the matrix with its transposed version
+2) Do the detrerminant and find the eigen values
+
+![image](https://github.com/user-attachments/assets/8f73e709-af5f-4cbc-b38d-0ea4c022c13f)
+
+3) From the eigenvalues you get the Epsilon
+
+![image](https://github.com/user-attachments/assets/31974b8a-dfae-4382-a1ab-6a89df7c717d)
+
+4) Find the eigenvectors by deducting lambda indetity matrix from the A transposed * A, then normalize and you get V
+
+![image](https://github.com/user-attachments/assets/2f831353-b683-4f61-8ad6-63f8aae75d3e)
+
+5) Find U by normalizing the product of A and V
+
+![image](https://github.com/user-attachments/assets/b58048ba-cddc-46e0-b59c-809976fd0e9e)
+
+At first I was not sure how to get the Epsilon, another problem was when doing the eigenvectors, how do I get v1 and v2 (i.e. equalizing the matrix to 0), but I found out that the right result is, by what column vector do I have to multiply the matrix in order to get 0. Another issue was finding U, just how to do it. Many websites that do 'step-by-step' tutorial actually skip some steps because they assume the reader knows, but thanks to the MIT video most of these 'holes' were fixed. The above is my final exercise, but below are all my notes :)
+
+![image](https://github.com/user-attachments/assets/dcc6d23f-bb04-4b0f-a9d9-958b0bbbf121)
+![image](https://github.com/user-attachments/assets/2a25a6d9-860a-487a-9bb6-02247ff303f5)
+![image](https://github.com/user-attachments/assets/40328283-bd52-47eb-848f-92df546f9cb1)
+![image](https://github.com/user-attachments/assets/fb921a32-07f6-467d-a003-f595adbc0b86)
+![image](https://github.com/user-attachments/assets/8f4d9e74-4634-4216-b6d7-6fcbf69560ff)
+![image](https://github.com/user-attachments/assets/f4e6d6a5-8187-438d-ba23-c29c068e3467)
+![image](https://github.com/user-attachments/assets/803d7e0e-165f-4a4b-92ed-cd941a338a46)
+![image](https://github.com/user-attachments/assets/c419ec08-a070-42e7-bd6b-578f339d59e4)
+![image](https://github.com/user-attachments/assets/ab759274-3ce8-4cb5-9aa4-ec2b05149044)
+![image](https://github.com/user-attachments/assets/5a8ad570-dcc4-4dcc-8aa5-9d1bfc4c9eec)
+![image](https://github.com/user-attachments/assets/37985cb7-de7d-4fd7-8024-c552d5a98f34)
+![image](https://github.com/user-attachments/assets/75a7f003-5853-403a-9888-9d5b7ebf9ed6)
+![image](https://github.com/user-attachments/assets/4da50c28-9656-4ab6-9533-2d70d49961e8)
+![image](https://github.com/user-attachments/assets/8d87714a-ec3b-4517-81f4-12ad3edaa761)
+
+I am actually still not 100% sure the results I got for SVDs are correct only because I saw comments noting that minuses and the order of values in some matricis matters. I asked in a ML community but I am waiting for a response.
+
+
+That is all for today!
+
+See you tomorrow :) 

From 32fe09621a2bde9864a8aeaa09c032e1307d7c8b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:43:17 +0900
Subject: [PATCH 141/638] Update 2024-2-17-day-47.md

---
 _posts/2024-2-17-day-47.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-17-day-47.md b/_posts/2024-2-17-day-47.md
index be044e642..84a43ac80 100644
--- a/_posts/2024-2-17-day-47.md
+++ b/_posts/2024-2-17-day-47.md
@@ -1,4 +1,4 @@
-![image](https://github.com/user-attachments/assets/89152884-599e-461f-a8bd-4a6a40ab5159)---
+---
 layout: post
 title: (Day 47) Learning a bit more about GANs and finding more KAIST courses
 categories: [applying-knowledge,theory]

From 1a051685b47721ce9b14b104eaf1b536ef188ec3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:48:46 +0900
Subject: [PATCH 142/638] Create 2024-2-19-day-49.md

---
 _posts/2024-2-19-day-49.md | 53 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 _posts/2024-2-19-day-49.md

diff --git a/_posts/2024-2-19-day-49.md b/_posts/2024-2-19-day-49.md
new file mode 100644
index 000000000..a61e6439f
--- /dev/null
+++ b/_posts/2024-2-19-day-49.md
@@ -0,0 +1,53 @@
+---
+layout: post
+title: (Day 49) KAIST's AI503 Mathematics for AI (Continuous optimization, When models meet data, Linear regression)
+categories: [theory,math]
+---
+
+## Hello :) Today is Day 49!
+A quick summary of today:
+* continued with AI503
+  * continuous optimization
+  * when models meet data
+  * linear regression
+
+I think I found a good method for studying these topics:
+1) Read the chapter
+2) Check the professor's slides to see which points He brough up
+3) Take notes while reading the chapter again
+
+**Continuous optimization**
+
+![image](https://github.com/user-attachments/assets/9dc21542-c932-4341-a047-9ee6d3f2d88a)
+![image](https://github.com/user-attachments/assets/b84c43dc-0dbe-4766-973a-9e9ef8e4f6ce)
+![image](https://github.com/user-attachments/assets/992caaac-666d-403e-b352-5b1312f5dd87)
+![image](https://github.com/user-attachments/assets/6dc83809-341f-4b7f-9364-726974780768)
+![image](https://github.com/user-attachments/assets/8d57016a-b6c4-41bf-9eb3-a6e6239d004c)
+![image](https://github.com/user-attachments/assets/82a49df8-7f8f-486c-b766-aedb3e36d9f8)
+![image](https://github.com/user-attachments/assets/77cc063e-160a-44f2-9ddb-82e8b864c3c1)
+![image](https://github.com/user-attachments/assets/78209800-b2e2-4d44-a8cc-c268e966499b)
+![image](https://github.com/user-attachments/assets/081149be-f1aa-4e1c-aac1-cd4a31b59357)
+
+**When models meet data**
+
+![image](https://github.com/user-attachments/assets/f1295e54-857d-407b-bb3c-215c1706cd5e)
+![image](https://github.com/user-attachments/assets/a33f9d55-2653-4655-8be2-f6765a06958f)
+![image](https://github.com/user-attachments/assets/ef8383d3-fed5-444d-abde-9ea99c337a64)
+![image](https://github.com/user-attachments/assets/3a7d5be2-3078-44d5-94f4-8183c82e15e6)
+![image](https://github.com/user-attachments/assets/a840c0ea-8ec7-401e-bb3c-68650f2fdc92)
+![image](https://github.com/user-attachments/assets/4dba768b-b993-4ac1-884d-09015105e91a)
+![image](https://github.com/user-attachments/assets/2563ec10-a1cb-45d6-a923-cabcd4fd04fc)
+![image](https://github.com/user-attachments/assets/c849bfd1-9b0e-4c9d-8451-f440a466ebba)
+![image](https://github.com/user-attachments/assets/76116446-c0a0-40e2-b6b8-2ddb26ad2b76)
+![image](https://github.com/user-attachments/assets/d56f2bb9-5d9f-4b72-8d14-bf09d314ce84)
+
+**Linear regression**
+
+![image](https://github.com/user-attachments/assets/05c91ef1-adbe-4457-87ec-efabd64a66a1)
+![image](https://github.com/user-attachments/assets/180e66e3-eb82-4d26-9bbb-52f87b00dd05)
+![image](https://github.com/user-attachments/assets/fbd483b1-ddf3-4d19-a0b5-0363abcce6bf)
+![image](https://github.com/user-attachments/assets/b47a8de9-ea82-4722-abd1-23a0bb77bf9d)
+
+That is all for today!
+
+See you tomorrow :)

From 07f4c929cc058a68526df0f6bf837264ea0f7efc Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:55:32 +0900
Subject: [PATCH 143/638] Create 2024-2-20-day-50.md

---
 _posts/2024-2-20-day-50.md | 63 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 _posts/2024-2-20-day-50.md

diff --git a/_posts/2024-2-20-day-50.md b/_posts/2024-2-20-day-50.md
new file mode 100644
index 000000000..1c3975a62
--- /dev/null
+++ b/_posts/2024-2-20-day-50.md
@@ -0,0 +1,63 @@
+---
+layout: post
+title: (Day 50) My ML journey does not end today!] KAIST's AI503 Mathematics for AI (PCA, GMM, SVM)
+categories: [theory,math,statistics]
+---
+
+## Hello :) Today is Day 50!
+
+And it is not the last day. I will continue every day, studying about ML and DL and writing about it in this blog. After this post, I will go change the title to 50+ days of ML haha
+
+When I decided to do this (which was around Christmas) I did not know what was I going into, whether I will be motivated to not only study it every day but also write this blog about it. Tbh writing the blog is sometimes harder - mainly because I end up writing it around 2-3am where I am sleepy and just wanting to go to bed haha. 
+
+But Machine learning and Deep learning completely took over me. I am always attracted to the difficult. In ML I see something that is extremely difficult and challenging, a topic so deep and full of potential that makes me excited to study it everyday. ML involves so many things and topics that the amount just makes me excited and motivates me to continue. Because I know that even if I learn X, there is X+1 which is harder, but also many times there is X-1 (some basics) which are harder than X itself. 
+
+This vacation I found something... something that challenges me, and because it challenges me - it excites me. 
+
+A quick summary of today:
+* Lecture 5: Principal Component Analysis
+* Lecture 6: Gaussian Mixture Models
+* Lecture 7: Support Vector Machines
+
+These are the last 3 lectures and chapters of the [Mathematics for Machine learning](https://50daysml.blogspot.com/2024/02/Mathematics%20for%20Machine%20Learning) book
+
+Today I remembered an advice from lecture one of [KAIST's AI503 Math for AI](https://alinlab.kaist.ac.kr/ai503_2021.html) course:
+'Do not ponder where this mathematics is useful'
+
+When I started the course I kept this in mine. "I will focus on understanding how the math works, how we derive formulas" - because seeing how some formulas are useful and where are they useful is very hard because if I learn one formula in chapter 1, maybe after sometime I will learn that this formula is used in chapter 10 (of the same or maybe even a second book haha). 
+
+**PCA**
+
+![image](https://github.com/user-attachments/assets/68604080-9481-4315-b1e7-81e1698069b6)
+![image](https://github.com/user-attachments/assets/3189038a-a56e-45db-9739-50459a8f6bf7)
+![image](https://github.com/user-attachments/assets/7a058a9b-0f18-4d13-b469-5d34319a793f)
+![image](https://github.com/user-attachments/assets/2a4e39cf-9dba-4150-9c25-40f9a5883179)
+![image](https://github.com/user-attachments/assets/47c4981f-5ca7-4fdc-8c6c-1b95d6bdcda5)
+![image](https://github.com/user-attachments/assets/33e33240-0b00-42c6-b7b0-d0c68b628820)
+![image](https://github.com/user-attachments/assets/3bfda9fd-02f6-42c6-8bae-40036fc1aea4)
+![image](https://github.com/user-attachments/assets/c0950b73-9053-48f2-bb67-cc1f459a13a9)
+![image](https://github.com/user-attachments/assets/f9eab74f-6da2-4c00-b5b7-7da306801a2d)
+![image](https://github.com/user-attachments/assets/c03f9d2b-dfb9-4581-ac67-28b7382f6582)
+![image](https://github.com/user-attachments/assets/062e493d-ed7a-4d1b-a6bc-7b027b426e4c)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEicBr-MPMDt1ncrP8qLhyponO6L_ygLqAssSXsLKHS9xbAdB4l46KthmXo6vXm8uPij3x9hsGuFbC3W59idM7L5H_ezqk0-CEck4lJgf1BiunuPiegurJL5UAAUC9-pGQZfoT9H9DawcWP_bHRbGZbu3PpdSvjmpNvdgvaBgzV6iT2fqO22gy2JZcickH3u)
+
+**GMM**
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEiX9fi-z-bP1N7MeP-GeX8-so_0FvGJgMWIFf3kViwydncwGrFSQTwGFoDfTNGqzHx_RxCH1STr9I0sSkqpLBapuG6vJCeRQMOog0VUEV3knFT-YrGnRFFB8EK-831ieHwChROAXllJxS8hA8XfNhN_wOAijXFbxmYHPQYYMxwtG2tbJKSD8sc7oRwjCTHY)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEib7yNvkaU5v1uYwC3vqLit3asd1gYz3bMJxqp2Iyt0HtYTZx4zE8p2tgapig6-rbeeXqIlL3hYkpvr2vO3hEANZpu4U5ZHi-mFEPoCVomfxTl0Nyz3FIEqkbvW4Z9wA4uH1vc47qtDHItKWAEmAJ0TvJu9FtadaIsqO_Szv54gMcbCha9-pGn8HvTSWqMg)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjLpUcGPotxwmLyRHkTGsN3deE97iG-MkdCVuFa4VL9A_He0vzavYEg4gab96r29eUuiCwV5YLpF22GtsbLbCYDscBLLg6MezqmQrzM_Jrs5T8e9So4c2_OlZVpcLu_98O89yleHocYZ0MR5MV481z6xK0ujqvzGPoiMvMuqo5UnPQvUSFTYz-sRXDaD3RD)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhsHxShYyw1dPoGuS-m3Jm6o4jMfKmvwRZhgNtvnYWLPQxH2QW_BVR9c87WaWHzZXmXO7L_lcOhWDxK0lYx-ROagbkURNESvdatQxfPss8wERYLGWQzQb-9uATHBDDufxp-9Mmaabhr7XbeJhBP0klK7-Fu6Lxb0wdGJHjvYnL-Hr-Lc_TpWzFspb4Mfn2B)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEitne75PQ-sf6AQmBJJOOuYoeUhDpinwADtqPKwjG9HPGdI249iqh09zRSzrW4cEhmwvD6QwioHzC-K1ROVwriHNxwA9GAAXPYMwl_cvol49jlEBMPugF1hdzIr01Ri8Jp-I2AV97Zg4vTxYA_Gjq-avpAz4tD6EbOJ5upc5Iems-LV1zKlLk_2WiBJN2EX)
+
+**SVM**
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjch35Vbyuy7c5PfMUBS6KhSSdOZTByTE3EooCmETQb557SMxq-6GB3HT2Ys_qELrU_lxSZDA4vemSDvFyiSODZpH7N3tOcmgJsw2H2STd-4QzMBluMpRCAoIV1jHMw6_9KlbqPbZEwRk4Fz1-6avzD-XJVO1hEoT8usoc3b9R93Zq79vouPiGBlLiSNPib)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjg2OQUP68v95JMK1kZ7AFA3viqH02s55DhXULFxwlve2zKGN4onomTSfzbsVFy4rAgGUlmLzZVAh7qjKkz6M5jh2ibq6fRJfvS0Pj69ImargexsmDEyTQO2aucRkD5OUKdSNcZxfAMYmNvh6R3Iw8SR-cNKsS7xQj9_MmkpXZtD9Cv3Z5RtZTlW0Kjmsa5)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjhoonB8APOA690z3fvWzuYyST0XWcl6mAIMwwH2yiOz9SVuUcxfP0heSUuhKgz4gmQUglNNEg6T_1t4u7VpsII8z_w3r9kh44AxHebj8e8ByfkEqgl8ek-eMiNE1dWEJzEO_YM_gq14lutRCl61huRtkeqiz38aln6C9Lla_t8qVVM6BPqyWPTmi46x7Fh)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhH_5d9swesQY-fcFit3dvL39bHUojj34oCTSE8NJczWnRfDreOwfAH8ygKgRDwqOzlqH8PNAOcmywIQVdC07KLlzxfgnq4CFzKwRx5h7Pck76zzudEUiyDokNxq78x-D9trlOIkaCVtemlSicqfd1S686wl1tyoBz88xj8XLUb0-YtPMTAMueD1Tcx90fx)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEiHektKumNQANkGa8-z8eHxvxnEP-kPRSzunjaJq5CbwD5qyBWrIo150fitVRVINtkIZF5h3-cMav_-a5DfvElAhzFMusVzm6pVdaS7_8peqUsDs-Agdwos83BevOEM-ZMJViXqZY0XNbTEDu0af2-WI0ZmyeaIYWR4YqJ7Lgx58wrLq9LpNBHzZZK0NkYA)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhhKVvcnqLNAmb3o7QmR9EuwMd-XIjwGfVWjYY8FDhztiuZp9CzCNdDYzDasCpTbmokICgdslfmfVgpRQpY0YfPLQaJUCCkwS2BCTThJqGCfVhVKK_GufE6TXhG8_E8X4cl-7Uj8qCZpzlCnll6Rk5UdKR3DxCza-47CLKQIojRTns1TOCaKKX1VMkkzFb9)
+
+That is all for today, day 50! Not the end!
+
+See you tomorrow :) 

From 30f61ac55ef8432d9df9deaa08b48011a052b0d2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 20:56:35 +0900
Subject: [PATCH 144/638] Update 2024-2-20-day-50.md

---
 _posts/2024-2-20-day-50.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-20-day-50.md b/_posts/2024-2-20-day-50.md
index 1c3975a62..c01ca345e 100644
--- a/_posts/2024-2-20-day-50.md
+++ b/_posts/2024-2-20-day-50.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 50) My ML journey does not end today!] KAIST's AI503 Mathematics for AI (PCA, GMM, SVM)
+title: (Day 50) My ML journey does not end today! - KAIST's AI503 Mathematics for AI (PCA, GMM, SVM)
 categories: [theory,math,statistics]
 ---
 

From 90679529ea0f332d1b2d108db9233b236c37dc3e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:05:07 +0900
Subject: [PATCH 145/638] Create 2024-2-21-day-51.md

---
 _posts/2024-2-21-day-51.md | 48 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 _posts/2024-2-21-day-51.md

diff --git a/_posts/2024-2-21-day-51.md b/_posts/2024-2-21-day-51.md
new file mode 100644
index 000000000..3f008c898
--- /dev/null
+++ b/_posts/2024-2-21-day-51.md
@@ -0,0 +1,48 @@
+---
+layout: post
+title: (Day 51) More of AI503 - High-dim space, random walks and markov chains, VC-dims
+categories: [theory,math,statistics]
+---
+
+## Hello :) Today is Day 51!
+A quick summary of today:
+* Lecture 8: High-dimensional Space 
+* Lecture 9: Random Walks and Markov Chains 
+* Lecture 10: VC-Dimension 
+
+**High dim space**
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgWhe71dEDkH9o5XXk7QLC_UyIBu8S-1Z6kimcvG-HQUbFq-DZ9vs7MptKip9V9DMmCFY2ofwfMfPHRDO_LPSfgu3acGMjGNhqTSMcQplIyWRYhi-PDEdZOydtqhFlRsRedDk8ZkT-HzkvCS9agLJ7BTfXL9xciXcXKTt-jHvqthhIet7U2GtuDRsysBM1v)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgbze6GePgF49kTeIjzdW_qeW_3vxncQLAle5_8w9-GnEPU2l3ViegQda6KISq1wrso6TOwq3LBy597pvh9dc3Vx2PyPKuN-Pqd9iRI6ZP334UzWXoqwcqL_dtfYo7Afeph91-go0KZdkfA1SBKZVmInisqikxQPs5U7IPzMpja8ODnh4O5u4WhEynnWaY2)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjDBYGhp5_mHN9xM-REm-nfOK_6QAsAonRBlmWX30HKhp_vbKlpwUHzwbx3SfD759qy4wKTczDJ9MPQVFBtX4kSJx1jPtz4eaORX3QflIxd5GI4fcHIk8e7g1AhL6Ms0OWLUTRMD2XQ7__RAbBYIf9SJaxXVmVM2n8WQF7QgEKPdGKt2hlA1o87Yqx0ReKM)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhWnbj2BoOla7_QeNswTGeOtmyVPzeTMyBpGm58q8PPmuekVda-jdPXfjMxXkKgmz8pyN91yw31vQOvFpNc-z4Hp1kceMeqYKp8EQ5z5L26CIBIc3lebpSxQEzPlafYLHsU_GXtxLI4N4288CCfgipxdGqzKFEN9El5nlDTydkDQdlRypFhqylAgmkQFAyF)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEii0xz9riqFcMrbaqEiq9JkeFn0bXWiQRINAz0JIBSFDtGi-y-z-e3Dcaw12Et7fnBhKnBiMW2Z4iQh4FfK0KuJFmVPqTWVWKUzwJwfHf9Uy6Dj8KEZ2zzPT_TnxSq_muM-Zkv67JyUKmpFMD0Vvvdo6f4EXxB6mt9rbuNnpjcA531PLy9ZxXjmwiJidChS)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgy-dKEWQywzB-SDzDHeSSjuv9muAyX41EX-wjZFJdXU_1Wv1BynvrjImmfLTtuZukaf_T-SJwKSH3WDBi0oFd1EmGF-oHFC0_t_nV7fAkm3Lg8MwP2i0hlSEuzzSSPGu9vTDUQbM0gxIqbYQaFqRVoX2S5hmqGmcP6-8lBHc1VgUAbCAz23ZHRJh1PXAhe)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgu45gF_jF-Kcjgiok-w5Xn986rZQOkTuTrA_zv_MHHVWJRv2CTziNwynqntf4DzqszrGR5TD_SMB230U0_RtOvBC9JtNfobhCDEW-ruM2LI0ckwvasC8QiJAxtfXja7_Mi8xA6tW0WidXHEbqu99GlQaKu5P6LqAJ2zDBFCJHYgUieCUFzfO9aBWZ1B9AT)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhrTxB2a6shnRSOcftzUGdyp3IdXwjG07okuSY8joRLNdejYIA61jc3N7z4_jAdve-Wq9zjYuSZ5biVKCJJ26_cU3fsTKd3Rx59aNfjPksd0Q7WrECxDDrukUf_dvYZc7h7GgQpFYi0TESz25jIANJyfLJu4_RTqlxUKVwkMrkBZEztUINu2klc8SHPFGdV)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEipJq5I9W1a0jOOwMhimTn1NSK8eCCpoEoxwbBRzhZvm3bXrNVqFGuRRjPYWtpPoGp9QxsrJttyeritT7vXNLgJlbYGeTlE84qmOXK190agjboHwN03LpysrWkbgbZ-Ev0dg4jGFd95LS6MQvHWiIavW8OYViLiZA3mDL6fmZ8iJPMpnb4FRNQyNx1mSKSO)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjGVKViNKyJHfI1dQvvKu6XiR6kAzR0Rpd7LnEQ9xmzlw2LsXRpq8UG4RQrFz781UAtOYpHolMmTTyZMqQ1tGwcZPBV_yL_LRKEq-D5X-MJqNamF6SM-wf7gp5QRAlemxMk_KlCJSZDEZgyIoAkBtWk5eRnJVRPhoOTMswMl3V73dmOOqLFLtP5MbCKR1l2)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEj78kie7ZbBfm26Iy3QaLwuJFdbJ1MLbk3IYWUhBplHx07b42N82t6Ix0jjyv_AzFXvWHRAgRDjOht9Drz59leQEEjDMh0kS0y8udsWPJtWbS90HUgofGWQola2yS706tBVjk4wkTEl5avpDWdDUzCzo6fGD41lQc_I9PJpRwlnZGilhdSof4Ix9pVYSOGx)
+
+**Random walks and Markov chains**
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjDTlkqYsPo2wD533lYogYyPsEu9aaSIfosRqhSYlBeFfBUzgEuXrk2uI-VEFA24Jll4FDaCVaTQYVjz54mYpTzHhN76-RM6YZgU6yPonzM7gJq9-7u0vk1e-QFIXCSb_nwEaqiB80BOl7gWfx0POO7mIN_zQrEnA_MJOA3n82S8CD_KUKcyM9xGbG3t0wd)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjbGPxSAZh7awwd8BTYYoJa1r8ZR7oLKCCiy4_o2213tv87jbDOLJS9Z5YKhzAAGBXt9MZyeHx2LPWnrVK-HCCQJsY7asTc2dl_Qn-RI6nOti8c2N7RBXbZDDnshkImplTksxM9d_uqOKcIU27JR9CATVV-bY0Qrkr-6d_veFx1G_ak4eCBea_ssKvZ45ia)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEh9075ANFjjpsvlyUfaco31I0eDwLRFEbRXcHvplNu0DR2QGhkRLwreoEIXpsteWCWeKD5cUZ_wry0UF2TahCPAvyBhzrvxVeAlzMIEcCXqZ_6FtjOcdYDuGZgj88qDdfdDLA8ySvp9pk0NhvGpCo1y2-mpn9jqFVmgyjo2WorZfAFGqN7Jcala8Dvb7qGO)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEiHMdhY07VUVTLu2JLyDCdHF7p7jSegok19hbZ4vHblpetPejIak5bqSfOD88ku0uQ1ylBPvPKOapVmNr7yE4qYEM1VR8r-thDViLlUZB-ljmh1ncam6QQwOGYbLs2L-fywBLc6fKprgcL8D7RayBFtJ4LARcBYHtV9YZ8RsdFzoc-pjApXuliZ7gCYbFwO)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEi-pFd_0qcOXdzlMAOuZD3-vMlOJP6TR3jMw1aBc4x5uZ6WO9_yaRSC0Pex3C2CrjZF30A5OT_dxWRCIJwl1TUNeduyxZOIRvsgS5stvd8Y7t9slZwdxYoPMWcwwBoqL0pF_YxAKjU49OBgpikXYzOVMvDGM5smts_YpIpXN9Gj7qmQXn-3OIa05zz5tn5L)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjMBIKpeZTqMy4shxIIJopbZ63Z2p1C7mGMflwC6Qv-YYn73fZmFxm5fL2dKrYWgy6TyRANCOtRQkPUK2wdzaOKrUv9MLdPrcBWwsiuXFxot2ocn6UFQxSGBGasIp943eJfuvE_zVmxsCAyXM9WpJnJjRia09x2i22Ff4vbw0MKAFCbGjSmmzmSZk2SOO9T)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgL0MGJqGGvP6LaenFQAv6sf636DgARhpLbwZ9EV2_mo0Ni957dTT9OPKN9gBlxn28-dKSCzqslJV-v3EIgpq1q1HRdoOZmnt6LilG4m890XPghe0j_A1eoMHXordPpOFFCoN60hF1Xrqku--Yq9WrpracrrgwAvq8oHZQJdmi4Rh29uZ-EeQdtzHYTRl2H)
+
+**VC-dimension**
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEiXer6TuY5-IrE7xqR-Os-oGw90MaySPCoqJhIAv_VppThreKV03yU3zpz7ytoGPxkVkg6AU15bAIQORB0mPJiDx6OPjwNRhLBLfq4C_EABYIjXXFLkSK4CrqrWjHuk5pdm0nK2UGG7F62H93i5jK6n70GMbvU4fkem6GCTd_vOeTySyByUmu5D2T13qseS)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgi8DTTjxV7Onj5ZcxcTI1T2IBG8ybMzv88j1uqgvO0sqRExnCV9KmEbRatl2832Lf9i1zhmFNLAxxcvJNIqnhP2BHAe_pkl8UzRf-eQUQCyH3xe9DqmU9DmayarW0zkq9DigPOrTz3sUoUI5z1A39MHyQV9pII_ZAQen46qpRjnmJuq_ua7MP2LoYK0oHR)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgEjzBsaZgyvQwZ6eDQtQwIlhnIO_wVFQ1RJqb1UP4purY-6H5KrJIQD59ra8VEa9n8T0CnUInJYGFfJdx-pbwg6XtmcKskEYYNZgXd82E9Me39jnRz3cCc44_qC7j_geFROzgiVOWz0-hy9mGe6qmI2SNYMkxmA747aSHvUy4TssdOHRX1GxMPOkble06u)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjWXntivu6JSGo-_fZ7gGGNcvpuOr1lF7AYNr11xMlvcPxSS_drst0juLzKvDK5MDQkC644cZdDbPBfAKZDLHMSdTr8PYharMjYufUM3hkDmxmD9tRQRLIWX6GH9TUogVpKiuwx9ea4-BSIaRjfwrq0dHZ_zFBt4NAAyJiQ0lvxoFJi25nIrlsfRoHt6lHt)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhphf6TjGhNEkUMsG4ZBU0UEFvooY-bmqv0q5d_cmEwdncYT1vb427OccAtjADrIfnrDj9HfndbFSpxOrAzv5XmkQGCa4U8iIaJPsDKAV40Jb_ILZfRegPEQ4mv4Fr52eZ4AGNSb-ihs5_t1YRpNcJ2MZVAJM7yIiNwh8MYA3-LVS2SvvLm1M6LE6HorEAc)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhB6NeYGm9mClyXjSDF_fdwpKQmefeItzEnggpqSL2dzd1xdV5nK3i-3m8-Bni9PmUEzoxJp1KlPuUBPs2CVLELcyZBMzY2GFLpPKsxBADHNIek-_DGX4FatlhUDTokN_ydwb0Tmm-d1zV4Lwod9aES32teiriZRjzaVw1ZsuZ4PhxFU2cAv0OSPC6SodLQ)
+
+That is all for today!
+
+See you tomorrow :)

From d6a2290f60c4cc1abd50713ef92ae09ed44d231a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:13:28 +0900
Subject: [PATCH 146/638] Create 2024-2-22-day-52.md

---
 _posts/2024-2-22-day-52.md | 133 +++++++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 _posts/2024-2-22-day-52.md

diff --git a/_posts/2024-2-22-day-52.md b/_posts/2024-2-22-day-52.md
new file mode 100644
index 000000000..25d00aa25
--- /dev/null
+++ b/_posts/2024-2-22-day-52.md
@@ -0,0 +1,133 @@
+---
+layout: post
+title: (Day 52) Learning more about transformers with Andrej Karpathy
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 52!
+A quick summary of today:
+* [GPT from scratch with Andrej Karpathy](https://youtu.be/kCc8FmEb1nY)
+* Re-watching his other tutorials 
+
+Firstly, I will mention about what I learned from the GPT from scratch tutorial.
+Andrej Karpathy's videos introduced me to PyTorch for the 1st time. Actually the first video I saw which was about building a neural network from scratch, I kinda got what was happening, but as he went deeper and deeper and started writing code as if it was PyTorch (or building a neural network on a lower level compared to TensorFlow), I felt like I got slapped in the face hahaha. TF is much higher level than PyTorch, and at first it felt weird to have to write the whole dataset creation, model, training, eval, etc by myself. But with some practice it grew on me and now PyTorch feels more comfortable than TF, it feels more 'down-to-earth' ('down-to-python') haha. 
+
+So, on Day 46 I really tried to understand and learn transformers through KAIST's professor Choi and his lecture. And it was incredibly helpful. And today I got reminded that Andrej Karpathy has a lecture/video about GPT from scratch, and I can use that to confirm whether I understand transformers, and even see an implementation from scratch from the man himself - Andrej Karpathy.
+
+So, here are some of my notes from the GPT from scratch video. 
+
+In the transformer there is the trick that is done for efficient self-attention.
+
+We can do it with a for loop, but this is inefficient. 
+
+When we do self-attention, we want for every token to have the attention to itself and the tokens before it. A basic way to do it is to use for loops to average the past context. As in the pic below.
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEj7zPLBtERl6Q69wDpzDgDabPuF-PRO3q0PycqdJuKRhfBUx9jE8CGmBcdDCGuLoqH7mKydoSlhA3wQnMIIXJzVenGmTLL2vP_pT998rsPKvZlwoV8Ynw_p90xTW3rXTP-dJ2VDlN1WqQpXW53NAR_RNn3bVg0YigunOAWz1GgN8A-3FS7uOyOTSiVD0EcP)
+
+In this pic, what we do is in xbox, the 1st row is a an avg of the 1st row in x, then the 2nd row is an avg of the first two rows (tokens), etc. But this is not what is used in transformers, it is inefficient. 
+
+Instead,
+in pytorch there is this torch.tril method that creates a lower triangular matrix in size X, X. (like var a in the picture)
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEi8oAqRiIdD5d22JZAPxn9hsqx7zqSOkb1BUzqCMAFxFnWiqKYzbr1fVJAN1yziRhroBlNe9dZDkczJehzpWTwKs4N6RwfD9IMxIaWUBEp6wAbU5P2OK4vixA2-FF2KxHNFxDqaHh6vLRopJJqpt-jwjsJrIdxTnnxh0VCncDilDfphxNxECNwOOVT7Nfcb)
+
+and when we can also make the matrix to have each row normalized - which is what we have in the printed a. And we can get that same result as in the for loop, but by using matrix multiplication which is more efficient. 
+Here we create xbow2 which is the result of matrix multiplication 
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEg7AaaIDHaD3o6HiJFjwQMBW3x7MLQIEpEyk4irlpchLi9yt4e154bzf6Lr6sumYmA-7fN6WkKbN2qOUbmfRRaWW_0fe7dNDengVmEEC4k0i_uFbUARjc1njxVWLC7MydoXLWM5Lb0d0cvUnlIAWG0J7NLfZxIe--R46lnoyKScCvxgfLHbh06tG9W7nJXX)
+
+and when we compare xbow (from the for loop) with xbow2 (from the matrix multiplication method), we get the same matrices, as below.
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjl2nhaRXDixrudqfFNpS6NBssnb-mVhhgOFj0H-TnlFup30TPl9U7_qvEmr8e-iZPc2pToDAH6Ou5camrQCfbHCoxBjN7coU3KerJ475hZZvdE8DDb0Yy21Q3vbGKYSL5a3Nr717ImqQkeO8HpUNiRMry8GbQ8qgCgH_UfnmYl6rZj9HCVkDt8Vl8QPUTp)
+
+What we want to do next is apply softmax, but before that, rather than 0s in the right triangle of the matrix, we want to fill it with negtive infinites
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEiR1a148yRxzpWiGAtSy0zg6TVvGGe0meTfrNygkWNqCPEmtJAjVEksz7akaZMkBGxXvF7kZqltAvuO2AyCHbjy28dL6sFgus_gAAI20TG2kCa_71Be77jJTXb0UxgVc4fo-GIZOuyB6snYNK52_Yp9Vbifqm2psoao4yZPiTg9Zdh5DkPZzHwYJ4UK45dZ)
+
+This is because when we pass -inf to softmax, it will give 0 - which is what we want. 
+Btw, the first row, where we have 1 value and the rest is inf, this means that in that row, this is the 1st token in the sequence, and then we mask the future, because we don't want the model to cheat and look into the future when it predicts, so we pass new info one by one. Then the 2nd row is 1st and 2nd token, etc. This is why we care for masking the right half of the triangle - to stop the model from cheating by looking into the future tokens. We want it to predict the next token based on the current available information only. 
+
+Going back to softmax, after we have the above, and we apply softmax to the rows as they are, we get:
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEh5IvatZYoB7X_YpW5bDKddHKfHreEnwfcNndotKJluoOP-5vRa4IfDLrmuVdGsLlyxsU57wT2Y7rIFaNu4TXCGCQcnL7YU7pzK6vY99wK1D2l3LEeDbZi3KtTX4e9gFQ-AhcXmSsRe7FH1O2-u_OyvnAWqaxjdo9E3D2iW15PBshDzYBYlx0YqvR4kae2V)
+
+each row in the lower triangular normalized. 
+
+But doing it this way with average, we don't learn much about each word. For example in a sentence some words have more importance than others, and some words 'look' for other words. Like in 'Ivan is the guy who ate the sandwich' - probably the word guy places higher priority on Ivan. We want some tokens to find other tokens more interesting. 
+That is where self-attention comes in. Every token will emmit a Q and a K, and essentially - Q is what am I looking for, and K is what do I contain. So how we do it is a dot product of my Q with all the other Ks, and that dot product, becomes our wei (weights).
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEigr5ABrBz2wQ0MTi9NTq0_0XpPbVD_2N_QIT2-67rdRPjIzC6yyKKSFc_frymBqwmm0ZtuiTrjxOkUweJHGrVvsCnTdReE4d4HRhOOhpYZ9jAyvKxZvssX6o2Uxhq2S1mZGktXrqliM-aKxxDsmQRWig1hwIyPqWzJJuJrHdsZcErJQuChcTRfIIDiKeg9)
+
+These are the raw weights:
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgDWBUFg8Pt7yr9vxXtNxZ-jM2VmGzyp6H9gu17gCbUwEW523RszGjySq5jzhSB1edI7B_YYJLZ-Bvs_0n3gd6OpCsgPdKfuDmEkkMNko90imwi88VrJRxIKJ_Vc5NzB5iBXvTtflS6vRgAeRHbnxwI5yDWezYWoLnJmWCDwaMdKKyVX5UdaBspVSzhsGfU)
+
+Then we mask the future tokens
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEj3VD_YuFLVTn1wLTovL_3KkLJQEzleYq0KSWAqRUTYGik-Sx3Nx-tVhwd-ZJl6Jk61f-0Zk3JD7ypGjMuaexK22GT0RxFsgAAN5dBuv_2iQR9egpBoMlRPXhva7dWFm9kIz3MCe2ykJ1Ws1td0i1IPQ9SvyBan4sqEQGb6CDlXGt8siXFZHUIxkxWlxK0K)
+
+And apply softmax and now we can tell how much of info to agg from any of these tokens in the past.
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhjId9Hni5TSdm9O_k-lf6tZ2-IWyYCrSyU9WNPSCJmcAldETMwv4iv5kHurrm9nKylhVVeEPflJyaY5Nk0SoNq9-tcLMy3Mf5iBtsqSVB57GB-DWB34F23Q3HoeU_evgpp3nRDE1vwvTllESATC5q0zWZUExu2-qyBj8o8QmyGCqTNiKqZWwVwM59cbHXd)
+
+For example in the last row of wei[0] we can tell that the last index 'likes' the 3rd index hence it gave it 0.22 priority. 
+
+So far so good. And here is the self-attention formula from the paper.
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhqUC4GQ3n1byIi5fNibqpAdf4eG5nz-rrqm_TS6bIo9DEg3JptVJ3fCaqn3JJIHvdVmEbQ97_GYDgdnHfHMOgOVGvcbYPbHBL1lxjzDzv7WH7ffm6X79hxXJsyVUBqkIDg6wVirvpA6XCBEfUIxMMqDgrwFFp-YA55Z8xYS8Jdu4URY6gFPxH7mgDZPlGl)
+
+We have softmax, Q, K and V, but we do not have the division by sqrt(dk) which is the amount of heads. 
+Since we have unit Guassian inputs, the variance of the wei will be similar to the amount of heads (in the code it was 16), and below the var is 15.9.
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhkIQNyYZKnD9z61FOnKURplSU0cbSkUQQniaLh7QI05QatUnRN2UN7g7AaF9iZx3d2YBvp-avbhuUFKqTac1nMrM0JbeHZqFMOfuTUtEJLg3axJW2pw8-g-0whERwMwTcldAt-1XGSYC94itMrVaKHxNjhbmbieJ08Mo6kPeSx_kLcS5W9x0_qcBGjeUCB)
+
+If we divide by the sqrt(head_size) it is all fixed
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjlZ_8YQl0hylvo-asoWn6oBdbYdZqy9Bu-Krg5VYeu8qsTNQtdMM8383EU7fSFgFFb_85lHRgWuXHewNrOqlLqWFsgDNNxOEPCuK0_deXFmW2lHXeppscxVutaUg-5pgU3AOjvvRkv_Nq6wiRJrk7_sS_dSxAwSsMeXG-hSL2wo7Y9ag5ICCrEAEXP4pfh)
+
+But why is that important? Because if we give softmax very big values, it will start returning tensors that have a value very close to 1 and the others close to 0. 
+Below, we see in the 1st case if we give softmax values that are closer to zero, we get normal outputs.
+But in the 2nd cell, if we skyrocket the values, the results are close to one-hot-encoding. And why is that a problem? This would result in all nodes aggregating info from one single node, so they will not be learning a lot, which is important especially in the beginning of training. 
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEgMkYhHPDmwFEPWublhXhsVQABSn-hqHqyEY91cvC4R-08B43oUXTRbUvhQtXaEuwIJ8eMAMo2QpJCNwBxHBg9lpoQYpqqZFsqm6UIXAKKYy9A1B7bh7nfLrMUZQ86o2GTeHP0wWNK3cRVpBn8yz1fJDb4NOyvcQtEvE5qDXny9KEq8ogWBPB4F8efHI9Dj)
+
+I am glad I saw Andrej Karpathy's video today. Also I saw that he even uploaded a new video about GPT's tokenizer. Definitely something on my to-do list. 
+
+
+Also, I rewatched the below three videos of him. There are two more as part of this 'makemore' series, which I will review later. 
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEjJcDV8l1DvqDuBJLB2N5H5CJKGPE__oDZvS7-denKYtKZxq3-QuDnc6Y4JVcMAaMVl5V4ag5l2FOK0yeAOUNNhccGqiuy8vmQf8pDSgNdzgM8_RmrdVZTxBTcFWLFhQmO3iPGHFy2VGpO_ZBDp0b_DVFLQm2zAWP4aJPhjDFDQt7B3Zy_cAkbHiaaB2iGs)
+
+I noticed from these videos that compared to the 1st time when I watched them, I understand what is going on haha. I think the main problem was that tensorflow does a lot things for the dev whereas with pytorch I have to write things myself. Rather than taking notes and thinking: wait, why ? where? how?, rewatching these felt like a 'oh yes, yes, this and that' - affirming my knowledge and understanding. Simply put, I got some confirmation that I am learning :)
+
+Nevertheless I took some pics of something interesting - related to parameter initialization. 
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEhtzzaW5t7LaYPw-k2tg9eHk-S0I23cOptFLwT4cTc159sopj8ReL1N75j_tDFqXhpQKixm2VINCQqfxgP-hKIos3kV94iYRjrTYErnlFoCImP7CG6G-WtBRx3qmU7e0VU5_LV4_qKqZN71jMhGWfu9mBIFo0lsulymLiSS-84iv2YdliTOvbRVCWw1LTdD)
+
+In this pic, the first epoch's loss is 27, and then it jumps down rapidly. At init, the network assigns to some characters higher probability distribution compared to others and some characters are very confident and some are the opposite. And the model becomes very confidently wrong which results in the high loss - and this can be caused by the activations in the last layers on in the first layers. If we init correctly, we can help training be more efficient and avoid hockey stick loss curves.
+Also if we see the output of the cell below, we get 3.29, which is the negative log likelihood that we should expect in our case where we have 27 characters - log(1/27.). Such big loss on the 1st epoch may indicate incorrect param inits, and if we init correctly, we should get a loss that is close to what we would expect. 
+
+Also with regard to vanishing gradients he made these nice graphs
+
+[](https://blogger.googleusercontent.com/img/a/AVvXsEiFRiF2whqK_c60IusT8NdGmnVuQFqiBHTM6M_6gNDhvr-_aqgl8-1LFkaCLhw5VZiw1XiRJo-TWHLGOAxyMfzlhJpBHSfXBVz9rmM4O3Mh2v7l1Gxsew_osdJadh4u-daP2RkSRR496V1-pmvzqEB5WoVIUsJzvtq_vZ93dQ6QD_vekapcjrjhS-dU4MwQ)
+[](https://blogger.googleusercontent.com/img/a/AVvXsEg5bEGhV9lkyR5JCp3DUKYMBFACm2F_4LtmsgiIC3GZb8OR3ZAEYkAbwYRqmMuLe7lzIzL67FgmUg43Xat7Ur4YcmBDoU7O5RHhUjWRMeo9_P8FSA9rifmdK8DOfSo9yH7JhO2JqcMWBOBIisvv6hmxyp9-_Q1ig7AS9Pb6zmoYJevkEK3i8QpepUp11b-C)
+
+The 1st one - activations' distributions across layers.
+The 2nd one - gradient's distributions across layers. 
+But what we see is that the distributions change throughout the model. 
+In the forward pass' case, the model we use, uses tanh activations, and tanh squashes the activations, and they start to reach towards -1 and 1, and if we zoom in on the pic, we can see that the 1st layer (blue) we see a normal distribution, but as we go, the distributions starts to become more saturated - which can result in vanishing gradients in the backward pass. 
+In the backward pass' case, we can see vanishing gradients, in the 1st layer (which in this case is the last layer - layer 4), the gradients have a normal distribution, but as we go through training, the distributions become squashed down and more saturaded - and we can see vanishing gradients.
+
+Actually I started watching the 'become a backprop ninja' video by Andrej, but in there he mentioned an article - [Yes you should understand backprop](https://karpathy.medium.com/yes-you-should-understand-backprop-e2f06eab496b), where I saw the word 'eingevalue' which was kind of cool. I learned about them and how to calculate them, and now I saw them in the context of matrices. 
+
+I will need to refer to this video/his code later when I want to build some kind of a gpt myself.
+
+Another note: I found two courses that have videos/notes + assignments:
+* [Stanford University's cs231n's assignments](https://cs231n.github.io/)
+* [Deep learning for computer vision by Justin Johnson (I saw him in one of Andrej's lectures)](https://web.eecs.umich.edu/~justincj/teaching/eecs498/WI2022/)
+
+Definitely, I need to do these 2 courses.
+
+That is all for today!
+
+See you tomorrow :)

From ea20f8722bb1ca8c21de268508bd28e7bd9b56b5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:18:37 +0900
Subject: [PATCH 147/638] Create 2024-2-23-day-53.md

---
 _posts/2024-2-23-day-53.md | 57 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 _posts/2024-2-23-day-53.md

diff --git a/_posts/2024-2-23-day-53.md b/_posts/2024-2-23-day-53.md
new file mode 100644
index 000000000..a5b48c2cf
--- /dev/null
+++ b/_posts/2024-2-23-day-53.md
@@ -0,0 +1,57 @@
+---
+layout: post
+title: (Day 53) Getting closer to becoming a 'backprop ninja' (thanks to Stanford Uni's cs231n assignments)
+categories: [applying-knowledge,theory,deep-learning]
+---
+
+## Hello :) Today is Day 53!
+A quick summary of today:
+* Started Stanford University's [cs231n](https://cs231n.github.io/)
+  * [Image Classification: Data-driven Approach, k-Nearest Neighbor, train/val/test splits](https://cs231n.github.io/classification/)
+  * [Linear classification: Support Vector Machine, Softmax](https://cs231n.github.io/linear-classify/)
+  * [Optimization: Stochastic Gradient Descent](https://cs231n.github.io/optimization-1/)
+  * [Backpropagation, Intuitions](https://cs231n.github.io/optimization-2/)
+
+First I will quickly share my notes from the 1st three, and then for the big one (backprop) - will do it last.
+
+1st lecture - Image classification using KNN
+Actually my impression is not so much from the theory in the lecture, but from the assignment exercises which the course has. 
+
+The KNN assignment was about implementing the loss of KNN (L1/L2 distance) and how to do it efficiently. 
+
+Firstly there was the most inefficient one - with two for loops
+
+![a](https://blogger.googleusercontent.com/img/a/AVvXsEgBxdteU9X-a6cq-N-JIdWhwmAj7OLZXewPFBW5AZjEDvTh5_tYpQOlwQvYX0tXdJGY0TP2p5VjTHCTM01P2jAlm7fa2YkSwgsoota9cvj4O5as-mmE4zTaKVtmEJV4R61eVD8BmQfNJyIm0bxcW5gtqy_uQRIv4mBNN-JFE85uWZEdvN8_6-9Ac6WWpkP7)
+
+Then it was with one for loop
+
+![a](https://blogger.googleusercontent.com/img/a/AVvXsEgk0OCj_DkIyM69INv3vEFrnW_pFIhAoQEdF3KZUxnyB3vPDypPQeWfEWO3kaDkP00swmZ1GMe23yFAwwlxf-Uc-tSTqDT-eXbNt8izImDPbFCZ6gRnUDT0CVkzX3zIfvYv3ZuBYDnSc8lELtodGOg_S6exHBdYTzMaQz2-GzlIAKRTzbC3XOFao5jc1Gzz)
+
+And then, the most efficient one which uses matrix multiplication magic
+
+![a](https://blogger.googleusercontent.com/img/a/AVvXsEirDSwFbTeiRJB-l8Fx9sjngVLOyPcEtZZljrD5jZh1nmbFnOK2TYkSZtsm6Y1W4Nm3RExZylaT8Kg9QcApFC8UCwrEQR7cRlS27SPTHFRBMekcQT7WMiIgLNCJhh_1T01lmlFhMw1CxsyDntw8vwmaVaSKqBlc-h9BKqZbbl7e5p9RgBim0JZa_qtzOZm1)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From 75d9c36388113e2e83639e4eaac350c90b522c08 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:19:46 +0900
Subject: [PATCH 148/638] Update 2024-2-22-day-52.md

---
 _posts/2024-2-22-day-52.md | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/_posts/2024-2-22-day-52.md b/_posts/2024-2-22-day-52.md
index 25d00aa25..0d5a68737 100644
--- a/_posts/2024-2-22-day-52.md
+++ b/_posts/2024-2-22-day-52.md
@@ -22,95 +22,95 @@ We can do it with a for loop, but this is inefficient.
 
 When we do self-attention, we want for every token to have the attention to itself and the tokens before it. A basic way to do it is to use for loops to average the past context. As in the pic below.
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEj7zPLBtERl6Q69wDpzDgDabPuF-PRO3q0PycqdJuKRhfBUx9jE8CGmBcdDCGuLoqH7mKydoSlhA3wQnMIIXJzVenGmTLL2vP_pT998rsPKvZlwoV8Ynw_p90xTW3rXTP-dJ2VDlN1WqQpXW53NAR_RNn3bVg0YigunOAWz1GgN8A-3FS7uOyOTSiVD0EcP)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj7zPLBtERl6Q69wDpzDgDabPuF-PRO3q0PycqdJuKRhfBUx9jE8CGmBcdDCGuLoqH7mKydoSlhA3wQnMIIXJzVenGmTLL2vP_pT998rsPKvZlwoV8Ynw_p90xTW3rXTP-dJ2VDlN1WqQpXW53NAR_RNn3bVg0YigunOAWz1GgN8A-3FS7uOyOTSiVD0EcP)
 
 In this pic, what we do is in xbox, the 1st row is a an avg of the 1st row in x, then the 2nd row is an avg of the first two rows (tokens), etc. But this is not what is used in transformers, it is inefficient. 
 
 Instead,
 in pytorch there is this torch.tril method that creates a lower triangular matrix in size X, X. (like var a in the picture)
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEi8oAqRiIdD5d22JZAPxn9hsqx7zqSOkb1BUzqCMAFxFnWiqKYzbr1fVJAN1yziRhroBlNe9dZDkczJehzpWTwKs4N6RwfD9IMxIaWUBEp6wAbU5P2OK4vixA2-FF2KxHNFxDqaHh6vLRopJJqpt-jwjsJrIdxTnnxh0VCncDilDfphxNxECNwOOVT7Nfcb)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi8oAqRiIdD5d22JZAPxn9hsqx7zqSOkb1BUzqCMAFxFnWiqKYzbr1fVJAN1yziRhroBlNe9dZDkczJehzpWTwKs4N6RwfD9IMxIaWUBEp6wAbU5P2OK4vixA2-FF2KxHNFxDqaHh6vLRopJJqpt-jwjsJrIdxTnnxh0VCncDilDfphxNxECNwOOVT7Nfcb)
 
 and when we can also make the matrix to have each row normalized - which is what we have in the printed a. And we can get that same result as in the for loop, but by using matrix multiplication which is more efficient. 
 Here we create xbow2 which is the result of matrix multiplication 
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEg7AaaIDHaD3o6HiJFjwQMBW3x7MLQIEpEyk4irlpchLi9yt4e154bzf6Lr6sumYmA-7fN6WkKbN2qOUbmfRRaWW_0fe7dNDengVmEEC4k0i_uFbUARjc1njxVWLC7MydoXLWM5Lb0d0cvUnlIAWG0J7NLfZxIe--R46lnoyKScCvxgfLHbh06tG9W7nJXX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg7AaaIDHaD3o6HiJFjwQMBW3x7MLQIEpEyk4irlpchLi9yt4e154bzf6Lr6sumYmA-7fN6WkKbN2qOUbmfRRaWW_0fe7dNDengVmEEC4k0i_uFbUARjc1njxVWLC7MydoXLWM5Lb0d0cvUnlIAWG0J7NLfZxIe--R46lnoyKScCvxgfLHbh06tG9W7nJXX)
 
 and when we compare xbow (from the for loop) with xbow2 (from the matrix multiplication method), we get the same matrices, as below.
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjl2nhaRXDixrudqfFNpS6NBssnb-mVhhgOFj0H-TnlFup30TPl9U7_qvEmr8e-iZPc2pToDAH6Ou5camrQCfbHCoxBjN7coU3KerJ475hZZvdE8DDb0Yy21Q3vbGKYSL5a3Nr717ImqQkeO8HpUNiRMry8GbQ8qgCgH_UfnmYl6rZj9HCVkDt8Vl8QPUTp)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjl2nhaRXDixrudqfFNpS6NBssnb-mVhhgOFj0H-TnlFup30TPl9U7_qvEmr8e-iZPc2pToDAH6Ou5camrQCfbHCoxBjN7coU3KerJ475hZZvdE8DDb0Yy21Q3vbGKYSL5a3Nr717ImqQkeO8HpUNiRMry8GbQ8qgCgH_UfnmYl6rZj9HCVkDt8Vl8QPUTp)
 
 What we want to do next is apply softmax, but before that, rather than 0s in the right triangle of the matrix, we want to fill it with negtive infinites
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEiR1a148yRxzpWiGAtSy0zg6TVvGGe0meTfrNygkWNqCPEmtJAjVEksz7akaZMkBGxXvF7kZqltAvuO2AyCHbjy28dL6sFgus_gAAI20TG2kCa_71Be77jJTXb0UxgVc4fo-GIZOuyB6snYNK52_Yp9Vbifqm2psoao4yZPiTg9Zdh5DkPZzHwYJ4UK45dZ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiR1a148yRxzpWiGAtSy0zg6TVvGGe0meTfrNygkWNqCPEmtJAjVEksz7akaZMkBGxXvF7kZqltAvuO2AyCHbjy28dL6sFgus_gAAI20TG2kCa_71Be77jJTXb0UxgVc4fo-GIZOuyB6snYNK52_Yp9Vbifqm2psoao4yZPiTg9Zdh5DkPZzHwYJ4UK45dZ)
 
 This is because when we pass -inf to softmax, it will give 0 - which is what we want. 
 Btw, the first row, where we have 1 value and the rest is inf, this means that in that row, this is the 1st token in the sequence, and then we mask the future, because we don't want the model to cheat and look into the future when it predicts, so we pass new info one by one. Then the 2nd row is 1st and 2nd token, etc. This is why we care for masking the right half of the triangle - to stop the model from cheating by looking into the future tokens. We want it to predict the next token based on the current available information only. 
 
 Going back to softmax, after we have the above, and we apply softmax to the rows as they are, we get:
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEh5IvatZYoB7X_YpW5bDKddHKfHreEnwfcNndotKJluoOP-5vRa4IfDLrmuVdGsLlyxsU57wT2Y7rIFaNu4TXCGCQcnL7YU7pzK6vY99wK1D2l3LEeDbZi3KtTX4e9gFQ-AhcXmSsRe7FH1O2-u_OyvnAWqaxjdo9E3D2iW15PBshDzYBYlx0YqvR4kae2V)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh5IvatZYoB7X_YpW5bDKddHKfHreEnwfcNndotKJluoOP-5vRa4IfDLrmuVdGsLlyxsU57wT2Y7rIFaNu4TXCGCQcnL7YU7pzK6vY99wK1D2l3LEeDbZi3KtTX4e9gFQ-AhcXmSsRe7FH1O2-u_OyvnAWqaxjdo9E3D2iW15PBshDzYBYlx0YqvR4kae2V)
 
 each row in the lower triangular normalized. 
 
 But doing it this way with average, we don't learn much about each word. For example in a sentence some words have more importance than others, and some words 'look' for other words. Like in 'Ivan is the guy who ate the sandwich' - probably the word guy places higher priority on Ivan. We want some tokens to find other tokens more interesting. 
 That is where self-attention comes in. Every token will emmit a Q and a K, and essentially - Q is what am I looking for, and K is what do I contain. So how we do it is a dot product of my Q with all the other Ks, and that dot product, becomes our wei (weights).
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEigr5ABrBz2wQ0MTi9NTq0_0XpPbVD_2N_QIT2-67rdRPjIzC6yyKKSFc_frymBqwmm0ZtuiTrjxOkUweJHGrVvsCnTdReE4d4HRhOOhpYZ9jAyvKxZvssX6o2Uxhq2S1mZGktXrqliM-aKxxDsmQRWig1hwIyPqWzJJuJrHdsZcErJQuChcTRfIIDiKeg9)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEigr5ABrBz2wQ0MTi9NTq0_0XpPbVD_2N_QIT2-67rdRPjIzC6yyKKSFc_frymBqwmm0ZtuiTrjxOkUweJHGrVvsCnTdReE4d4HRhOOhpYZ9jAyvKxZvssX6o2Uxhq2S1mZGktXrqliM-aKxxDsmQRWig1hwIyPqWzJJuJrHdsZcErJQuChcTRfIIDiKeg9)
 
 These are the raw weights:
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgDWBUFg8Pt7yr9vxXtNxZ-jM2VmGzyp6H9gu17gCbUwEW523RszGjySq5jzhSB1edI7B_YYJLZ-Bvs_0n3gd6OpCsgPdKfuDmEkkMNko90imwi88VrJRxIKJ_Vc5NzB5iBXvTtflS6vRgAeRHbnxwI5yDWezYWoLnJmWCDwaMdKKyVX5UdaBspVSzhsGfU)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgDWBUFg8Pt7yr9vxXtNxZ-jM2VmGzyp6H9gu17gCbUwEW523RszGjySq5jzhSB1edI7B_YYJLZ-Bvs_0n3gd6OpCsgPdKfuDmEkkMNko90imwi88VrJRxIKJ_Vc5NzB5iBXvTtflS6vRgAeRHbnxwI5yDWezYWoLnJmWCDwaMdKKyVX5UdaBspVSzhsGfU)
 
 Then we mask the future tokens
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEj3VD_YuFLVTn1wLTovL_3KkLJQEzleYq0KSWAqRUTYGik-Sx3Nx-tVhwd-ZJl6Jk61f-0Zk3JD7ypGjMuaexK22GT0RxFsgAAN5dBuv_2iQR9egpBoMlRPXhva7dWFm9kIz3MCe2ykJ1Ws1td0i1IPQ9SvyBan4sqEQGb6CDlXGt8siXFZHUIxkxWlxK0K)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj3VD_YuFLVTn1wLTovL_3KkLJQEzleYq0KSWAqRUTYGik-Sx3Nx-tVhwd-ZJl6Jk61f-0Zk3JD7ypGjMuaexK22GT0RxFsgAAN5dBuv_2iQR9egpBoMlRPXhva7dWFm9kIz3MCe2ykJ1Ws1td0i1IPQ9SvyBan4sqEQGb6CDlXGt8siXFZHUIxkxWlxK0K)
 
 And apply softmax and now we can tell how much of info to agg from any of these tokens in the past.
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhjId9Hni5TSdm9O_k-lf6tZ2-IWyYCrSyU9WNPSCJmcAldETMwv4iv5kHurrm9nKylhVVeEPflJyaY5Nk0SoNq9-tcLMy3Mf5iBtsqSVB57GB-DWB34F23Q3HoeU_evgpp3nRDE1vwvTllESATC5q0zWZUExu2-qyBj8o8QmyGCqTNiKqZWwVwM59cbHXd)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhjId9Hni5TSdm9O_k-lf6tZ2-IWyYCrSyU9WNPSCJmcAldETMwv4iv5kHurrm9nKylhVVeEPflJyaY5Nk0SoNq9-tcLMy3Mf5iBtsqSVB57GB-DWB34F23Q3HoeU_evgpp3nRDE1vwvTllESATC5q0zWZUExu2-qyBj8o8QmyGCqTNiKqZWwVwM59cbHXd)
 
 For example in the last row of wei[0] we can tell that the last index 'likes' the 3rd index hence it gave it 0.22 priority. 
 
 So far so good. And here is the self-attention formula from the paper.
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhqUC4GQ3n1byIi5fNibqpAdf4eG5nz-rrqm_TS6bIo9DEg3JptVJ3fCaqn3JJIHvdVmEbQ97_GYDgdnHfHMOgOVGvcbYPbHBL1lxjzDzv7WH7ffm6X79hxXJsyVUBqkIDg6wVirvpA6XCBEfUIxMMqDgrwFFp-YA55Z8xYS8Jdu4URY6gFPxH7mgDZPlGl)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhqUC4GQ3n1byIi5fNibqpAdf4eG5nz-rrqm_TS6bIo9DEg3JptVJ3fCaqn3JJIHvdVmEbQ97_GYDgdnHfHMOgOVGvcbYPbHBL1lxjzDzv7WH7ffm6X79hxXJsyVUBqkIDg6wVirvpA6XCBEfUIxMMqDgrwFFp-YA55Z8xYS8Jdu4URY6gFPxH7mgDZPlGl)
 
 We have softmax, Q, K and V, but we do not have the division by sqrt(dk) which is the amount of heads. 
 Since we have unit Guassian inputs, the variance of the wei will be similar to the amount of heads (in the code it was 16), and below the var is 15.9.
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhkIQNyYZKnD9z61FOnKURplSU0cbSkUQQniaLh7QI05QatUnRN2UN7g7AaF9iZx3d2YBvp-avbhuUFKqTac1nMrM0JbeHZqFMOfuTUtEJLg3axJW2pw8-g-0whERwMwTcldAt-1XGSYC94itMrVaKHxNjhbmbieJ08Mo6kPeSx_kLcS5W9x0_qcBGjeUCB)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhkIQNyYZKnD9z61FOnKURplSU0cbSkUQQniaLh7QI05QatUnRN2UN7g7AaF9iZx3d2YBvp-avbhuUFKqTac1nMrM0JbeHZqFMOfuTUtEJLg3axJW2pw8-g-0whERwMwTcldAt-1XGSYC94itMrVaKHxNjhbmbieJ08Mo6kPeSx_kLcS5W9x0_qcBGjeUCB)
 
 If we divide by the sqrt(head_size) it is all fixed
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjlZ_8YQl0hylvo-asoWn6oBdbYdZqy9Bu-Krg5VYeu8qsTNQtdMM8383EU7fSFgFFb_85lHRgWuXHewNrOqlLqWFsgDNNxOEPCuK0_deXFmW2lHXeppscxVutaUg-5pgU3AOjvvRkv_Nq6wiRJrk7_sS_dSxAwSsMeXG-hSL2wo7Y9ag5ICCrEAEXP4pfh)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjlZ_8YQl0hylvo-asoWn6oBdbYdZqy9Bu-Krg5VYeu8qsTNQtdMM8383EU7fSFgFFb_85lHRgWuXHewNrOqlLqWFsgDNNxOEPCuK0_deXFmW2lHXeppscxVutaUg-5pgU3AOjvvRkv_Nq6wiRJrk7_sS_dSxAwSsMeXG-hSL2wo7Y9ag5ICCrEAEXP4pfh)
 
 But why is that important? Because if we give softmax very big values, it will start returning tensors that have a value very close to 1 and the others close to 0. 
 Below, we see in the 1st case if we give softmax values that are closer to zero, we get normal outputs.
 But in the 2nd cell, if we skyrocket the values, the results are close to one-hot-encoding. And why is that a problem? This would result in all nodes aggregating info from one single node, so they will not be learning a lot, which is important especially in the beginning of training. 
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgMkYhHPDmwFEPWublhXhsVQABSn-hqHqyEY91cvC4R-08B43oUXTRbUvhQtXaEuwIJ8eMAMo2QpJCNwBxHBg9lpoQYpqqZFsqm6UIXAKKYy9A1B7bh7nfLrMUZQ86o2GTeHP0wWNK3cRVpBn8yz1fJDb4NOyvcQtEvE5qDXny9KEq8ogWBPB4F8efHI9Dj)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgMkYhHPDmwFEPWublhXhsVQABSn-hqHqyEY91cvC4R-08B43oUXTRbUvhQtXaEuwIJ8eMAMo2QpJCNwBxHBg9lpoQYpqqZFsqm6UIXAKKYy9A1B7bh7nfLrMUZQ86o2GTeHP0wWNK3cRVpBn8yz1fJDb4NOyvcQtEvE5qDXny9KEq8ogWBPB4F8efHI9Dj)
 
 I am glad I saw Andrej Karpathy's video today. Also I saw that he even uploaded a new video about GPT's tokenizer. Definitely something on my to-do list. 
 
 
 Also, I rewatched the below three videos of him. There are two more as part of this 'makemore' series, which I will review later. 
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjJcDV8l1DvqDuBJLB2N5H5CJKGPE__oDZvS7-denKYtKZxq3-QuDnc6Y4JVcMAaMVl5V4ag5l2FOK0yeAOUNNhccGqiuy8vmQf8pDSgNdzgM8_RmrdVZTxBTcFWLFhQmO3iPGHFy2VGpO_ZBDp0b_DVFLQm2zAWP4aJPhjDFDQt7B3Zy_cAkbHiaaB2iGs)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjJcDV8l1DvqDuBJLB2N5H5CJKGPE__oDZvS7-denKYtKZxq3-QuDnc6Y4JVcMAaMVl5V4ag5l2FOK0yeAOUNNhccGqiuy8vmQf8pDSgNdzgM8_RmrdVZTxBTcFWLFhQmO3iPGHFy2VGpO_ZBDp0b_DVFLQm2zAWP4aJPhjDFDQt7B3Zy_cAkbHiaaB2iGs)
 
 I noticed from these videos that compared to the 1st time when I watched them, I understand what is going on haha. I think the main problem was that tensorflow does a lot things for the dev whereas with pytorch I have to write things myself. Rather than taking notes and thinking: wait, why ? where? how?, rewatching these felt like a 'oh yes, yes, this and that' - affirming my knowledge and understanding. Simply put, I got some confirmation that I am learning :)
 
 Nevertheless I took some pics of something interesting - related to parameter initialization. 
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhtzzaW5t7LaYPw-k2tg9eHk-S0I23cOptFLwT4cTc159sopj8ReL1N75j_tDFqXhpQKixm2VINCQqfxgP-hKIos3kV94iYRjrTYErnlFoCImP7CG6G-WtBRx3qmU7e0VU5_LV4_qKqZN71jMhGWfu9mBIFo0lsulymLiSS-84iv2YdliTOvbRVCWw1LTdD)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhtzzaW5t7LaYPw-k2tg9eHk-S0I23cOptFLwT4cTc159sopj8ReL1N75j_tDFqXhpQKixm2VINCQqfxgP-hKIos3kV94iYRjrTYErnlFoCImP7CG6G-WtBRx3qmU7e0VU5_LV4_qKqZN71jMhGWfu9mBIFo0lsulymLiSS-84iv2YdliTOvbRVCWw1LTdD)
 
 In this pic, the first epoch's loss is 27, and then it jumps down rapidly. At init, the network assigns to some characters higher probability distribution compared to others and some characters are very confident and some are the opposite. And the model becomes very confidently wrong which results in the high loss - and this can be caused by the activations in the last layers on in the first layers. If we init correctly, we can help training be more efficient and avoid hockey stick loss curves.
 Also if we see the output of the cell below, we get 3.29, which is the negative log likelihood that we should expect in our case where we have 27 characters - log(1/27.). Such big loss on the 1st epoch may indicate incorrect param inits, and if we init correctly, we should get a loss that is close to what we would expect. 
 
 Also with regard to vanishing gradients he made these nice graphs
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEiFRiF2whqK_c60IusT8NdGmnVuQFqiBHTM6M_6gNDhvr-_aqgl8-1LFkaCLhw5VZiw1XiRJo-TWHLGOAxyMfzlhJpBHSfXBVz9rmM4O3Mh2v7l1Gxsew_osdJadh4u-daP2RkSRR496V1-pmvzqEB5WoVIUsJzvtq_vZ93dQ6QD_vekapcjrjhS-dU4MwQ)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEg5bEGhV9lkyR5JCp3DUKYMBFACm2F_4LtmsgiIC3GZb8OR3ZAEYkAbwYRqmMuLe7lzIzL67FgmUg43Xat7Ur4YcmBDoU7O5RHhUjWRMeo9_P8FSA9rifmdK8DOfSo9yH7JhO2JqcMWBOBIisvv6hmxyp9-_Q1ig7AS9Pb6zmoYJevkEK3i8QpepUp11b-C)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiFRiF2whqK_c60IusT8NdGmnVuQFqiBHTM6M_6gNDhvr-_aqgl8-1LFkaCLhw5VZiw1XiRJo-TWHLGOAxyMfzlhJpBHSfXBVz9rmM4O3Mh2v7l1Gxsew_osdJadh4u-daP2RkSRR496V1-pmvzqEB5WoVIUsJzvtq_vZ93dQ6QD_vekapcjrjhS-dU4MwQ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg5bEGhV9lkyR5JCp3DUKYMBFACm2F_4LtmsgiIC3GZb8OR3ZAEYkAbwYRqmMuLe7lzIzL67FgmUg43Xat7Ur4YcmBDoU7O5RHhUjWRMeo9_P8FSA9rifmdK8DOfSo9yH7JhO2JqcMWBOBIisvv6hmxyp9-_Q1ig7AS9Pb6zmoYJevkEK3i8QpepUp11b-C)
 
 The 1st one - activations' distributions across layers.
 The 2nd one - gradient's distributions across layers. 

From f9f7de449a4489e17a3ac3092167b12ba61215e7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:20:48 +0900
Subject: [PATCH 149/638] Update 2024-2-21-day-51.md

---
 _posts/2024-2-21-day-51.md | 48 +++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/_posts/2024-2-21-day-51.md b/_posts/2024-2-21-day-51.md
index 3f008c898..e4bc1432e 100644
--- a/_posts/2024-2-21-day-51.md
+++ b/_posts/2024-2-21-day-51.md
@@ -12,36 +12,36 @@ A quick summary of today:
 
 **High dim space**
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgWhe71dEDkH9o5XXk7QLC_UyIBu8S-1Z6kimcvG-HQUbFq-DZ9vs7MptKip9V9DMmCFY2ofwfMfPHRDO_LPSfgu3acGMjGNhqTSMcQplIyWRYhi-PDEdZOydtqhFlRsRedDk8ZkT-HzkvCS9agLJ7BTfXL9xciXcXKTt-jHvqthhIet7U2GtuDRsysBM1v)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgbze6GePgF49kTeIjzdW_qeW_3vxncQLAle5_8w9-GnEPU2l3ViegQda6KISq1wrso6TOwq3LBy597pvh9dc3Vx2PyPKuN-Pqd9iRI6ZP334UzWXoqwcqL_dtfYo7Afeph91-go0KZdkfA1SBKZVmInisqikxQPs5U7IPzMpja8ODnh4O5u4WhEynnWaY2)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjDBYGhp5_mHN9xM-REm-nfOK_6QAsAonRBlmWX30HKhp_vbKlpwUHzwbx3SfD759qy4wKTczDJ9MPQVFBtX4kSJx1jPtz4eaORX3QflIxd5GI4fcHIk8e7g1AhL6Ms0OWLUTRMD2XQ7__RAbBYIf9SJaxXVmVM2n8WQF7QgEKPdGKt2hlA1o87Yqx0ReKM)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhWnbj2BoOla7_QeNswTGeOtmyVPzeTMyBpGm58q8PPmuekVda-jdPXfjMxXkKgmz8pyN91yw31vQOvFpNc-z4Hp1kceMeqYKp8EQ5z5L26CIBIc3lebpSxQEzPlafYLHsU_GXtxLI4N4288CCfgipxdGqzKFEN9El5nlDTydkDQdlRypFhqylAgmkQFAyF)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEii0xz9riqFcMrbaqEiq9JkeFn0bXWiQRINAz0JIBSFDtGi-y-z-e3Dcaw12Et7fnBhKnBiMW2Z4iQh4FfK0KuJFmVPqTWVWKUzwJwfHf9Uy6Dj8KEZ2zzPT_TnxSq_muM-Zkv67JyUKmpFMD0Vvvdo6f4EXxB6mt9rbuNnpjcA531PLy9ZxXjmwiJidChS)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgy-dKEWQywzB-SDzDHeSSjuv9muAyX41EX-wjZFJdXU_1Wv1BynvrjImmfLTtuZukaf_T-SJwKSH3WDBi0oFd1EmGF-oHFC0_t_nV7fAkm3Lg8MwP2i0hlSEuzzSSPGu9vTDUQbM0gxIqbYQaFqRVoX2S5hmqGmcP6-8lBHc1VgUAbCAz23ZHRJh1PXAhe)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgu45gF_jF-Kcjgiok-w5Xn986rZQOkTuTrA_zv_MHHVWJRv2CTziNwynqntf4DzqszrGR5TD_SMB230U0_RtOvBC9JtNfobhCDEW-ruM2LI0ckwvasC8QiJAxtfXja7_Mi8xA6tW0WidXHEbqu99GlQaKu5P6LqAJ2zDBFCJHYgUieCUFzfO9aBWZ1B9AT)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhrTxB2a6shnRSOcftzUGdyp3IdXwjG07okuSY8joRLNdejYIA61jc3N7z4_jAdve-Wq9zjYuSZ5biVKCJJ26_cU3fsTKd3Rx59aNfjPksd0Q7WrECxDDrukUf_dvYZc7h7GgQpFYi0TESz25jIANJyfLJu4_RTqlxUKVwkMrkBZEztUINu2klc8SHPFGdV)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEipJq5I9W1a0jOOwMhimTn1NSK8eCCpoEoxwbBRzhZvm3bXrNVqFGuRRjPYWtpPoGp9QxsrJttyeritT7vXNLgJlbYGeTlE84qmOXK190agjboHwN03LpysrWkbgbZ-Ev0dg4jGFd95LS6MQvHWiIavW8OYViLiZA3mDL6fmZ8iJPMpnb4FRNQyNx1mSKSO)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjGVKViNKyJHfI1dQvvKu6XiR6kAzR0Rpd7LnEQ9xmzlw2LsXRpq8UG4RQrFz781UAtOYpHolMmTTyZMqQ1tGwcZPBV_yL_LRKEq-D5X-MJqNamF6SM-wf7gp5QRAlemxMk_KlCJSZDEZgyIoAkBtWk5eRnJVRPhoOTMswMl3V73dmOOqLFLtP5MbCKR1l2)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEj78kie7ZbBfm26Iy3QaLwuJFdbJ1MLbk3IYWUhBplHx07b42N82t6Ix0jjyv_AzFXvWHRAgRDjOht9Drz59leQEEjDMh0kS0y8udsWPJtWbS90HUgofGWQola2yS706tBVjk4wkTEl5avpDWdDUzCzo6fGD41lQc_I9PJpRwlnZGilhdSof4Ix9pVYSOGx)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgWhe71dEDkH9o5XXk7QLC_UyIBu8S-1Z6kimcvG-HQUbFq-DZ9vs7MptKip9V9DMmCFY2ofwfMfPHRDO_LPSfgu3acGMjGNhqTSMcQplIyWRYhi-PDEdZOydtqhFlRsRedDk8ZkT-HzkvCS9agLJ7BTfXL9xciXcXKTt-jHvqthhIet7U2GtuDRsysBM1v)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgbze6GePgF49kTeIjzdW_qeW_3vxncQLAle5_8w9-GnEPU2l3ViegQda6KISq1wrso6TOwq3LBy597pvh9dc3Vx2PyPKuN-Pqd9iRI6ZP334UzWXoqwcqL_dtfYo7Afeph91-go0KZdkfA1SBKZVmInisqikxQPs5U7IPzMpja8ODnh4O5u4WhEynnWaY2)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjDBYGhp5_mHN9xM-REm-nfOK_6QAsAonRBlmWX30HKhp_vbKlpwUHzwbx3SfD759qy4wKTczDJ9MPQVFBtX4kSJx1jPtz4eaORX3QflIxd5GI4fcHIk8e7g1AhL6Ms0OWLUTRMD2XQ7__RAbBYIf9SJaxXVmVM2n8WQF7QgEKPdGKt2hlA1o87Yqx0ReKM)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhWnbj2BoOla7_QeNswTGeOtmyVPzeTMyBpGm58q8PPmuekVda-jdPXfjMxXkKgmz8pyN91yw31vQOvFpNc-z4Hp1kceMeqYKp8EQ5z5L26CIBIc3lebpSxQEzPlafYLHsU_GXtxLI4N4288CCfgipxdGqzKFEN9El5nlDTydkDQdlRypFhqylAgmkQFAyF)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEii0xz9riqFcMrbaqEiq9JkeFn0bXWiQRINAz0JIBSFDtGi-y-z-e3Dcaw12Et7fnBhKnBiMW2Z4iQh4FfK0KuJFmVPqTWVWKUzwJwfHf9Uy6Dj8KEZ2zzPT_TnxSq_muM-Zkv67JyUKmpFMD0Vvvdo6f4EXxB6mt9rbuNnpjcA531PLy9ZxXjmwiJidChS)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgy-dKEWQywzB-SDzDHeSSjuv9muAyX41EX-wjZFJdXU_1Wv1BynvrjImmfLTtuZukaf_T-SJwKSH3WDBi0oFd1EmGF-oHFC0_t_nV7fAkm3Lg8MwP2i0hlSEuzzSSPGu9vTDUQbM0gxIqbYQaFqRVoX2S5hmqGmcP6-8lBHc1VgUAbCAz23ZHRJh1PXAhe)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgu45gF_jF-Kcjgiok-w5Xn986rZQOkTuTrA_zv_MHHVWJRv2CTziNwynqntf4DzqszrGR5TD_SMB230U0_RtOvBC9JtNfobhCDEW-ruM2LI0ckwvasC8QiJAxtfXja7_Mi8xA6tW0WidXHEbqu99GlQaKu5P6LqAJ2zDBFCJHYgUieCUFzfO9aBWZ1B9AT)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhrTxB2a6shnRSOcftzUGdyp3IdXwjG07okuSY8joRLNdejYIA61jc3N7z4_jAdve-Wq9zjYuSZ5biVKCJJ26_cU3fsTKd3Rx59aNfjPksd0Q7WrECxDDrukUf_dvYZc7h7GgQpFYi0TESz25jIANJyfLJu4_RTqlxUKVwkMrkBZEztUINu2klc8SHPFGdV)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEipJq5I9W1a0jOOwMhimTn1NSK8eCCpoEoxwbBRzhZvm3bXrNVqFGuRRjPYWtpPoGp9QxsrJttyeritT7vXNLgJlbYGeTlE84qmOXK190agjboHwN03LpysrWkbgbZ-Ev0dg4jGFd95LS6MQvHWiIavW8OYViLiZA3mDL6fmZ8iJPMpnb4FRNQyNx1mSKSO)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjGVKViNKyJHfI1dQvvKu6XiR6kAzR0Rpd7LnEQ9xmzlw2LsXRpq8UG4RQrFz781UAtOYpHolMmTTyZMqQ1tGwcZPBV_yL_LRKEq-D5X-MJqNamF6SM-wf7gp5QRAlemxMk_KlCJSZDEZgyIoAkBtWk5eRnJVRPhoOTMswMl3V73dmOOqLFLtP5MbCKR1l2)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj78kie7ZbBfm26Iy3QaLwuJFdbJ1MLbk3IYWUhBplHx07b42N82t6Ix0jjyv_AzFXvWHRAgRDjOht9Drz59leQEEjDMh0kS0y8udsWPJtWbS90HUgofGWQola2yS706tBVjk4wkTEl5avpDWdDUzCzo6fGD41lQc_I9PJpRwlnZGilhdSof4Ix9pVYSOGx)
 
 **Random walks and Markov chains**
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjDTlkqYsPo2wD533lYogYyPsEu9aaSIfosRqhSYlBeFfBUzgEuXrk2uI-VEFA24Jll4FDaCVaTQYVjz54mYpTzHhN76-RM6YZgU6yPonzM7gJq9-7u0vk1e-QFIXCSb_nwEaqiB80BOl7gWfx0POO7mIN_zQrEnA_MJOA3n82S8CD_KUKcyM9xGbG3t0wd)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjbGPxSAZh7awwd8BTYYoJa1r8ZR7oLKCCiy4_o2213tv87jbDOLJS9Z5YKhzAAGBXt9MZyeHx2LPWnrVK-HCCQJsY7asTc2dl_Qn-RI6nOti8c2N7RBXbZDDnshkImplTksxM9d_uqOKcIU27JR9CATVV-bY0Qrkr-6d_veFx1G_ak4eCBea_ssKvZ45ia)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEh9075ANFjjpsvlyUfaco31I0eDwLRFEbRXcHvplNu0DR2QGhkRLwreoEIXpsteWCWeKD5cUZ_wry0UF2TahCPAvyBhzrvxVeAlzMIEcCXqZ_6FtjOcdYDuGZgj88qDdfdDLA8ySvp9pk0NhvGpCo1y2-mpn9jqFVmgyjo2WorZfAFGqN7Jcala8Dvb7qGO)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEiHMdhY07VUVTLu2JLyDCdHF7p7jSegok19hbZ4vHblpetPejIak5bqSfOD88ku0uQ1ylBPvPKOapVmNr7yE4qYEM1VR8r-thDViLlUZB-ljmh1ncam6QQwOGYbLs2L-fywBLc6fKprgcL8D7RayBFtJ4LARcBYHtV9YZ8RsdFzoc-pjApXuliZ7gCYbFwO)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEi-pFd_0qcOXdzlMAOuZD3-vMlOJP6TR3jMw1aBc4x5uZ6WO9_yaRSC0Pex3C2CrjZF30A5OT_dxWRCIJwl1TUNeduyxZOIRvsgS5stvd8Y7t9slZwdxYoPMWcwwBoqL0pF_YxAKjU49OBgpikXYzOVMvDGM5smts_YpIpXN9Gj7qmQXn-3OIa05zz5tn5L)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjMBIKpeZTqMy4shxIIJopbZ63Z2p1C7mGMflwC6Qv-YYn73fZmFxm5fL2dKrYWgy6TyRANCOtRQkPUK2wdzaOKrUv9MLdPrcBWwsiuXFxot2ocn6UFQxSGBGasIp943eJfuvE_zVmxsCAyXM9WpJnJjRia09x2i22Ff4vbw0MKAFCbGjSmmzmSZk2SOO9T)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgL0MGJqGGvP6LaenFQAv6sf636DgARhpLbwZ9EV2_mo0Ni957dTT9OPKN9gBlxn28-dKSCzqslJV-v3EIgpq1q1HRdoOZmnt6LilG4m890XPghe0j_A1eoMHXordPpOFFCoN60hF1Xrqku--Yq9WrpracrrgwAvq8oHZQJdmi4Rh29uZ-EeQdtzHYTRl2H)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjDTlkqYsPo2wD533lYogYyPsEu9aaSIfosRqhSYlBeFfBUzgEuXrk2uI-VEFA24Jll4FDaCVaTQYVjz54mYpTzHhN76-RM6YZgU6yPonzM7gJq9-7u0vk1e-QFIXCSb_nwEaqiB80BOl7gWfx0POO7mIN_zQrEnA_MJOA3n82S8CD_KUKcyM9xGbG3t0wd)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjbGPxSAZh7awwd8BTYYoJa1r8ZR7oLKCCiy4_o2213tv87jbDOLJS9Z5YKhzAAGBXt9MZyeHx2LPWnrVK-HCCQJsY7asTc2dl_Qn-RI6nOti8c2N7RBXbZDDnshkImplTksxM9d_uqOKcIU27JR9CATVV-bY0Qrkr-6d_veFx1G_ak4eCBea_ssKvZ45ia)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh9075ANFjjpsvlyUfaco31I0eDwLRFEbRXcHvplNu0DR2QGhkRLwreoEIXpsteWCWeKD5cUZ_wry0UF2TahCPAvyBhzrvxVeAlzMIEcCXqZ_6FtjOcdYDuGZgj88qDdfdDLA8ySvp9pk0NhvGpCo1y2-mpn9jqFVmgyjo2WorZfAFGqN7Jcala8Dvb7qGO)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiHMdhY07VUVTLu2JLyDCdHF7p7jSegok19hbZ4vHblpetPejIak5bqSfOD88ku0uQ1ylBPvPKOapVmNr7yE4qYEM1VR8r-thDViLlUZB-ljmh1ncam6QQwOGYbLs2L-fywBLc6fKprgcL8D7RayBFtJ4LARcBYHtV9YZ8RsdFzoc-pjApXuliZ7gCYbFwO)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi-pFd_0qcOXdzlMAOuZD3-vMlOJP6TR3jMw1aBc4x5uZ6WO9_yaRSC0Pex3C2CrjZF30A5OT_dxWRCIJwl1TUNeduyxZOIRvsgS5stvd8Y7t9slZwdxYoPMWcwwBoqL0pF_YxAKjU49OBgpikXYzOVMvDGM5smts_YpIpXN9Gj7qmQXn-3OIa05zz5tn5L)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjMBIKpeZTqMy4shxIIJopbZ63Z2p1C7mGMflwC6Qv-YYn73fZmFxm5fL2dKrYWgy6TyRANCOtRQkPUK2wdzaOKrUv9MLdPrcBWwsiuXFxot2ocn6UFQxSGBGasIp943eJfuvE_zVmxsCAyXM9WpJnJjRia09x2i22Ff4vbw0MKAFCbGjSmmzmSZk2SOO9T)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgL0MGJqGGvP6LaenFQAv6sf636DgARhpLbwZ9EV2_mo0Ni957dTT9OPKN9gBlxn28-dKSCzqslJV-v3EIgpq1q1HRdoOZmnt6LilG4m890XPghe0j_A1eoMHXordPpOFFCoN60hF1Xrqku--Yq9WrpracrrgwAvq8oHZQJdmi4Rh29uZ-EeQdtzHYTRl2H)
 
 **VC-dimension**
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEiXer6TuY5-IrE7xqR-Os-oGw90MaySPCoqJhIAv_VppThreKV03yU3zpz7ytoGPxkVkg6AU15bAIQORB0mPJiDx6OPjwNRhLBLfq4C_EABYIjXXFLkSK4CrqrWjHuk5pdm0nK2UGG7F62H93i5jK6n70GMbvU4fkem6GCTd_vOeTySyByUmu5D2T13qseS)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgi8DTTjxV7Onj5ZcxcTI1T2IBG8ybMzv88j1uqgvO0sqRExnCV9KmEbRatl2832Lf9i1zhmFNLAxxcvJNIqnhP2BHAe_pkl8UzRf-eQUQCyH3xe9DqmU9DmayarW0zkq9DigPOrTz3sUoUI5z1A39MHyQV9pII_ZAQen46qpRjnmJuq_ua7MP2LoYK0oHR)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEgEjzBsaZgyvQwZ6eDQtQwIlhnIO_wVFQ1RJqb1UP4purY-6H5KrJIQD59ra8VEa9n8T0CnUInJYGFfJdx-pbwg6XtmcKskEYYNZgXd82E9Me39jnRz3cCc44_qC7j_geFROzgiVOWz0-hy9mGe6qmI2SNYMkxmA747aSHvUy4TssdOHRX1GxMPOkble06u)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjWXntivu6JSGo-_fZ7gGGNcvpuOr1lF7AYNr11xMlvcPxSS_drst0juLzKvDK5MDQkC644cZdDbPBfAKZDLHMSdTr8PYharMjYufUM3hkDmxmD9tRQRLIWX6GH9TUogVpKiuwx9ea4-BSIaRjfwrq0dHZ_zFBt4NAAyJiQ0lvxoFJi25nIrlsfRoHt6lHt)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhphf6TjGhNEkUMsG4ZBU0UEFvooY-bmqv0q5d_cmEwdncYT1vb427OccAtjADrIfnrDj9HfndbFSpxOrAzv5XmkQGCa4U8iIaJPsDKAV40Jb_ILZfRegPEQ4mv4Fr52eZ4AGNSb-ihs5_t1YRpNcJ2MZVAJM7yIiNwh8MYA3-LVS2SvvLm1M6LE6HorEAc)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhB6NeYGm9mClyXjSDF_fdwpKQmefeItzEnggpqSL2dzd1xdV5nK3i-3m8-Bni9PmUEzoxJp1KlPuUBPs2CVLELcyZBMzY2GFLpPKsxBADHNIek-_DGX4FatlhUDTokN_ydwb0Tmm-d1zV4Lwod9aES32teiriZRjzaVw1ZsuZ4PhxFU2cAv0OSPC6SodLQ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiXer6TuY5-IrE7xqR-Os-oGw90MaySPCoqJhIAv_VppThreKV03yU3zpz7ytoGPxkVkg6AU15bAIQORB0mPJiDx6OPjwNRhLBLfq4C_EABYIjXXFLkSK4CrqrWjHuk5pdm0nK2UGG7F62H93i5jK6n70GMbvU4fkem6GCTd_vOeTySyByUmu5D2T13qseS)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgi8DTTjxV7Onj5ZcxcTI1T2IBG8ybMzv88j1uqgvO0sqRExnCV9KmEbRatl2832Lf9i1zhmFNLAxxcvJNIqnhP2BHAe_pkl8UzRf-eQUQCyH3xe9DqmU9DmayarW0zkq9DigPOrTz3sUoUI5z1A39MHyQV9pII_ZAQen46qpRjnmJuq_ua7MP2LoYK0oHR)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgEjzBsaZgyvQwZ6eDQtQwIlhnIO_wVFQ1RJqb1UP4purY-6H5KrJIQD59ra8VEa9n8T0CnUInJYGFfJdx-pbwg6XtmcKskEYYNZgXd82E9Me39jnRz3cCc44_qC7j_geFROzgiVOWz0-hy9mGe6qmI2SNYMkxmA747aSHvUy4TssdOHRX1GxMPOkble06u)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjWXntivu6JSGo-_fZ7gGGNcvpuOr1lF7AYNr11xMlvcPxSS_drst0juLzKvDK5MDQkC644cZdDbPBfAKZDLHMSdTr8PYharMjYufUM3hkDmxmD9tRQRLIWX6GH9TUogVpKiuwx9ea4-BSIaRjfwrq0dHZ_zFBt4NAAyJiQ0lvxoFJi25nIrlsfRoHt6lHt)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhphf6TjGhNEkUMsG4ZBU0UEFvooY-bmqv0q5d_cmEwdncYT1vb427OccAtjADrIfnrDj9HfndbFSpxOrAzv5XmkQGCa4U8iIaJPsDKAV40Jb_ILZfRegPEQ4mv4Fr52eZ4AGNSb-ihs5_t1YRpNcJ2MZVAJM7yIiNwh8MYA3-LVS2SvvLm1M6LE6HorEAc)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhB6NeYGm9mClyXjSDF_fdwpKQmefeItzEnggpqSL2dzd1xdV5nK3i-3m8-Bni9PmUEzoxJp1KlPuUBPs2CVLELcyZBMzY2GFLpPKsxBADHNIek-_DGX4FatlhUDTokN_ydwb0Tmm-d1zV4Lwod9aES32teiriZRjzaVw1ZsuZ4PhxFU2cAv0OSPC6SodLQ)
 
 That is all for today!
 

From 3993a3dc2f20665ef791b6ea269c3e200b8577e3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:21:29 +0900
Subject: [PATCH 150/638] Update 2024-2-20-day-50.md

---
 _posts/2024-2-20-day-50.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/_posts/2024-2-20-day-50.md b/_posts/2024-2-20-day-50.md
index c01ca345e..9fd0575c2 100644
--- a/_posts/2024-2-20-day-50.md
+++ b/_posts/2024-2-20-day-50.md
@@ -43,20 +43,20 @@ When I started the course I kept this in mine. "I will focus on understanding ho
 
 **GMM**
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEiX9fi-z-bP1N7MeP-GeX8-so_0FvGJgMWIFf3kViwydncwGrFSQTwGFoDfTNGqzHx_RxCH1STr9I0sSkqpLBapuG6vJCeRQMOog0VUEV3knFT-YrGnRFFB8EK-831ieHwChROAXllJxS8hA8XfNhN_wOAijXFbxmYHPQYYMxwtG2tbJKSD8sc7oRwjCTHY)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEib7yNvkaU5v1uYwC3vqLit3asd1gYz3bMJxqp2Iyt0HtYTZx4zE8p2tgapig6-rbeeXqIlL3hYkpvr2vO3hEANZpu4U5ZHi-mFEPoCVomfxTl0Nyz3FIEqkbvW4Z9wA4uH1vc47qtDHItKWAEmAJ0TvJu9FtadaIsqO_Szv54gMcbCha9-pGn8HvTSWqMg)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjLpUcGPotxwmLyRHkTGsN3deE97iG-MkdCVuFa4VL9A_He0vzavYEg4gab96r29eUuiCwV5YLpF22GtsbLbCYDscBLLg6MezqmQrzM_Jrs5T8e9So4c2_OlZVpcLu_98O89yleHocYZ0MR5MV481z6xK0ujqvzGPoiMvMuqo5UnPQvUSFTYz-sRXDaD3RD)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhsHxShYyw1dPoGuS-m3Jm6o4jMfKmvwRZhgNtvnYWLPQxH2QW_BVR9c87WaWHzZXmXO7L_lcOhWDxK0lYx-ROagbkURNESvdatQxfPss8wERYLGWQzQb-9uATHBDDufxp-9Mmaabhr7XbeJhBP0klK7-Fu6Lxb0wdGJHjvYnL-Hr-Lc_TpWzFspb4Mfn2B)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEitne75PQ-sf6AQmBJJOOuYoeUhDpinwADtqPKwjG9HPGdI249iqh09zRSzrW4cEhmwvD6QwioHzC-K1ROVwriHNxwA9GAAXPYMwl_cvol49jlEBMPugF1hdzIr01Ri8Jp-I2AV97Zg4vTxYA_Gjq-avpAz4tD6EbOJ5upc5Iems-LV1zKlLk_2WiBJN2EX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiX9fi-z-bP1N7MeP-GeX8-so_0FvGJgMWIFf3kViwydncwGrFSQTwGFoDfTNGqzHx_RxCH1STr9I0sSkqpLBapuG6vJCeRQMOog0VUEV3knFT-YrGnRFFB8EK-831ieHwChROAXllJxS8hA8XfNhN_wOAijXFbxmYHPQYYMxwtG2tbJKSD8sc7oRwjCTHY)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEib7yNvkaU5v1uYwC3vqLit3asd1gYz3bMJxqp2Iyt0HtYTZx4zE8p2tgapig6-rbeeXqIlL3hYkpvr2vO3hEANZpu4U5ZHi-mFEPoCVomfxTl0Nyz3FIEqkbvW4Z9wA4uH1vc47qtDHItKWAEmAJ0TvJu9FtadaIsqO_Szv54gMcbCha9-pGn8HvTSWqMg)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjLpUcGPotxwmLyRHkTGsN3deE97iG-MkdCVuFa4VL9A_He0vzavYEg4gab96r29eUuiCwV5YLpF22GtsbLbCYDscBLLg6MezqmQrzM_Jrs5T8e9So4c2_OlZVpcLu_98O89yleHocYZ0MR5MV481z6xK0ujqvzGPoiMvMuqo5UnPQvUSFTYz-sRXDaD3RD)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhsHxShYyw1dPoGuS-m3Jm6o4jMfKmvwRZhgNtvnYWLPQxH2QW_BVR9c87WaWHzZXmXO7L_lcOhWDxK0lYx-ROagbkURNESvdatQxfPss8wERYLGWQzQb-9uATHBDDufxp-9Mmaabhr7XbeJhBP0klK7-Fu6Lxb0wdGJHjvYnL-Hr-Lc_TpWzFspb4Mfn2B)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEitne75PQ-sf6AQmBJJOOuYoeUhDpinwADtqPKwjG9HPGdI249iqh09zRSzrW4cEhmwvD6QwioHzC-K1ROVwriHNxwA9GAAXPYMwl_cvol49jlEBMPugF1hdzIr01Ri8Jp-I2AV97Zg4vTxYA_Gjq-avpAz4tD6EbOJ5upc5Iems-LV1zKlLk_2WiBJN2EX)
 
 **SVM**
 
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjch35Vbyuy7c5PfMUBS6KhSSdOZTByTE3EooCmETQb557SMxq-6GB3HT2Ys_qELrU_lxSZDA4vemSDvFyiSODZpH7N3tOcmgJsw2H2STd-4QzMBluMpRCAoIV1jHMw6_9KlbqPbZEwRk4Fz1-6avzD-XJVO1hEoT8usoc3b9R93Zq79vouPiGBlLiSNPib)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjg2OQUP68v95JMK1kZ7AFA3viqH02s55DhXULFxwlve2zKGN4onomTSfzbsVFy4rAgGUlmLzZVAh7qjKkz6M5jh2ibq6fRJfvS0Pj69ImargexsmDEyTQO2aucRkD5OUKdSNcZxfAMYmNvh6R3Iw8SR-cNKsS7xQj9_MmkpXZtD9Cv3Z5RtZTlW0Kjmsa5)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEjhoonB8APOA690z3fvWzuYyST0XWcl6mAIMwwH2yiOz9SVuUcxfP0heSUuhKgz4gmQUglNNEg6T_1t4u7VpsII8z_w3r9kh44AxHebj8e8ByfkEqgl8ek-eMiNE1dWEJzEO_YM_gq14lutRCl61huRtkeqiz38aln6C9Lla_t8qVVM6BPqyWPTmi46x7Fh)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhH_5d9swesQY-fcFit3dvL39bHUojj34oCTSE8NJczWnRfDreOwfAH8ygKgRDwqOzlqH8PNAOcmywIQVdC07KLlzxfgnq4CFzKwRx5h7Pck76zzudEUiyDokNxq78x-D9trlOIkaCVtemlSicqfd1S686wl1tyoBz88xj8XLUb0-YtPMTAMueD1Tcx90fx)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEiHektKumNQANkGa8-z8eHxvxnEP-kPRSzunjaJq5CbwD5qyBWrIo150fitVRVINtkIZF5h3-cMav_-a5DfvElAhzFMusVzm6pVdaS7_8peqUsDs-Agdwos83BevOEM-ZMJViXqZY0XNbTEDu0af2-WI0ZmyeaIYWR4YqJ7Lgx58wrLq9LpNBHzZZK0NkYA)
-[](https://blogger.googleusercontent.com/img/a/AVvXsEhhKVvcnqLNAmb3o7QmR9EuwMd-XIjwGfVWjYY8FDhztiuZp9CzCNdDYzDasCpTbmokICgdslfmfVgpRQpY0YfPLQaJUCCkwS2BCTThJqGCfVhVKK_GufE6TXhG8_E8X4cl-7Uj8qCZpzlCnll6Rk5UdKR3DxCza-47CLKQIojRTns1TOCaKKX1VMkkzFb9)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjch35Vbyuy7c5PfMUBS6KhSSdOZTByTE3EooCmETQb557SMxq-6GB3HT2Ys_qELrU_lxSZDA4vemSDvFyiSODZpH7N3tOcmgJsw2H2STd-4QzMBluMpRCAoIV1jHMw6_9KlbqPbZEwRk4Fz1-6avzD-XJVO1hEoT8usoc3b9R93Zq79vouPiGBlLiSNPib)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjg2OQUP68v95JMK1kZ7AFA3viqH02s55DhXULFxwlve2zKGN4onomTSfzbsVFy4rAgGUlmLzZVAh7qjKkz6M5jh2ibq6fRJfvS0Pj69ImargexsmDEyTQO2aucRkD5OUKdSNcZxfAMYmNvh6R3Iw8SR-cNKsS7xQj9_MmkpXZtD9Cv3Z5RtZTlW0Kjmsa5)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjhoonB8APOA690z3fvWzuYyST0XWcl6mAIMwwH2yiOz9SVuUcxfP0heSUuhKgz4gmQUglNNEg6T_1t4u7VpsII8z_w3r9kh44AxHebj8e8ByfkEqgl8ek-eMiNE1dWEJzEO_YM_gq14lutRCl61huRtkeqiz38aln6C9Lla_t8qVVM6BPqyWPTmi46x7Fh)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhH_5d9swesQY-fcFit3dvL39bHUojj34oCTSE8NJczWnRfDreOwfAH8ygKgRDwqOzlqH8PNAOcmywIQVdC07KLlzxfgnq4CFzKwRx5h7Pck76zzudEUiyDokNxq78x-D9trlOIkaCVtemlSicqfd1S686wl1tyoBz88xj8XLUb0-YtPMTAMueD1Tcx90fx)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiHektKumNQANkGa8-z8eHxvxnEP-kPRSzunjaJq5CbwD5qyBWrIo150fitVRVINtkIZF5h3-cMav_-a5DfvElAhzFMusVzm6pVdaS7_8peqUsDs-Agdwos83BevOEM-ZMJViXqZY0XNbTEDu0af2-WI0ZmyeaIYWR4YqJ7Lgx58wrLq9LpNBHzZZK0NkYA)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhhKVvcnqLNAmb3o7QmR9EuwMd-XIjwGfVWjYY8FDhztiuZp9CzCNdDYzDasCpTbmokICgdslfmfVgpRQpY0YfPLQaJUCCkwS2BCTThJqGCfVhVKK_GufE6TXhG8_E8X4cl-7Uj8qCZpzlCnll6Rk5UdKR3DxCza-47CLKQIojRTns1TOCaKKX1VMkkzFb9)
 
 That is all for today, day 50! Not the end!
 

From 9debef805d34180ea00ed74bd411b2136604114f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:26:13 +0900
Subject: [PATCH 151/638] Update 2024-2-23-day-53.md

---
 _posts/2024-2-23-day-53.md | 72 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-2-23-day-53.md b/_posts/2024-2-23-day-53.md
index a5b48c2cf..580ec8827 100644
--- a/_posts/2024-2-23-day-53.md
+++ b/_posts/2024-2-23-day-53.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 53) Getting closer to becoming a 'backprop ninja' (thanks to Stanford Uni's cs231n assignments)
-categories: [applying-knowledge,theory,deep-learning]
+categories: [applying-knowledge,theory,math]
 ---
 
 ## Hello :) Today is Day 53!
@@ -12,7 +12,7 @@ A quick summary of today:
   * [Optimization: Stochastic Gradient Descent](https://cs231n.github.io/optimization-1/)
   * [Backpropagation, Intuitions](https://cs231n.github.io/optimization-2/)
 
-First I will quickly share my notes from the 1st three, and then for the big one (backprop) - will do it last.
+**First** I will quickly share my notes from the 1st three, and then for the big one (backprop) - will do it last.
 
 1st lecture - Image classification using KNN
 Actually my impression is not so much from the theory in the lecture, but from the assignment exercises which the course has. 
@@ -31,27 +31,95 @@ And then, the most efficient one which uses matrix multiplication magic
 
 ![a](https://blogger.googleusercontent.com/img/a/AVvXsEirDSwFbTeiRJB-l8Fx9sjngVLOyPcEtZZljrD5jZh1nmbFnOK2TYkSZtsm6Y1W4Nm3RExZylaT8Kg9QcApFC8UCwrEQR7cRlS27SPTHFRBMekcQT7WMiIgLNCJhh_1T01lmlFhMw1CxsyDntw8vwmaVaSKqBlc-h9BKqZbbl7e5p9RgBim0JZa_qtzOZm1)
 
+I have never implemented knn by myself so this was a nice challenge.
+Also all 3 were compared and the results are:
+- Two loop version took 297.302739 seconds
+- One loop version took 306.791257 seconds
+- No loop version took 0.973995 seconds
 
+Also after using the distance formula, it code showed me a distance map which was quite interesting in that it shows some rows where the distance is brighter and some is darker (whiter and blacker)
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiNjEgkp8iUmdlV7sRhvz7Zy0dYPfyy53rVkpuJzP1d3OgTVH4gBifbzBFEe6svCK9FHWjpBYDtEoBm7sN0Q6inKjFM0FrW53NXBfD65wVBVpoH-lrOxXl_AlAlF5Z9MHuYTaZbfqPo89RL0Qh_sR3awU1zEJWgUOKVVLc3E_9hyliXTDwONYL124G2kDDN)
 
+Turns out the explanation is that bright rows are caused by the test img being different vs train - if the test is very different from train, we get a bright row. Whereas bright columns are caused by the train image being very dif from the test image, so we end up with bright columns. 
 
 
+**2nd lecture: Linear classification: SVM and softmax**
 
+Here I learned about Multiclass SVM loss versus the softmax for the image classification. And I understood the formulas for both. I knew the softmax (cross-entropy) loss, but the multiclass SVM loss was new to me. 
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgQwhbuXvutM1KUJ_TZ4dxwa5uRtXm2F42jDEGM6DB56I71HMJ0F0k6yyjOl_KBU6rMjedZWwVeixknNZjgHhtDwgNpHLeMOi2WkWGOOlMYvqa0qL-NVuUt9FwzNI8qVc3OOVCyGGejp7TtoZlfgzDSMTK0k4XiaZN2lraq94t9D32vI9kajJlqiUvpl-Bb)
 
+It is the sum of max between 0 and the difference where the labels are different and some delta which controls how strict the classification should be, and then a regularization loss. But when I saw the implementation (below) I was a bit lost.
+Ok, side note. I just wrote the above sentence and my head told me I cannot leave it like this. So I went on google to search for the implementation in code (in a vectorized way) so that I can understand it. And I actually found this, which has nice comments and helps me understand what is going on haha. 
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgGYFLz7Dm8UVBvIjNxB3ikEkJ97k5EH0wEWnH7r5thciBZhqgB8WnxB8yGCSJOYXjk8KreT3RVztKpmiPr35uVNDkQzO3HclJF0cJb8GL6qc8F-2dtfI5JhR4voOzyMVDK3GkH6fwXTcpLR1SFx7r6hD-6UbTSo28tLCnn19TaeM6v2MkvOB6XF8M3b1eZ)
 
+The only bit that is tricky right now is the y_hat_true line and why do we need to create it in that way. But I can explore that tomorrow.
 
+**3rd lecture: Optimization: Stochastic Gradient Descent **
 
+This was just how gradient descent work and it did not have a particular exercise to it.
 
 
+**4th lecture: Backpropagation**
 
+We finally reached the most exciting part of my day. As I was reading through the lecture, I was thinking "oh, will I be implementing every step of forward and backprop in a neural network similar to Andrej Karpathy's 'become a backprop ninja video' ?' well it turns out the answer is yes. And it was really exciting to do it a slow steady manner. 
 
+The lecture went over derivative rules that we all know and love
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjNcMpYxqKfW0dr18mmO-uOxfgwGnyY_BnA5Rr_uheaNer5maHc-_SbX2g7AxZCwJ9f312OUPl-gD3NQJ0Ceo2HieRHx5ugU3PXDx29GoS1Nyl5e1PwowoXkREMZXPAc_PDNLMdEOaeJ62EaJUIDpuH5QME0IXSS-lc1PEX8AWHQD22aGpUOk_SkXmNBSsl)
 
+and for sigmoid's derivative, it is (1-sigmoid(x))*sigmoid(x)
 
+Also, I did not know but the derivate when going through a max gate is
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgQbJrr8-HoNYf5gTRKHKVtdr8lQavEX8oyv4XiGS9AxRVPNkpM-arAmgb0NAQb0R40RpG_D1GufxyOjzTMqmRuCe8QQUdN3kiZxJT9BuAB8WW0BfdWS7IBJT9gARu00wEe3DoBsA_JEFbQt5w2LE2iw4C-jl-9fAUPUsheOPZLLqQIXRpATh4h8HSr5OqX)
 
+The lecture showed short graphs like:
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi6cfFHC52J5vWeW7Bx1a3DauIA-1fARRh9gHb_5WiNlfsCsOTLQsaOZy-LoBNSpT0cOEdFlPI31sTOKGt6Avy1tCYvFBvistRJdy7ngHY5tW9n7CydqYIav2_sqBgf3FpIdNuX4-i5VJyklMmfDKLlVuvjw5jKV86D15xQNckatuYTG0NhLak0t1Xuqfyq)
 
+and this motivated me to do something similar by hand. And so I did.
 
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjjdyNXUhfHayZ0JiSuwEYZF0Jm4hFrpOR1k6hTAJIENRRIzXiUDK3Axs1yu6rdX3zzXWZscAMvGUW1BA4etQ0DOoFiabOHTnXe7a282F8mTPHDNn0L1vmmQYUhqiGJVEbly5aqeKjbpbVngIoDHOx3A6c0lDE4Nj5vIvjKfD34ijA1GzqIETB_qhNeIEsV)
+
+I hope the image is clear when zoomed in, but I just wanted to go back on a simple neural network + the sigmoid. 
+
+Also found this exercise:
+
+For the below 2 layer neural netowork, write out the derivatives
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjMF3f4HSHdHBwnw3_d5kwlGbEoAM8DZN2dWjQRyzrIM-06Syaxq7xKbIfgBdJJK6tBZD-CaH3tTGoGBYy1nVnAcf8eDQWhWO-90mp9B0YEKLMlj5AtNL30xTIdF7-75aciUGlFJSq2_vcZp1iT8ZdbdfeDbWxP3oedWaW-oYryDMrqL5x0OhszMxRuCSdX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiAWC2dDWLybexzQ0cLTLKwYnYD5mfX8bYidcP6kyQERL4zCYlnO96qwM6PTT5pKL5wXQVQBmBRS6VJiAvmUDxUGLj3CW7FD79fPY-MHW3x2E13sH1UfcVk6MoYOUNFEAVdfDn9-fpI8Y7Z6kPuc48L3H6BnbOaB5zR0qSXue5kq5oO3pwde3HPHF3_3f6R)
+
+Not gonna lie - it felt good knowing that I know these things. 
+
+At the end of the lecture there was this paper about [Vector, Matrix, and Tensor Derivatives](http://cs231n.stanford.edu/vecDerivs.pdf) by Erik Learned-Miller. It explains in a very simple way, what is happening when we multiply vectors and matrices and how the size should align when we want to do matrix multiplication or other operations.
+
+And with this knowledge from the short explanation paper and my confidence in the bag, I went onto the exercise assignment of the lecture which requires me to create forward and backward of a 2 layer neural network in code.
+
+First, the forward function:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEje-QXdnLnU5NEZ2P5uR3rWCl6SvfZW2mv4bQSzGTT7D0-uQ1OavbuQBBp94VW5W_RXKafnYK_vH59lHaDgevFy9yqHiZAnLyqTakLLCC_ose2p3nceG6ddamEb7yNtC1bLKA6c95wQ8HiFTrLU9yjrgh0lxhXot0ODMUg5VasIiht_IhxTOIkn1sty73SU)
+
+A big fear and a reason of not knowing what is happening was because of not knowing the shapes when we do matrix multipication, and when I watch someone do it, I am not sure why they do it, why do they change the shape of some matrix? Well in the codes below, I did a lot of prints figuring out how and why. 
+
+Backward function
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgk3dC4phRTM92sCzjeJuJBGiWw2rcER3NAOFgUlkLnE-gUTxQN6ZjRdziiWn5ViKC3bgjFLOaliZ_6BmDLl7sIxIitmk3KgzQDRYiEVa991aV1IgvM529hgDiaimE36wqRs4T2yDqa0kRlZ-vf4Qm7uLyYpZEgARZACebeKEeBK3kJrng9P-Mfiub8zh98)
+
+Relu forward:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjpGX23MtTkUQ8l_U6Hr6bdShaZswrKVOIR2L_T1ye6Hf_ZcrC51ZHPp26jO2cNk7WKam-0UUThWh1UQH1I7DAGfjp6dkXZTIwiW3WCQgdChSmdxpz2L2APwdqipbLF-rgc_8yvZTbN7Wvucy6lw4wMQDfgyO5twSDBt7sMteO0lpDdMsjU9HMY50IOwBDe)
+
+Relu backward:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgQ95N_jdtPfecROzeIpxtZfl1A0_Wt3WTS7_nzhXgEATNJHOFqthilz24ZK2pvuZSFl-o9Loi6eEvEPZwOWZBviUg3WuNu9INhR1Gu8Df_HNE-fWoNyLrr4hvRPnlOzC9zowiVY6b2rBoTll0Jd-s4huZGBG6fd5KxP5-uIls6By6kTR_RpQJmiXI9Uxv5)
+
+In the exercises there are parts about batch norm, layer norm, dropout, conv, maxpool, and softmax - all forward and backward.
+So exciting! I feel I am getting closer to becoming a, as Andrej Karpathy calls it, backprop ninja.
+
+That is all for today!
+
+See you tomorrow :) 

From d1a6921d39759f5d4c780d7ce4b2373f7f833e9c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:34:06 +0900
Subject: [PATCH 152/638] Create 2024-2-24-day-54.md

---
 _posts/2024-2-24-day-54.md | 94 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 _posts/2024-2-24-day-54.md

diff --git a/_posts/2024-2-24-day-54.md b/_posts/2024-2-24-day-54.md
new file mode 100644
index 000000000..52fc1bde2
--- /dev/null
+++ b/_posts/2024-2-24-day-54.md
@@ -0,0 +1,94 @@
+---
+layout: post
+title: (Day 54) I became a backprop ninja! (woohoo)
+categories: [applying-knowledge,theory,math]
+---
+
+## Hello :) Today is Day 54!
+A quick summary of today:
+* Read through cs231n's neural network lectures [1](https://cs231n.github.io/neural-networks-1/)[2](https://cs231n.github.io/neural-networks-2/)[3](https://cs231n.github.io/neural-networks-3/)
+* Spent 8 hours going over Andrej Karpathy's [backprop ninja tutorial](https://youtu.be/q8SA3rM6ckI)
+
+
+After yesterday's confidence boost in doing backprop, I came in today with the intention of becoming of, as Andrej Karpathy calls it, a backprop ninja. 
+
+The cs231n NN lectures were the usual about how NN became to be, param initialization, activation functions, loss functions, regularization and optimizers. Nothing out of the ordinary, but still was good to reassure myself of that knowledge. But that was definitely not the main even of today.
+
+As I mentioned, I wanted to become (or at least attempt to become) a backprop ninja today. I have watched that particular tutorial twice, the 1st time I had no idea what is going on, the 2nd time I could follow, and now it was the 3rd time - and now, I wanted to participate and actually see the why and the how of the things we have to do, when we do manual backprop. My main goal of today was to become the doge on the left.
+
+![image](https://github.com/user-attachments/assets/6a0e89a8-7911-4f3a-b65a-f7e6b52ca1ab)
+
+
+So my journey began...
+Here is a [link](https://colab.research.google.com/drive/1s47-_nL18XH9VeYz1eFKYM4myc2BHdsP?usp=drive_link) to the google colab with the manual backprop with *all* my notes.
+Below, cmp is a helper function to check whether what we are calculating is correct.
+
+To begin with, the code gave us a 2 layer model
+
+![image](https://github.com/user-attachments/assets/8873ba07-deef-4a60-a2af-2201fbffbd16)
+
+And the forward pass which was broken into simple pieces for the purpose of manual backprop
+
+![image](https://github.com/user-attachments/assets/7bdd7c9f-f824-4618-b464-a7bfbf9781a4)
+
+And now, the backprop line-by-line backwards began.
+A note: I found that seeing the shapes of all vars involved before doing the backprop is extremely useful. Just gives me an idea of what the result should look like and also what the inputs to a said line look like. 
+
+![image](https://github.com/user-attachments/assets/b8a858ac-7a1a-49ed-984e-8c223cb8fe30)
+![image](https://github.com/user-attachments/assets/04af107b-6122-48a8-af30-fde0ade1d9f1)
+![image](https://github.com/user-attachments/assets/f5a29a52-a9aa-4166-9ce3-279bc29776eb)
+![image](https://github.com/user-attachments/assets/6c152a04-7c15-4e39-b22b-2f3f63a0625c)
+![image](https://github.com/user-attachments/assets/96737b83-9c34-436a-b847-281f287acc5a)
+![image](https://github.com/user-attachments/assets/5895590e-ecfa-4e4b-83ec-bb4bdd1bc73d)
+![image](https://github.com/user-attachments/assets/276b9c22-1510-4675-8d77-9ab3ae9a68d0)
+![image](https://github.com/user-attachments/assets/ba430699-7a24-4b13-86cb-29ad7c5c6604)
+![image](https://github.com/user-attachments/assets/095d79b8-291c-469e-95d1-3d8f80b1518c)
+![image](https://github.com/user-attachments/assets/94d052ac-8327-47fb-b9d2-02ec30b5148d)
+![image](https://github.com/user-attachments/assets/4fbd36bf-0e32-41ef-b7b5-dfac038a8b24)
+![image](https://github.com/user-attachments/assets/855621ab-3133-4016-b820-18d722dff6dc)
+![image](https://github.com/user-attachments/assets/1de33d9e-183c-479a-ad5b-fa05ff684b6f)
+![image](https://github.com/user-attachments/assets/041f87d6-a3fe-4aaf-bfd0-039d9392b27d)
+![image](https://github.com/user-attachments/assets/02128be4-ee6a-4a1a-a889-6bf1deb5815a)
+
+I was surpised :D that I could do most of these bymyself, just by exploring the shapes of items, and doing derivatives. 
+
+After the manual line-by-line, now it was time for some corner cuts and simplifications.
+
+in the forward pass, when we want to obtain the loss firsly we had to go through 8 lines of code, but we can do the same just by taking the logits and the true Ys and give them to pytorch's cross_entropy loss function. 
+
+![image](https://github.com/user-attachments/assets/4d4dde13-5473-426f-9aa3-8b70135edf19)
+
+And now to backprop through that, Andrej showed some nice math about it, that I wrote myself too, and might even put it on my wall next to my neural network basics poster :D
+
+![image](https://github.com/user-attachments/assets/aa1aa2a7-83b5-4783-a94a-9d0a36497efb)
+
+But in code, it is as simple as:
+
+![image](https://github.com/user-attachments/assets/f970b816-9c4f-4e8a-a05a-4cfcf118ec45)
+
+Then, for the batch norm forward pass and backward pass, the long 7 line version, could be done with
+
+![image](https://github.com/user-attachments/assets/4134f225-3247-41b5-bcea-fc7bb3f17de0)
+
+And for the backprop there was a nice math explanation too on how to get the derivative
+
+![image](https://github.com/user-attachments/assets/83db22de-2307-4191-9686-79bf758ee78e)
+
+And in code, instead of the long backward pass, we get:
+
+![image](https://github.com/user-attachments/assets/54cf7b4e-0177-46c9-8888-dfa29e96a3d0)
+
+For the full version, again, I invite you to look at my google drive link, and also Andrej Karpathy's full video. 
+At the end of the day, yes I did spend my whole day over this. But I think it is definitely worth it. Though Andrej's help was needed at some spots, when I just could not figure out what to multiply by what (especially in some of the more complicated backward pass lines), I do believe I have a strong understanding of the backward propagation process. 
+
+At the end of his video Andrej says: "If you understood a good chunk of it and if you have a sense of that then you can count yourself as one of these buffed doges on the left." That said...
+
+_Mission accomplished!_
+
+![image](https://github.com/user-attachments/assets/211d552f-1841-4ac0-91a3-883a5c5302cc)
+
+I will still review this, and I will probably think a lot about the code and review it in my head, to increase my % of understanding, but what I got today is invaluable. Thank you, Andrej Karpathy! :)
+
+That is all for today!
+
+See you tomorrow :)

From c05da3d4b20fabbfebe40fe74faa12d85760a786 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:41:56 +0900
Subject: [PATCH 153/638] Create 2024-2-25-day-55.md

---
 _posts/2024-2-25-day-55.md | 149 +++++++++++++++++++++++++++++++++++++
 1 file changed, 149 insertions(+)
 create mode 100644 _posts/2024-2-25-day-55.md

diff --git a/_posts/2024-2-25-day-55.md b/_posts/2024-2-25-day-55.md
new file mode 100644
index 000000000..74e71c6af
--- /dev/null
+++ b/_posts/2024-2-25-day-55.md
@@ -0,0 +1,149 @@
+---
+layout: post
+title: (Day 55) Learning about tokenization in LLMs
+categories: [applying-knowledge,theory,math]
+---
+
+## Hello :) Today is Day 55!
+A quick summary of today:
+* continuing on the Andrej Karpathy streak from the last few days, today I watched his latest video about [tokenization in LLMs](https://youtu.be/zduSFxRajkE)
+* I used a pokemon name dataset on yesterday's MLP code (with the manual backprop) and posted it on [Kaggle for easier access](https://www.kaggle.com/code/divakaivan12/pokemon-name-mlp-from-scratch-backprop-tutorial/notebook)
+
+So... tokenization in language models. 
+
+Using [this](https://tiktokenizer.vercel.app/) nice website we can see an example below.
+
+![image](https://github.com/user-attachments/assets/2785a684-b2f0-4748-8279-ea3091efb3c1)
+![image](https://github.com/user-attachments/assets/7c28744f-7e43-4af6-bc41-f6e19c6f72f3)
+
+Using the above picture as refernce, tokenization is how we turn a piece of text to a representation through numbers. In the above pic, each coloured piece is a different token that is part of GPT2, and on the right is each number.
+There are different ways to turn a piece of text into numbers, and one of the common ones is to use utf8 encoding, which has 256 base bytes. 
+
+![image](https://github.com/user-attachments/assets/f5788872-24a3-43a2-a209-d98d59c4e86e)
+
+As seen in the example pic, using utf8 we can encode this piece of text. 
+In addition to utf8, there is utf16 and utf32, but apparently, those encodings 'overshoot' and are too much for our task. And as we can see in the pic below, there are a bunch of 0s being added, which is unnecessarily. 
+
+![image](https://github.com/user-attachments/assets/cc1b56ee-33c0-4e30-9c52-92452ae041ad)
+
+Ok, we can use utf8, but just using utf8 will result in very long sequences of bytes, and transformers only have a limited amount of attention (for computational reasons), so we want to squash these long sequences into something more appropriate. We dont want to use the raw bytes, so instead we can use [byte pair encoding](https://en.wikipedia.org/wiki/Byte_pair_encoding).
+
+(Apparently there is research that is looking into being able to do tokenization-free encodings, without needed to squash them, but a proper method is yet to be found)
+
+![image](https://github.com/user-attachments/assets/23f143b3-d1f2-420f-9d56-7c4c828275a7)
+
+So, how do we do byte pair encoding (bpe)?
+
+![image](https://github.com/user-attachments/assets/085ce2ec-e9f4-49ac-8b56-eac53986a45a)
+
+Take a piece of text, encode it, turn it into a mapping and a list so we get proper numbers (otherwise its bytes). 
+
+And then
+
+![image](https://github.com/user-attachments/assets/6a515641-beac-4fc7-9f8a-7fe9c50ab91e)
+
+pair up the resulting tokens, and count their occurance. For example in this case, the byte pairing of (101,32) occurred 20 times. 
+
+![image](https://github.com/user-attachments/assets/0bce44d6-7e61-441d-885d-a978fe103887)
+
+And in that text, 101 and 32 refer to the letter e and whitespace. So "e " has occurred the most (20 times). 
+Through this merge function which takes in a list of bytes, a pair and with what index to replace it, we do the bpe.
+
+![image](https://github.com/user-attachments/assets/fb9f886f-02ae-4178-93d6-0cbcc31a0980)
+
+The example is: 
+example: merge([5, 6, 6, 7, 9, 1], (6, 7), 99) ---> [5, 6, 99, 9, 1]
+replace the pair (6,7) with 99
+
+This merging through the merge function, we can do it iteratively and the amount of times we do it (find the most common pair, replace it, repeat) is a hyperparam and up to us. the more times we do it, the larger will be our vocab and the shorter will be the resulting sequence, and we want to find the 'sweet' spot.
+
+Using a longer piece of text, the next picture shows doing merging multiple times. And towards the end - 256 which is the 1st new token, is actually part of a common pair itself. 
+
+![image](https://github.com/user-attachments/assets/d451e652-cfc9-4e95-abb0-2dc7178f44b7)
+
+There is also the compression ratio, which measures how by how much did we reduce the bytes of our text. 
+
+![image](https://github.com/user-attachments/assets/67ace974-7862-4c07-aa88-db3550be6272)
+
+In the GPT2 paper, they mention that sometimes the same work is tokenized multiple times when it is next to some special symbols.
+
+![image](https://github.com/user-attachments/assets/ac32e853-8c4a-4d35-834a-af828af15dcc)
+
+So what they do is, they have a regex function as below.
+
+![image](https://github.com/user-attachments/assets/49c2e92e-a83a-49a3-bd08-762e32157184)
+
+To avoid unnecessary tokenization, as I wrote in the comments the text is ran through a regex and then each input is processed individually and then concat all together. 
+
+Special tokens
+
+Special tokens are used to create a special structure to token streams. Popular ones are:
+
+<|endoftext|> <|eos|> <|pad|> <|bos|> <|eol|> <|math|> <|doc|> <|reward|>
+
+For example the <|endoftext|> one, tells the model that the text that comes after has no relation to the one before it. But if we pass them to chatgpt they trigger weird behaviour. 
+
+Actuall this last token <|reward|> I found from [this presentation](https://youtu.be/bZQun8Y4L2A) from Andrej Karpathy on Microsoft open dev day.
+
+![image](https://github.com/user-attachments/assets/6b10f04e-70cb-4884-bdac-6f6a0729ca1f)
+
+I passed it to chatgpt and:
+
+![image](https://github.com/user-attachments/assets/4c4b3cb6-0dc2-432d-b2c7-8b7fd463dea8)
+
+
+It does not see it, and also does not react to any of the other special tokens. In other more complicated prompts it will ignore it and may cause other weird behaviour. (which is interesting, and kind of funny - can chatgpt be 'broken' by using some complicated prompt with these special tokens?)
+
+After the lecture finished, I went onto my personal code colab and started playing around with the tokenization on random text. And also did one of the tests in Andrej Karpathy's github, which is testing the wikipedia bpe example, and seeing if we get a correct output representation.
+
+![image](https://github.com/user-attachments/assets/8bcfa285-d8b4-4bf1-8dd4-41f48db4bdd1)
+
+And indeed we get the right answer.
+
+Andrej Karpathy also gave some questions to which tokenization was the answer, and I took note of his answers.
+
+* Why can't LLM spell words? **Tokenization**. 
+  * Some tokens are too long and if a token is input (i.e. .DefaultCellStyle) GPT becomes confused.
+* Why can't LLM do super simple string processing tasks like reversing a string? **Tokenization**. 
+  * Same as above
+* Why is LLM worse at non-English languages (e.g. Japanese)? **Tokenization**. 
+  * Lack of training data, and the tokenizer also is not trained well on languages other than English. (안녕하세요 is 3 tokens, Hello is 1 token)
+* Why is LLM bad at simple arithmetic? **Tokenization**. 
+  * Sometimes numbers are tokenized in one way, other times, in another way, very arbitrary. For example, if we give 3215, the tokenizer once will see 3 and 215, another case it will see 32 and 15.
+* Why did GPT-2 have more than necessary trouble coding in Python? Partly **Tokenization**. 
+  * Spacing in tokenization.
+* What is this weird warning I get about a "trailing whitespace"? **Tokenization**. 
+  * The model has not seen whitespace by itself a lot, so it can cause the warning if we put whitespace after the given text and ":", when we do text completion.
+* Why the LLM break if I ask it about "SolidGoldMagikarp"? **Tokenization**. 
+  * https://www.lesswrong.com/posts/aPeJE8bSo6rAFoLqg/solidgoldmagikarp-plus-prompt-generation
+* Why should I prefer to use YAML over JSON with LLMs? **Tokenization**. 
+  * YAML is efficient in tokens, compared to JSON
+* What is the real root of suffering? **Tokenization**. xD
+
+Now, for the pokemon name generator based on the manual backprop code. 
+I did not change much from the original code, the main change as I mentioned yesterday is me adding notes while doing the backprop, so that I can fully understand what is happening. Today I decided to try to adopt it to a new dataset (Andrej used human names, I decided to use pokemon names) and also play around with the model's hyperparams and see its reaction. 
+
+The full notebook (which is not different from the google colab link I shared yesterday) is [here](https://www.kaggle.com/code/divakaivan12/pokemon-name-mlp-from-scratch-backprop-tutorial/notebook). 
+
+But the most interesting part is the generated names:
+
+* libico	abesawing	cabezllish
+* kirzons	argola	drapiad
+* xutdi	swirlixs	toldan
+* simiroty	ledics	bruxishoinpaul
+* ferrorua	nidoran♂	hicmy
+* hippoono	typcanth	skorudi
+* koffing	charmer	roserire
+* tornu	venonat	girdreec
+* derno	herdeer	beynode
+* snobbull	eevile	wlabin
+* scazarill	slowperno	savimzar
+* cramsampardos	electakron	
+
+Not bad haha. This is using 5 context length, 10 as embeddings size and 128 hidden layers. (full info in the kaggle link)
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From c4bb13b04a3184eab0b9ef80c4a4bd9596cfad18 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 21:58:31 +0900
Subject: [PATCH 154/638] Create 2024-2-26-day-56.md

---
 _posts/2024-2-26-day-56.md | 46 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 _posts/2024-2-26-day-56.md

diff --git a/_posts/2024-2-26-day-56.md b/_posts/2024-2-26-day-56.md
new file mode 100644
index 000000000..f59703928
--- /dev/null
+++ b/_posts/2024-2-26-day-56.md
@@ -0,0 +1,46 @@
+---
+layout: post
+title: (Day 56) I found my next step in the ladder - cs224n NLP with DL by Stanford
+categories: [applying-knowledge,theory,math]
+---
+
+## Hello :) Today is Day 56!
+A quick summary of today:
+* Looked at [Natural Language Processing with Transformers](https://www.oreilly.com/library/view/natural-language-processing/9781098136789/)
+* Found Stanford Uni's [cs224n](https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1234/) NLP with DL course
+
+Today... was a bit weird, I suppose. My goal was to start something related to NLP and build on my knowledge from the previous days. 
+
+Before that, I will share about the book (from the 1st point).
+
+I saw this book about NLP with transformers, and looking at the content it looked awesome, covers a variety of topics in NLP - transformers, text classification, named entity recognition, text generation, summarization, question answering, dealing with different data. 
+
+So I decided to give it a go, and took notes on the 1st chapter, which was great. (you can see them at the end of this post) 
+
+I learned about how the transformer came to be, its 'story'. And my notes are below. 
+
+Then the book starts to use huggingface - a popular library that provides a vast amount of models, datasets, tokenizers and NLP tools. But I feel like that is kind of cheating. 
+
+I actually saw a meme online, where transformers' theory is so long and hard to understand and grasp, but in reality, you just do import transformers haha
+
+But my goal is not that. I wanna actually learn everything, all the small bits. And of course, from Professor Choi's KAIST course, and Andrej Karpathy and Andrew Ng, I believe I have a good grasp on the transformer architecture, but in order for me to have a good grasp on NLP as a field, I need to learn about the other issues, models, architectures, etc that were and are used in the field. Just like when I learned about computer vision - I went through AlexNet, VGGNet, GoogLeNet, ResNet, YOLO and MaskRCNN, I want to see what is out there in terms of NLP.
+
+With that in mind, I started to look around what to do. [NLP specialization by DeepLearning.ai](https://www.coursera.org/specializations/natural-language-processing) ? [KAIST's AI605 DL for NLP](https://seominjoon.github.io/kaist-ai605/) ? Stanford's [CS224N](https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1234/) NLP with DL ? Find and read another book ? Read research ? (I actually searched papers, pic below, about tokenization because Andrej Karpathy made it sound very interesting, so I considered this as an option)
+
+![image](https://github.com/user-attachments/assets/29ca7eed-c615-4c40-8ccc-03a859f78a39)
+
+And I started to 'audit' all the options, while also looking for what other people have suggested on r/deeplearning, r/machinelearning. I watched maybe 1.5hrs of each of the 3 courses, found a book by Michael Nielsen (Neural networks and Deep learning), and a ['mit-deep-learning-book'](https://github.com/janishar/mit-deep-learning-book-pdf). But I think what made the most sense to me was to commit to the Stanford course. Why? It was the most up to date (half is from 2021, half is from 2023), for each lecture they provide an extensive list of extra readings, there are slides and notes for each lecture, the course has a big community for questions, after the basics, they start using pytorch, and others.
+
+So, from tomorrow I will commit and be like a student that takes that class. It is not my first time taking a free Stanford course, so I am excited for the amazing teaching, and excited to dive into the world of NLP!
+
+My notes from chapter 1 of the NLP with Transformers book. 
+
+![image](https://github.com/user-attachments/assets/797297a3-729f-4cfe-8291-20c36d2b2447)
+![image](https://github.com/user-attachments/assets/a57e0f09-04fa-49ac-85a3-38b3f5507de6)
+![image](https://github.com/user-attachments/assets/e3ed3c79-86c6-419b-be62-c0e04c1a042b)
+![image](https://github.com/user-attachments/assets/acb6567f-017c-40cf-8d86-422c6ea4ca13)
+![image](https://github.com/user-attachments/assets/d4e6400a-b5e6-4d74-873f-58cbfaa76bf7)
+
+That is all for today!
+
+See you tomorrow :)

From fd8fbc954e2e316cd37f25ac679d31c32a58ba44 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:05:18 +0900
Subject: [PATCH 155/638] Create 2024-2-27-day-57.md

---
 _posts/2024-2-27-day-57.md | 145 +++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100644 _posts/2024-2-27-day-57.md

diff --git a/_posts/2024-2-27-day-57.md b/_posts/2024-2-27-day-57.md
new file mode 100644
index 000000000..9185d477f
--- /dev/null
+++ b/_posts/2024-2-27-day-57.md
@@ -0,0 +1,145 @@
+---
+layout: post
+title: (Day 57) Stanford CS224N - Lecture 1. Word vectors
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 57!
+A quick summary of today:
+* Got introduced to word vectors with [lecture 1](https://youtu.be/rmVRLeJRkl4?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) + small showcase
+* Read the paper and took notes about:
+* Efficient Estimation of Word Representations in Vector Space [(word2vec)](https://arxiv.org/abs/1301.3781)
+* [Distributed Representations of Words and Phrases and their Compositionality (negative sampling)](https://proceedings.neurips.cc/paper_files/paper/2013/file/9aa42b31882ec039965f3c4923ce901b-Paper.pdf)
+
+First I watched the lecture by Professor Manning, and then I read the papers so some of the material was overlapping, but there were still interesting new parts in each of the three. 
+
+**1) Lecture notes**
+
+My notes from the lecture were not that long so I will first share them.
+
+![image](https://github.com/user-attachments/assets/47d63eeb-3b02-4588-afe4-acfbd3e0f859)
+![image](https://github.com/user-attachments/assets/b9ca1561-13f7-414d-aa36-55fc41f25013)
+![image](https://github.com/user-attachments/assets/d9d9d158-a1e3-4b20-a33e-faba19d5fd2c)
+![image](https://github.com/user-attachments/assets/15169ac7-6184-4b35-9466-c095ea7cc2c4)
+
+After the lecture, there was a small colab document I could run through. 
+In the lecture, we mention about matrix multiplication and the larger the dot product between the 2 words, the more similar they are -> exp(u0T @ vc)
+
+![image](https://github.com/user-attachments/assets/ef9ba2a1-e9a3-4deb-ac5a-db65e370a861)
+
+So in the colab there was a model loaded.
+And I searched for the word embeddings of bread and baguette which in my view are similar words. 
+In the below pic, if a 2 values from both matrixes are both positive, or both negative, that means the output of the matrix mul for that specific value will be positive, which overall means, more similar.
+
+![image](https://github.com/user-attachments/assets/5ccd843e-61b3-4bc3-92c9-04b02543744f)
+![image](https://github.com/user-attachments/assets/06bb94d9-323a-4642-aa28-4e0e561dfbf7)
+
+Then I went to see what words are similar to bread according to the model:
+
+![image](https://github.com/user-attachments/assets/36c2da61-ded3-456e-b041-18f10d6b33ab)
+
+And then. took flour and bread and saw that compared to break and baguette, their True values are more (which by itself is not a complete indicator of similarity, but still is can indicate some similarity)
+
+![image](https://github.com/user-attachments/assets/511db6f2-1133-4e99-b8e6-43462e9acadb)
+
+Then in the word2vec paper, they proposed that with such word vectors we can do analogies by simple algebraic operations. I.e. start with a word like king, substract man from it, add woman and what word is the word that should be given? (queen)
+
+![image](https://github.com/user-attachments/assets/cb6621b7-6486-4d79-8820-9ffba61335a1)
+
+And I had some fun with this too:
+
+![image](https://github.com/user-attachments/assets/4bde6a45-467e-4102-b90e-5a6b188a25d6)
+
+But as we can see if the words we are searching for did not exist in the model, or were very rare, then the output does not make sense. In the first one, hanbok is korea's traditional piece of clothing, but I have no idea what zenana is haha.
+
+**2) word2vec paper summary**
+
+1. Intro
+
+Before word2vec, many nlp systems treated words as single units that do not have any notion of similarity. while this provided simplicity and robustness, such techniques had limits. for instance higher volume of data resulted in good performance, but if big data was not available, the models had limitations. Data was the limitation. but with the progress of ML, the newer and more complex models could be trained on larger data and outperform the simpler ones.
+
+They used a, back then, recent technique for word representation that not only puts similar words closer to each other, but also words can have multiple degrees of similarity.
+
+2. Popular neural network models
+
+  * Feedforward Neural Net Language Model (NNLM)
+
+It consists of input, projection, hidden and output layers. At the input layer, N previous words are encoded using 1-of-V coding, V is the size of the vocab. After, the input layer is projected to a projection layer P that has dimensionality N × D. Also, since only N inputs are active at any given time composition of the projection layer is relatively cheap. However, it becomes complex for computation between the projection and the hidden layer, as values in the projection layer are dense.
+
+  * Recurrent Neural Net Language Model (RNNLM)
+
+RNN based language models were proposed to overcome some of the limitations of the NNLM, such as the need to specify the context length, and because RNNs can theoretically represent more complex patterns than shallow NNs, and through a recurrent matrix, that forms like a short-term memory, info from the past is passed to the current input. 
+
+3. New log-linear models
+
+Previous models come with non-linear hidden layers, which complexify them. But the paper wants to look for a method that might not be able to represent data as precisely as NNs, but can be trained efficiently on much more data.
+Furthermore, they propose 2 models.
+  * Continuous Bag-of-Words CBOW Model
+
+Similar to the FF NNLM where the non-linear hidden layer is removed and the projection layer is shared with all words (not just the projection matrix), so all the words’ vectors are averaged. This model is called bag of words since the order of words in the history does not matter when it comes to the projection. In addition, they use words from the future. The paper mentions that they achieved the best results using a log-linear classifier with 4 future, 4 past words as the input, and the goal is to predict the middle word.
+
+![image](https://github.com/user-attachments/assets/19bf9e48-690c-4ffe-b3b7-e56d8b787834)
+
+  * Continuous Skip-gram Model
+
+this one is similar to CBOW, but instead of predicting the middle word based on its context, it uses a word as the input to a log-linear classifier with continuous projection layer, and then predict its context. This method was found to be efficient when trying to predicted larger context, but with that the computational complexity rose.
+
+![image](https://github.com/user-attachments/assets/ecc4902e-1ae0-41e2-b83c-4cf3d7fc499b)
+
+4. Results
+
+To show results, they proposed to use analogies, for example big is to bigger is as small is to ____? And they found that such questions can be answered with algebraic operations with the vector representation of words. 
+
+X = vector(‘bigger’) - vector(‘big’) + vector(‘small’), then from the vector space a word that is closes to X (measured by cosine distance) is given as the answer. They evaluated the accuracy of the model and assumed a correct answer only if the closest word to the vector computed is exactly the same as the correct word in the question. For instance, if the model is given the pair (king, queen) the correct word that the model should predict as closest might be "queen”. (more pairs in the 1st pic below)
+
+![image](https://github.com/user-attachments/assets/f597325e-5ffc-47fb-b62c-5774c46d56ad)
+
+The results also compared the RNNLM, NNLM, CBOW and Skip-gram models. And amongst them the CBOW (bag of words model) did the best when it comes to syntactic accuracy with 64%, but it lacked in semantic accuracy and came in 2nd with 24%. Whereas the skip-gram model scored more than double compared to the 2nd and 3rd place modes when it comes to semantic accuracy, and it was 2nd in syntactic. 
+
+![image](https://github.com/user-attachments/assets/f9a51eb4-3e69-4e91-8159-db4d71495e3c)
+
+3) negative sampling paper
+1. Intro
+
+Distributed representations of words in a vector space helps learning algorithms to achieve better performance in NLP tasks by grouping similar words. Compared to, back then, other models, the skip gram model we saw in the 1st paper, does not invlove matrix multiplication into its training process which makes that process very efficient and a singple machine implementation can train on more than 100 billion words in 1 day. In this paper, the researchers propose that subsampling of frequent words during training results in a significant speedup and improves accuracy of the representations of less frequent words. They also propose a simplified variant of Noise Constrastive Estimation (technique that involves distinguishing between true data and noise samples) for training the skip-gram model which comes with better results compared to the originally used hierarchical softmax.
+
+2. The skip-gram model
+
+Its goal is to find word representations that are useful for predicting the context around a word/document. To achieve this, the model aims to max the avg log probability:
+
+![image](https://github.com/user-attachments/assets/df6ae938-e7d7-4dda-88d4-e660bb1b49ac)
+
+and the basic skip-gram defines p(Wt+j | Wt) with the softmax
+
+![image](https://github.com/user-attachments/assets/8a3ffca8-d175-4baf-8b02-4a16a0d2ef02)
+
+But this formulation is impractical because the cost of computing delta log p(wo, wi) is proportional to W, which is often 10^5~10^7.
+To overcome that, a computationally efficient approximation of the full softmax is called the hierarchical softmax. And its main advantage is that instead of evaluating W output nodes in the NN to obtain the probability distribution, it only needs to evaluate log2(W) nodes. 
+
+3. Negative sampling
+It is an alternative to the above is NCE which says that a good model should be able to differentiate data from noise by means of logistic regression. It works by randomly sampling a set of "negative" examples (words not within the context of the target) alongside positive examples (words that are within the context of the target) during training. 
+4. Subsampling of frequent words
+
+In a very large body of text, words like “the”, “a”, “in” can occur millions of times, and they provide less value compared to the rarer words. For example the words France and Paris are helpful to the skip-gram model, but the occurance of France and the togehter right next to each other will be most likely higher. To deal with this imbalance of words, they used a sub-samping approach where each word Wi in the training data is discarded with a probability:
+
+![image](https://github.com/user-attachments/assets/2f3eb071-44ab-4dd2-b051-7137fa097294)
+
+where f(Wi) is the frequency of the word, and t is the chosen threshold.
+
+5. Evaluation
+
+To evaluate Hierarchical softmax (HS), Noise contrastive estimation (NCE), negative sampling (NGE) and subsampling of the training words, they used the same small:smaller = big: ? method as the word2vec paper. And the results are: 
+
+![image](https://github.com/user-attachments/assets/13aa6544-ab08-4923-bdd9-359515778cca)
+
+6. Learning phrases
+Many words cannot be explain just by learning the meaning of individual words - such as New York Times. To tackle this, they find words that appear frequantly together and infrrequentyl in other contexts, and such phrases are replaced by unique tokens. But cases like “this is” remains as is. After doing so, the analogy method was used again for evaluation. Results:
+
+![image](https://github.com/user-attachments/assets/7ad21317-4a58-403b-9f14-0000de908761)
+
+Looking forward to the next lecture tomorrow :)
+
+
+That is all for today!
+
+See you tomorrow :)

From 7e993106a976807d09ea8684b7616630dc4706be Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:12:42 +0900
Subject: [PATCH 156/638] Create 2024-2-28-day-58.md

---
 _posts/2024-2-28-day-58.md | 251 +++++++++++++++++++++++++++++++++++++
 1 file changed, 251 insertions(+)
 create mode 100644 _posts/2024-2-28-day-58.md

diff --git a/_posts/2024-2-28-day-58.md b/_posts/2024-2-28-day-58.md
new file mode 100644
index 000000000..15100c27e
--- /dev/null
+++ b/_posts/2024-2-28-day-58.md
@@ -0,0 +1,251 @@
+---
+layout: post
+title: (Day 58) Stanford CS224N (NLP with DL) Lecture 2 - Neural classifiers (diving deeper into word embeddings)
+categories: [applying-knowledge,theory,research]
+---
+
+## Hello :) Today is Day 58!
+A quick summary of today:
+* covered [Lecture 2](https://youtu.be/gqaHkPEZAew?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) of Stanford's NLP with DL
+* did [assignment 1](https://colab.research.google.com/drive/19tmvOb4-izzCs8baOv3WM4Cwv26cdvKL?usp=drive_link) on google colab which covered some exercises on count-based and prediction-based methods
+* Read 6 papers about word embeddings and wrote down some basic summaries 
+  * GloVe: Global Vectors for Word Representation
+  * Improving Distributional Similarity with Lessons Learned from Word Embeddings
+  * Evaluation methods for unsupervised word embeddings
+  * A Latent Variable Model Approach to PMI-based Word Embeddings
+  * Linear Algebraic Structure of Word Senses, with Applications to Polysemy
+  * On the Dimensionality of Word Embedding
+
+
+Firstly, I will share my notes from the lecture:
+
+![image](https://github.com/user-attachments/assets/d8ac659e-4171-4339-ad02-c6d82db6c7ee)
+![image](https://github.com/user-attachments/assets/71c2b8f2-b82b-41d9-bc40-1363531a6b67)
+![image](https://github.com/user-attachments/assets/6bf238ea-e243-4e57-91b9-35913a7d5341)
+![image](https://github.com/user-attachments/assets/bc34a1cf-268b-4407-a32e-14c8cb2c2395)
+![image](https://github.com/user-attachments/assets/e598fe78-4318-498f-8856-42f2d33a5962)
+![image](https://github.com/user-attachments/assets/6f5ef840-ba00-4e14-b7b3-04588d8fdb59)
+![image](https://github.com/user-attachments/assets/02667e96-a74b-496a-8fad-1729ebc8dc81)
+
+Next, are the short summaries of each paper. These were definitely interesting papers, I feel like I am learning the history of something big, and after a few lectures I will be in the present haha. These papers, that preceeded the infamous transformer, and that laid grounds for modern embeddings were very interesting to read.
+
+### GloVe paper
+   
+1. Intro
+
+Evaluation methods for word vector models often focus on the distance or angle between word vectors. Recent approaches focus on word analogies and the two main model families for learning word vectors are global matrix factorization methods like latent semantic analysis (LSA) and local context window methods like the skip-gram model. The authors propose a global log-bilinear regression model trained on global word-word co-occurrence counts to efficiently capture the multi-clustering idea of distributed representations.
+
+Count and prediction based methods both try to do word representations. But with glove, the authours argue that the two methods and not that different since they both operate on the co-occurance stats of the data. So the paper proposes GloVe which utilizes the benefits of the count-based (capturing global statistics), and also the benefits of prediction-based models (adding the linear substructures popular in models like word2vec).
+
+2. Related work
+
+Matrix Factorization methods.
+
+These utilise low-rank approximations to decompose large matrices that have statistical info about the data. These models use matrices with raw word count and popular words like ‘the’ and ‘a’ can poison the data. To counter this, many models did transformations on the raw counts (as seen in the lecture notes).
+
+Shallow window-based methods. 
+
+Another method is to learn the word representations by making predictions within context windows. 
+
+Window based methods do not operate directly on the co-occurance matrices (unlike the matrix factorization methods).
+
+3. The GloVe model
+
+GloVe wants to answer the questions - how meaning is generated from co-ocurance statistics, and how the resulting word vectors represent word meaning. 
+
+They show how meaning can be extracted through co-occurance probability
+
+![image](https://github.com/user-attachments/assets/4e1bdc75-f140-4e8a-8300-d9943ed80c7b)
+
+4. Experiments
+
+They use the word2vec method of evaluation. (analogies) and the results are:
+
+![image](https://github.com/user-attachments/assets/de5237df-3fff-4af2-b429-96b3bbf7a134)
+
+They also looked at how context length affects results:
+
+![image](https://github.com/user-attachments/assets/a4cdc9fa-5b6c-4cb5-824d-1c8407acb5c8)
+
+Performance is better on the syntactic subtask for small and asymmetric context windows,
+which aligns with the intuition that syntactic information is mostly drawn from the immediate context and can depend strongly on word order. Semantic information, on the other hand, is more frequently non-local, and more of it is captured with larger window sizes.
+
+They also share the performance on the analogy task as a function of iterations.
+
+![image](https://github.com/user-attachments/assets/a3692530-3eaf-4df9-ab13-0fed3c6b96ba)
+
+### 2nd paper: Improving Distributional Similarity with Lessons Learned from Word Embeddings
+
+1. Introduction
+
+Recent (back then in 2014) papers suggested that NN-based models for word embeddings outperform traditional count-based models on tasks like word similarity and analogy. However much of these benefits actually come from system design choises and hyperparam optiimzations, rather than the algorithm itself. What is more, these benefits can be used in traditional methods as well, and the paper finds that there are similar gains and the difference in performance is not significant.
+
+2. Background
+
+The paper considers 4 word representation methods: 2 count-based methods: explicit representations (PPMI matrix) and SVD factorization of matrices, and 2 NN or prediction-based methods: Skip-grams with negative sampling (SGNS) and GloVe.
+
+3. Transferable Hyperparams
+
+word2vec and GloVe’s hyperparameters, for analysis purpases, are grouped as: pre-processing hyperparams (input data), association metrix hyperparams, and post-processing hyperparams (modify resulting word vectors). 
+
+Pre-processing hyperparams:
+
+dynamic context window (dyn), subsampling (sub), deleting rare words (del).
+
+Association metrix hyperparams:
+
+shifted PMI (neg), context distribution smoothing (cds).
+
+Post-processing hyperparams:
+
+adding context vectors (w+c), eigenvalue weighting (eig), vector normalization (nrm).
+
+4. Results
+
+First they do a test where the hyperparams are set to default. 
+
+Next they use word2vec’s recommended configuration. 
+
+And finally, they show performance comparison when using each models’ mest configuration.
+
+![image](https://github.com/user-attachments/assets/1296c569-5ac2-40d7-8328-afebd394fef7)
+
+5. Recommendations by the authors
+
+The paper does various tests and comparisons, but at the end they give some practical recommendations.
+
+They advise to tune hyperparams but it could be computationally expensive. They give specific hyperparam numbers, but also they mention that SGNS is a robus baseline model which might not be the best at every task but does not underperform. And also it is by far the fastest to train and extremely memory and disk space efficient.
+
+### 3rd paper: Evaluation methods for unsupervised word embeddings
+
+They study how various evaluation criteria and embedding techniques, revealing different relative orderings of embeddings based on word relatedness, coherence, and downstream performance.
+
+They also introduce a new evaluation approach focusing on individual query comparisons.
+
+In addition, propose a model and data-driven approach to constructing query inventories, emphasizing diversity in frequency, parts-of-speech, and abstractness, and releasing a frequency-calibrated query inventory for systematic evaluation.
+
+And, bbserve word embeddings containing significant information about word frequency, even in models compensating for frequency effects, challenging the practice of using vanilla cosine similarity as a measure in the embedding space.
+
+All experiments are carried on the following models:
+
+CBOW, C&W embeddings, Hellinger PCA, GloVe, TSCCA and Sparse Random Projections. And using Wikipedia dump from 2008-03-01 for training all models besides C&W, which was trained on a 2007 snapshot.
+
+Results
+
+The below shows the results on 14 different datasets, and the performance of the 6 models. Something to note is that CBOW outperforms on 10 out of the 14 datasets.
+
+Results on intrinsic evaluation tasks:
+
+![image](https://github.com/user-attachments/assets/effcec3e-c565-4e63-872f-e193cfd37b04)
+
+Extrinsic tasks
+
+They measure performance on a specific tasks. They find that the assumption that that there is a consistent global ranking of word embedding quality and that higher quality will result in better results, fails.
+
+We can compare performance across tasks to gather info about a particular word embedding’s quality, but we cannot judge further that a particular word embedding can be a top performer across different tasks.
+
+Other findings include:
+
+Despite operating on similar input data and optimizing similar objective functions, word embeddings from different algorithms encode surprisingly different information.
+
+Word embeddings contain information about word frequency, which affects their performance in tasks such as predicting word frequency categories and computing cosine similarity.
+
+The amount of frequency information encoded in word embeddings varies across different algorithms. For instance, GloVe and CBOW embeddings, while both effective in intrinsic tasks, differ significantly in the amount of frequency information they encode.
+
+### 4th paper: A Latent Variable Model Approach to PMI-based Word Embeddings
+
+They propose a generative model that treats corpus generation as a dynamic process where the t-th word is created at step t, and the generative process is driven by a random walk of a discourse vector ct, and its coordinates show the context, plus each word has a time invariant latent vector vw that shows the word’s correlations with ct.
+
+ct is a slow random walk where the meaning of discourse vector at time t+1 is obtained from the discourse vector at time t + a small random displacement vector. At each time step the probability of a word is generated by a log-linear model that relates the word’s vector and the current discourse vector.
+
+Usual count-based methods use co-occurance matrices and for a selected k, they want to see how many times word 1 and word 2 appear in a block of k words. Using raw count matrices turned to be inefficient, so some non-linear functions started being applied. The most popular being logs, square root, or Pointwise Mutual Information (PMI)
+
+![image](https://github.com/user-attachments/assets/a1604a6a-b3eb-4b47-ba4f-8b79893fac7b)
+
+As seen in the GloVe paper above, an additional weighted-SVD factorization of the count matrix X was proposed too. And also word2vec show how to make word embeddings through neural nets.
+
+The authors argue that the previous models are like black boxes, and also there is an issue with dimensionality. The paper counters the popular assumption that simple models generalise better because their process of making word embeddings is unsupervised so the idea of generalization is not aplicable.
+
+![image](https://github.com/user-attachments/assets/04f28203-d6d9-4e31-9be8-5e7a388bd039)
+
+If a word vector aligns closely with the discourse vector, that word is more likely to be generated, and then the process is repeated.
+
+### 5th paper: On the Dimensionality of Word Embedding
+
+The paper argues that the dimensionality on word embeddings is an ongoing mistery. It is a crucial hyperparam that directly impacts the quality of a word vector (a small dim might not be able to capture all possible word relations, while a large one can suffer from overfitting). Also, dimensionality directly impacts computational complexity which can impact a model’s applicability, training and deployment.
+
+The proposed framework is based on the following preliminaries:
+
+1) unitary invariance of word embedding. This means that two embeddings are essentially the same if one can be obtained from the other just by performing a unitary operation (like a rotation). 
+
+2) Word embeddings from explicity matrix factorization. Many popular embedding algs use explicit matrix factorization, including Latent Semantics Analysis (LSA). In LSA, word mebeddings are obtained by truncated SVD of a matrix M which is usually based on cooccurance like PMI matrix. This M = UDV (SVD), and a k-dim embedding is obtained by trucating the left singular matrix U at dim k, and multiplying it by a power of the trucated diagonal matrix D ->
+
+![image](https://github.com/user-attachments/assets/0aeb67b4-28b9-42ed-9ec1-d61f6701fa9a)
+
+Caron (2001), Bullinaria and Levy (2012) found that a different alpha words for different tasks. And this paper discovers that alpha controls the robustness of embeddings against overfitting.
+3) Word embeddings from implicit matrix factorization
+
+The 2 most popular embedding models word2vec and GloVe  use implicit matrix factorization  and optimize over some objective function using stochastic gradient methods.
+
+PIP Loss: a novel unitary-invariant loss function for embeddings
+
+A reasonable loss function between embeddings must respect the unity-invariance, and this rules out options like direct comparison - taking the difference. The proposed new loss - Pairwise Inner Product (PIP) loss is:
+
+![image](https://github.com/user-attachments/assets/3ff25649-11df-4739-9b05-def92f503524)
+![image](https://github.com/user-attachments/assets/4b0d8121-4c39-4add-a7a7-c63e60ddb8ec)
+
+Earlier we saw that alpha can affect the embeddings’ robustness.
+For alpha = 0, they find:
+
+![image](https://github.com/user-attachments/assets/0a477672-a431-4170-950c-bf8f6475ea82)
+
+where they find that the optimal dimensionality k* which minimises the PIP loss is between 0 and d - the rank of the matrix M.
+
+For alpha (0, 1] - between (excluding) 0 and (including) 1, they define Theorem 2 
+
+![image](https://github.com/user-attachments/assets/c8b9ddd1-a693-4f87-b0b3-1917e827f44a)
+
+The first item is the bias as we lose part of the signal by choosing k <= d. The second term is variance on the magnitudes. The 3rd term is the variance on the directions.
+
+The final theorem they propose shows that the bias-variance trade-off reflects the signal-to-noise-ratio in dimensionality selection.
+
+![image](https://github.com/user-attachments/assets/b05ee742-ab9b-499e-aaf3-2177b1c2aa92)
+
+Theorem 3 shows that when dim is too small there is high bias, and when the dim is too large, too much noise is included (high variance).
+
+### 6th paper: Linear Algebraic Structure of Word Senses, with Applications to Polysemy
+
+Models such as word2vec use an objective function that includes a nonlinear function of the word co-occurances matrix. This type of complexity makes it hard to understand how these embeddings caputre different meanings of words with multiple meanings (polysemous word). There have been researchers that tried to include separate meanings of words into separate embeddings. But this paper argues that multiple meanings of a word are included in the linear superposition within a standard word embedding.
+
+The paper argues that through the relationship of a word with the words it co-occurs with, the resulting word embedding can capture multiple word meanings.
+
+They propose Theorem 1:
+
+![image](https://github.com/user-attachments/assets/d067825e-0f9f-4a63-8729-56281719cff2)
+
+This theorem shows that thre is a linear relationship between the vector of a word and the vectors of the words in its contexts.
+
+They also derive Theorem 2: Assuming that embeddings satisfy
+
+![image](https://github.com/user-attachments/assets/36377822-5461-46fe-aab9-2941528a40e5)
+
+as the corpus length tends to infinity, then:
+
+![image](https://github.com/user-attachments/assets/dffb05ad-fe27-4d0d-8f5b-9646c1c6a16d)
+
+Word Sense Induction (WSI)
+
+A task that considers the contextual information of multi-meaning words as a representation of word senses and utilizes clustering techniques to analyze the contextual information.
+
+The paper suggests a method for WSI using word embeddings and the concept of discourse aroms that are obtained from sparse coding on word embeddings where each word can be represented as a combination of these atoms. And the paper believes that these atoms can capture the different senses/meanings of a word.
+
+As the night went by, my summaries became 'a bit more summarised', but nevertheless the papers were nice and relatively easy to read and follow. 
+
+Tomorrow we continue with Lecture 3. 
+
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From abf29b7ae945f52eb09094782322ef6acf494e13 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:18:16 +0900
Subject: [PATCH 157/638] Create 2024-2-29-day-59.md

---
 _posts/2024-2-29-day-59.md | 44 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 _posts/2024-2-29-day-59.md

diff --git a/_posts/2024-2-29-day-59.md b/_posts/2024-2-29-day-59.md
new file mode 100644
index 000000000..586b0ac34
--- /dev/null
+++ b/_posts/2024-2-29-day-59.md
@@ -0,0 +1,44 @@
+---
+layout: post
+title: (Day 59) Stanford CS224N (NLP with DL) - Backprop and Dependency Parsing
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 59!
+A quick summary of today:
+* Lecture 3 of CS224N: backprop and neural nets
+* Lecture 4 of CS224N: dependency parsing
+
+**Lecture 3: Backprop**
+
+After Andrej Karpathy, backprop and I have a friend-friend relationship, so this felt like a nice overview over backprop in NNs.
+
+![image](https://github.com/user-attachments/assets/3a08849c-6212-43eb-8017-074e29618d30)
+![image](https://github.com/user-attachments/assets/5877997c-45bb-4f2d-b489-34b208c32983)
+![image](https://github.com/user-attachments/assets/33896d75-1e95-418f-97c9-d17f8a63179e)
+![image](https://github.com/user-attachments/assets/d4520736-c644-4bfd-b302-c0cfec44d359)
+![image](https://github.com/user-attachments/assets/e7dfdbed-2c0d-4017-9986-4357abd4eab2)
+![image](https://github.com/user-attachments/assets/38360e98-68d8-4cf0-bb1d-d95fdb8784e7)
+![image](https://github.com/user-attachments/assets/7fc40c9b-16d2-4431-92be-9e9ee23e27c2)
+![image](https://github.com/user-attachments/assets/adf44370-04ad-4a31-842c-016af6784382)
+
+**Lecture 4: Dependency Parsing**
+
+This one felt like a linguistics lesson, learning about how people interpret language and how such logic transferred to computers.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiiehYX3WxoGGicK0_cadAdXw82rcO0kS2Y0Q6CnY0DpkvS2psL0BYP1hEi2U2sMMITCvLo_EEYzbsXtf37VXnIqns0LLvHL_hKwbMJ9f5BcbO-zC2LfitJHoAgEfnmCsKn1LpgVYSKlaGrwDpDZ98oqmeJ5P3sswRcdZG8PJR2Faal7xHEyMKTJWPwBy3c)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEieaBEBNXIZPIKJI1L5hxhuc9OiNXBlE_Vuyy1ZOyijJjqq4okvGSWbVKDMGlsD6rDgmTMwEHuRp5ArIXxUzTcvYRtZDBiCon3WxIE-o5LdOSGbVB10AyRtHUwksiNKzdXmb3G9298B9bu9vo9V2PsUjGBsWj9uBuKmWZu54ZHrcGelv5SKRj1Yst3Mt88I)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiCJF7rl8loI2M45p3JID2dHHLjfBvR5wImjS7Y_Xj3X0HdzzdYWQgrjurnea1ouHiycAVHdoUv2HwcHV0dRejgFaZ5AXAorXv42YG2u1A3tsLTU3FbyN266urJZDviPxCcLR8EtpbxjC0awVbu50hW6V9HchaXcYcaAJFNIX0DTeopZuv7uDRzV3gYv8Pn)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgRFRA8Cwp-3cQwdIo8IxQVFU1jeaM9NWVwd7uXtStnqg6wBeuKUqUrcK8aB26gGmLA3LRNEo_yVaNOjJeD4gLL2VgE_tVESRpjrtrZHGV8DVaKS4Opdk1ah_ql1Pd7ti3rJN_M689RBKgCu-16_f---cQlEI2nTeS1qgdGqmkIWlmltmBqnnvcB8miyNRb)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjICXfGWN7Y_5rj2zXwtJWeUwi9kh_OdPoEl4pNN6frstlPwUBMCp5SrAVsv9ZIj7cTTCxFJwvbTnHX0QDTSeXvPOFf5D2hQZKVRVl2yLHI7p4ce6q-3XAmEOHZe6-9ZoRti3A3b5k38KCLuKsDDhKHAoakKLAAih3qRWOEOw85qRXXTx8yzMY-kMrxMB3u)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjYSUzL6bUPGv0HF7vuSY6CsaR4zKMjO4KniyoH8JYfFotmkCq9XndNF1Jyw-yLlh-j8wS2vPqWrKjxL6Grvz8yY_cW6KyVSkpW8EZyAZNKeslIt7SH2VCcZwK0OPwN6EakBaPsDeImHK9YCQ2mOkB7Yc1fkzCTq7WMhwgd1YzgPE8I9N6bDqZar2oKNQO5)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgribT77tkjfCWm8zrfWmtBv8lId3IUPfTjjrExv8rprzWW11-vNeHtiZ3mkSPWTgfYXBBEx5uWa6PIf4-MjmfnBdFJEywo_87fA6df3pNOOPw1bQ85XJ6WmvvSlgyDMyphwRufW8VdKzI175V4D07omiOdy73xqZa7eFpXJ5XP9f7TzjTbvVGEgYQmV-4J)
+
+Tomorrow is RNN's turn. Really exciting!
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From cd6208ab30a21ae830db3ff9e500e44e8149a47f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:26:23 +0900
Subject: [PATCH 158/638] Create 2024-3-1-day-60.md

---
 _posts/2024-3-1-day-60.md | 47 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 _posts/2024-3-1-day-60.md

diff --git a/_posts/2024-3-1-day-60.md b/_posts/2024-3-1-day-60.md
new file mode 100644
index 000000000..6ab3ac7f1
--- /dev/null
+++ b/_posts/2024-3-1-day-60.md
@@ -0,0 +1,47 @@
+---
+layout: post
+title: (Day 60) Stanford CS224N (NLP with DL) - Language modelling, RNNs and LSTMs
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 60!
+A quick summary of today:
+* Covered lecture [5](https://youtu.be/PLryWeHPcBs?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) and [6](https://youtu.be/0LixFSa7yts?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) from [CS224N](https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1214/)
+* Attempted [assignment 3](https://github.com/divakaivan/a3) - Neural Transition-Based Dependency Parsing
+
+So far, including today, the lectures have been just phenomenal. Based on my previous NLP knowledge and now with these lectures that dive deeper into how modern language models came to be - I am establishing a solid foundation.
+
+My notes from the lectures are below:
+
+**Lecture 5: Language models and RNNs**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEifWdjGMwNLKE-26OEwGiswg3sBVbb0bsdvo5crZ6aNQOh1jZ8Sp6jDK1OCGWDDiUMzrIwbdBUHobEkXGPf7B53ZWK7buKkTwAWRGuratOylCa5lRQBgus6DpQjj3qWUcFq5CahXvxAJIxNz_xZ32L2POGffRxAYbTNL0SI2FdvhkBcASNnCkHTmAQGRsxe)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj_jP9UDfcpXBNF8ocKQy6-gk31A5iOarOBFohXADJvRXfNCc90RN-rCBlaZjboIEUG-WDQosK88IgMnLBmn5NpDi9kf5olF-ON0kiJKHh3iQo2MKTqaUUILEdFnBcFH7WcFSsmdaMTSk1KtmPGYLyCGJEBWiagmpPSIkc5kZ8miy-dpiVQNi4BGtKoMA5O)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiHeFm6wlSTDWDR2UZf__10H61C4GM6dZVT67pehzJxjPYnx3SLMKvRwVMt8GBjeadqRzoKynFcLyQ2U1uY1pMw4bkBRbDxwIlGNR1ss9HceGgRkpoQzVocxHlARCboj-2TZ22iRy1YnY2_uQHs-gF4U9X_zxEFNXkKUZ4otwB5xIUUTfWG1FmfpNZ2-SQm)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh9q9oAB2Ou98YMB7tKRtuzHBhkJWDZB53eNng2eO5TJ96gCflynl1FYhnSDxZlTxTmSA0lpqHBVMp7eAYHgTmnE6k7Jn2Mg-0H8rpRdefYi3MfPlQsDBx9vzER5TEYiucLC_XIllKdh_sdtf4ETmaSLHIRiDDrDhhD4pwJ6tZLDzdZNqTYZ8BTxAJZ5wcG)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhBRN4GL0Ps7Q1UiS0PYiUjwd7S-g0FMdzxv8foxcfQB1Xc7lb0mhzbXWK6_hU4GbtgFDW3pHXi7TgUkTQmcmUcoeUlL2dbfqK2QYgegioFSCBpjGj6ieiYn4A_ns1hnvrE3FUWlJxwsuV2Es669N8CTnmoedvMhZpO-iUKm3MKf4GoT02I---mME1mtS5T)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjKKZW9ukwpRf5ih7-Cu-Dyc78VZLBnDPm0408OSDKRA5M2KzP2g9NumpnQlsubE4MsZPnbRXEBty5NhoKDhdClYOvR1FRMbweW25raA5pKod7SK0FvWawFkh21iU9_KVnkgk4x02FZDy52reLebjRzWZxyCFbDRRILWJDUhmOYGtBUqj8rEoB_j4amAC12)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhCngZFHPF0H-akX0DOwn6YfRg7x-yv6u9nAyo_w4a4OEJDn5w4mmSeAZQDsQmysZW9RqfewrD4gXF8qU2LvXJPHPmNu1YK8SKr_Gm1M-VikNcRImEPLwq44oWp9HQXABsvD05ectJH_OH9vBAwBZafhJY3kIm1N6rniaRJZruQzcs6Z9XeUfi0bK02wBwx)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjMKnUFOPhbwUnd9K0wHibJ_g9VSL3DSrfjtblSXEK3jMGEu_fa25bizSdn03RAVm25-V_kATeWvm1IhwchMRSswSVZ9QqmLWq66AejE6V0ihBSn09jgM3JmO3npfRzRCcAUWpTyMkebcaxuMbYt3Vq7Dn0tQ6IHeyYRkaMk6ek3qs4yBDrugzbFcEaXKt4)
+
+**Lecture 6: Simple and LSTM RNNs**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiYCr2DdCJ15EWG9CCcXPq06p04zBNy2IZiK0RaLJ75D9a6rg9rrCOBO_W8SAiT6oAiLwt6o7YkydjEF3u6ZmBXwhky6aU5TVATFV7-cSDZArlkDmszsUFjtgj4yQx2NTKm31RLa0vVaH-JYEINozlRYe6PNKqF6KpTQuT1Y1CMzbnrV6QqH0t9MAtwt_UP)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh6IDP0fCmHj2PntECgUAB8JWNNmO3sV-Vt05Ax5fpaa8C6Q_8NDf5flE1gGL6zcWmb-TqcJobMo7r9ifv1srXalkatUoU3Gsw2XuKqjOH1XaWKj_QITsSGkHWEJUyzrTD9nBSa7ch8xxI1ACxZJwljvh2-yup3dvZin_7_e0WC9HuLw25r0Zdrk17ATtdF)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEguGZ8jxC_VcRKqm182PdcJ7B4M5Pb4GVszY-zKhbcxOMTeUMHq7rkHunC_4NfIiw91mCc0qnwo4K_QoWN99dTxEKAWuOOWr3JqFVD7d--B6OgaDakOmuDJxQVehz1Q8YFSPlJZ2dwIFgM4ptu8dbDmcmdxCGEg9JqTjWGl3fdAtGnw2A9nDw_3rKaOuOG-)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg6Zbw7Yr_JjM4_pauyqxV_5HaQIDqhLyfeorgmO91N416prF46Q2Iy8v-YHlFoNmlwpuUxz_hqb70M8EFt26cN0EdMUsowmW-Qbmt5WBnyrMORbh1lIfwjfPCj3JlGQEr8OOMIM7uhOCIfqtOwn3hNbNx4R_hY1GD82-q04x-0Tamn8oRClJUWbm-2Prqz)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj8nZwc1VgQ5uxKkQ2sqj9pLzF1GXB7lGe0uqoFtsqkrfgXlKch5sdKBbHtihSkB6Lev2ojAZIzWS2zIgLx_eADDu2aUAFEMz19v-W4NjowR8pHxdB2qw7sDng92ARecyygzkiNog84g7wXL_0uSriHfVPwzGRESHjO1FSFThZVR5GDFeBtJcsVLvJ-nzX5)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhpvKHnF6vW1zWWW-xLAR8z1Ezyj4zT3LpH9Q91lFD3SKT6XVuTb49EnfRB_PSckMLGVC2U3nXs_uv3i57pTzdtngWEYV7_CTTXlIr4VeQhKyBFTqkaG326vZRrwa6TmFf4z8j271r5skyOF1mMkGoVvqWVm6kkQrtecpGYDTlaaT65r_LJU1_TwtZjXlr_)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi-xkEDKBHipbJahhvuXG29PgkzQaqKlEPt_6EKP3N4wBuX48y6JbLdD7ssjz1QtdQg16-Dc3hKxMPAw-PF8il2wq5mQXH03oUCic6XjFlAWXomdATizuOX_1bs0MRG74jCju0vJo554QE--KkPkBq2MN7riU79l8JGBFhEYvv4Jw3iraVlHdbU2D4fWq5w)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjx9pZbvDjTRvsn_cpQiyttiwRHDKd2TwPKfaKsh8WeAK574S-MaAh41YrWmbD5ctnvHusieyDvg33T41rOPaTKfFnKENsYy2h4epTaV_FmYhRwCD-w0hrEJ_tKB2cQI-1HjQKkO3GlBUkBuB8GCVV6O-h0TQlAsX0JKFMJG9uBGsf1ISi4-ymqeYIDvtTw)
+
+
+Tomorrow's turn is Lecture 7: Machine Translation, Attention, Subword Models and Lecture 8: Transformers depending on lecture 7's length and readings.
+
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 8a4673e13838f80af271faff858cedd86902976a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:33:12 +0900
Subject: [PATCH 159/638] Create 2024-3-2-day-61.md

---
 _posts/2024-3-2-day-61.md | 54 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 _posts/2024-3-2-day-61.md

diff --git a/_posts/2024-3-2-day-61.md b/_posts/2024-3-2-day-61.md
new file mode 100644
index 000000000..56502471b
--- /dev/null
+++ b/_posts/2024-3-2-day-61.md
@@ -0,0 +1,54 @@
+---
+layout: post
+title: (Day 61) Stanford CS224N (NLP with DL) - Machine translation, seq2seq + a side CDCGAN mini project
+categories: [applying-knowledge,theory,practice,cv]
+---
+
+## Hello :) Today is Day 61!
+A quick summary of today:
+* Covered [Lecture 7](https://youtu.be/wzfWHP6SXxY?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): machine translation, seq2seq, attention from Stanford CS224N
+* Tried to make a conditional DCGAN to generate MNIST numbers [(colab)](https://colab.research.google.com/drive/182I_gBSORJfNdVZuxcBYKFNxo0gNNLFx?usp=drive_link) [(kaggle)](https://www.kaggle.com/code/divakaivan12/cdcgan-mnist-number-generator)
+
+I will first cover the GAN story
+(then will share my notes from the lecture)
+
+So... while watching and taking notes today, I started thinking, what if I can use my notes as data to a model and afterwards, when I want, I can give it raw string text and it will output text in the format of my notes (with my handwriting).
+
+Well I started looking around and actually the first model architecture that came to my mind was the GAN (specifically conditional GAN) - I remembered there was a GAN architecture that alongside the pictures, we can give it the labels, and then on-demand generate. In retrospect, there are of course others, but I decided to go with GAN. 
+
+For maybe 2 hours I busted my head trying to make a simple model with the EMNIST dataset (english characters), and I kept getting weird input size issues. And after a bit I read online that the num of classes for the EMNIST from PyTorch is a bit weird (i.e.). What is more, I saw this: 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgNsRHOFYgkTUIo1ZMQt2aZbago04ZZQ27lsuMOaL3s69gqgMhDsPzLitWH7FAOckrfGqrkG3A17f_Wh2H-BeQv79r7s5rp9cyUlXmZJE3jTGNyj4WEvEh8eMKCusRnAPb1ImPXnE3JDYZo95V5FXv4_JNIsBkzQ9IrA5GXDdBhb8W4z1o2U2T0LYajOobN)
+
+This image with handwritten text is generated with a conditional deep conv GAN ([repo link](https://github.com/tomWitkowski/EMNIST-CONDITIONAL-GAN/tree/master)). And my mind was hooked on the idea -> I want to do it too. But firstly, I wanted to do it with just numbers, a bit more simple (and not having to struggle with loading the EMNIST dataset). 
+
+Thanks to [Aladdin Persson](https://youtu.be/Hp-jWm2SzR8)'s youtube channel, I got reminded how GAN's architecture works and I had a simple model, and started training in my google colab. After a few hours of adjusting params to optimize training and lower loss, and also doing inference - producing new number images, I had a working model. I was so happy. This felt soo long, at first with the EMNIST problems, and then just getting a conditional DCGAN to work. Aaaand... my colab gpu free time finished in a middle of a run and all was lost because I had not saved any weights :/ I did not even take a screenshot of the generated output numbers I got - and they were amazing, human-like. 
+Right now as I am writing this, in the colab, there is no output, but hopefully later if someone reads this, you will be able to see sample results (I will rerun it when I get my free gpu hours). 
+
+But also, I am running it on kaggle right now, will get a generated output pic and go to sleep because it is almost 2am.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhESTy9JKmwJ6SmXFZnmf4cfWdT5S71W8gu1n9OzCycccRMUa1N9Wjn7F-bu9MO3q0bBnVYyjJ4QqPhnnI-xnvqFVWT7R6GFdr3QLJwGU70nIZtW5Z8BQsHGge1u8PcnyZy8X3Y35r6zG6EvxFMiiIOr5BrLcnLDKmvL9-mwnqP2maM6ECib0fxmDH9VceR)
+
+So, this is the result. Not too bad. But in colab I could use tensorboard (there is a way in Kaggle too, but it gives me an error :/ ) ([Link](https://www.kaggle.com/code/divakaivan12/gan-mnist-number-generator/notebook?scriptVersionId=165157007) to the kaggle notebook)
+
+**As for my notes for the lecture 7:**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi84Rwlxve20YMNqV8YpQj7jlcxEQE2I2CRhSa8stLU_L7k03fjbNtd3BIr92im8oGX1cZRAaCUsB0IsAn97fH5LctE2kPnAvV9MycpLxdoGZHXPTJycdVeS7gqgQDpqdH0nXvWbd7I9Nk02HgdzvWldcB2-iNAelLMWu52K1JEKfM67WCF3BAMeY3XK3an)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjrXsptU4mi-uksXYqm0x8XIDzJ4Dm7pGbiUArkD6bOjcUlg3q4rQMaW4GUGBZrZ2iQ12NxpvLBKzQDHBgPAXfC_L28OjOMhyCiI64fHsJKXBU_-A0_sprBOOU-hAAJ0UqJ3oCTUd0YwvvwPXakBfLBpoJ1EpYngzghe2k53BtYsyApTP8-6KLBmDAOA4-u)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEizznICqyDCJ2lX4XhlzriZfWSAyMCBXOgVxmNg6OLts_l_pBJ5mFbLsT0Ue0s9_Pl8wwptEAR7A9rxz_jEFBDpe4Z_Bv4K3rVtd9xGHKsIh9tg7d9EJNSJ8WP2OPVIzNqF3obNkuRT9Xg_zHUIbN8wCo0C8HZSmDwSDhnObJUIcLzvW4JJJdfMTM0c9K9E)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgOS_Mt_J8jmspF6AquBiP-Y5f6SBzGJTAlfm9q-l1gaHSTHZBdz_aL8KHL6hdreb-zaIKrZLVAqXQdt96tEfAF250O3ncwL1F_jaUVCfk-rXkSm9hIXxrPNWrP_YClAkvv8UniC3jPirUJNbytbEiPUR11un9xtESPLNid_65GfpP8I9GSMtvi99MYxYr2)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhi3uT-5ULVJlO6grpP5gDaAzFZqSBqGyWTOFqF0MFNctCvGpKUKGs367RkpAMIEDRweBH2GiJGsB7UhYnkl9qJVPtJbuaT67snfEB0mI5QZnPfo8bqa_Jpny9ZjvEKSJfP2RWF0c2IgxMBIG99pkZyMkrkQcQJKlVU0-9iXjqVhwpc_XQUUxVwIrZVCtsL)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgpG72KVWtdTt-hPKcA6kW1WF5rvCL4RKEnlnteYfKOqKXFq05tpEB5vgJQFpWrQsyYhUgih7LSF3g1Oj2xJC6eT6hlGz54xEcgYNdbSvu7hbzjTXk_6CLMvr6oL9S0T9DItJc983tfL3zTEtdDDUFWMZlDgE94jrF6UEY2fejPYiUM6TewL0doMYsq0pZ2)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgeCUCHL9EuB6Ca5JkgRtGEdmgLCgpapiLjn7tA0NMDnpF07NOVV2oUGSOgZaYH7M-bGsR107fGvANsc1oswvrxgyvyH_N1cs2nIXuUdXN66K1SFrE3TAImakYqRUH_SwneOB42_1ON-A1y2sNaTZXg8S8oWBUA73N-bC8aZ6-eSnhYh8JAdI9hYxaVM0B9)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiuGnhGmKexRrUHyS0nkukydWv_FuxWrPBui3GyVOk7_J0o6jljPk6cWicxzehy4nFg1B5lbOZIw56RB88YUDbh-H6zj6hSbd85juQr9aKb8lzh07lCP0LH86vPPIhaLWLZb9BNHwFGzjHygssWeZ69kli44kTeZjVDJSZpVE3lAKPVkO2lzamo8BXo_KGd)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhNgDvslku5giN8ajDjqU-xZ1-xJ9NsD-M8BLxg1CGy646HVLXPRnTtr_dAWMeWu4byZldvhS6Ar6VpRVlFyjTMq01mFe0thUIrmnS2oERaTncVAMfQnVS8ibtcouDcum63LY06BxGYlNdcqtibFLNs57sl6qW7D7OH04TqNzWYEbYFJdW64hLvqDnfx1Fs)
+
+That is all for today, tomorrow we continue with CS224N! :)
+
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 0435d0df92511230bd8e2b75cba348d035a75a29 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:39:34 +0900
Subject: [PATCH 160/638] Create 2024-3-3-day-62.md

---
 _posts/2024-3-3-day-62.md | 69 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 _posts/2024-3-3-day-62.md

diff --git a/_posts/2024-3-3-day-62.md b/_posts/2024-3-3-day-62.md
new file mode 100644
index 000000000..576efac26
--- /dev/null
+++ b/_posts/2024-3-3-day-62.md
@@ -0,0 +1,69 @@
+---
+layout: post
+title: (Day 62) Stanford CS224N (NLP with DL) - Transformers, Pretraining, RLHF
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 62!
+A quick summary of today:
+* [Lecture 8](https://youtu.be/LWMzyfvuehA?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Self-Attention and Transformers
+* [Lecture 9](https://youtu.be/DGfCRXuNA2w?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Pretraining
+* [Lecture 10](https://youtu.be/SXpJ9EmG3s4?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Prompting, Reinforcement Learning from Human Feedback
+
+I watched this [State of GPT](https://youtu.be/bZQun8Y4L2A) talk by Andrej Karpathy a week or so ago and saw the below slide:
+
+![image](https://github.com/user-attachments/assets/14f22a19-d002-43be-b7a1-2ce2a8374068)
+
+And that was the first time I saw this GPT pipeline. Today, I got to learn about each of these steps. Feels nice knowing how GPT/ChatGPT works. 
+
+Anyway ~ my notes are below :)
+
+**Lecture 8: Self-Attention and Transformers**
+
+![image](https://github.com/user-attachments/assets/605f14d3-ca63-4ff0-9edf-f1d0ae18a97b)
+![image](https://github.com/user-attachments/assets/96e0b2f7-0173-4f37-9fe2-ba4dc7ca86e7)
+![image](https://github.com/user-attachments/assets/256ef7bb-0c5d-4fd8-b8f7-2614323d67e3)
+![image](https://github.com/user-attachments/assets/22511968-6125-4280-a19f-765686600fe6)
+![image](https://github.com/user-attachments/assets/c414c587-3a33-48e3-90e9-db144f5064d7)
+![image](https://github.com/user-attachments/assets/1914b878-1081-4566-b89b-04cb5e35d27d)
+![image](https://github.com/user-attachments/assets/2e90f7e0-8eab-43aa-92f8-13ac3fa26dfa)
+![image](https://github.com/user-attachments/assets/5c868bcf-bc77-4ba8-a432-7371c5c9b4fa)
+![image](https://github.com/user-attachments/assets/54cfdc45-bed6-4c41-9410-a3d238ebbd00)
+![image](https://github.com/user-attachments/assets/1429c8ed-3cd3-4401-83d1-1ea630c5bda0)
+![image](https://github.com/user-attachments/assets/68135587-0d5f-4c08-8024-9e1a6d841896)
+![image](https://github.com/user-attachments/assets/3811fc89-3b03-42ab-8efe-47f34d76817d)
+
+**Lecture 9: Pretraining**
+
+![image](https://github.com/user-attachments/assets/4599af29-73e5-47d3-b2b5-5ed76de76dd5)
+![image](https://github.com/user-attachments/assets/b5529876-a802-4756-b57f-a657c0b1e74b)
+![image](https://github.com/user-attachments/assets/bb85908d-0a9e-4dde-b2b2-89bcd7916ff7)
+![image](https://github.com/user-attachments/assets/e14616d1-0b5c-4833-930d-1973d6f5a5a4)
+![image](https://github.com/user-attachments/assets/feafa9c0-234b-4fe6-9c2b-eb4f3fef5196)
+![image](https://github.com/user-attachments/assets/1f6857e3-8300-4092-9a53-9a3324040bb2)
+![image](https://github.com/user-attachments/assets/d48ac3fd-61a0-4860-95ad-85d9d990e181)
+![image](https://github.com/user-attachments/assets/92bee2d8-f77c-4955-8818-84620c40f144)
+![image](https://github.com/user-attachments/assets/d946fcd9-872d-4ced-9600-b3cba030ed6b)
+![image](https://github.com/user-attachments/assets/71005827-29fa-43fd-93c0-520b0458c629)
+![image](https://github.com/user-attachments/assets/8d92096c-8409-49d5-bb29-f7f686c9cb51)
+
+**Lecture 10: Prompting, Reinforcement Learning from Human Feedback**
+
+![image](https://github.com/user-attachments/assets/8b8d8d85-22df-4a0c-ae87-14646089600e)
+![image](https://github.com/user-attachments/assets/0fe70cba-37b1-4cd9-960b-467d1d81ff57)
+![image](https://github.com/user-attachments/assets/62c0af45-d635-4557-ab11-38aeabc4f3ef)
+![image](https://github.com/user-attachments/assets/923799b1-a458-40ec-aa24-2455db773d64)
+![image](https://github.com/user-attachments/assets/10ff414b-20ef-458c-993a-8c4a3eaf8375)
+![image](https://github.com/user-attachments/assets/f3d07a34-5062-43ca-8cb2-5ae635e347b6)
+![image](https://github.com/user-attachments/assets/c24f4db1-7588-4f57-a06d-dd75385ae0f5)
+![image](https://github.com/user-attachments/assets/4c8a24ea-9235-4873-89f9-300074469b3c)
+
+
+
+Tomorrow we continue diving deeper into the world of NLP. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 2acc4ec38a3be4f90fb5220987d88516deb2779f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:45:51 +0900
Subject: [PATCH 161/638] Create 2024-3-4-day-63.md

---
 _posts/2024-3-4-day-63.md | 43 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 _posts/2024-3-4-day-63.md

diff --git a/_posts/2024-3-4-day-63.md b/_posts/2024-3-4-day-63.md
new file mode 100644
index 000000000..63a433146
--- /dev/null
+++ b/_posts/2024-3-4-day-63.md
@@ -0,0 +1,43 @@
+---
+layout: post
+title: (Day 63) Stanford CS224N (NLP with DL) - Natural Language Generation, Question Answering
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 63!
+A quick summary of today:
+* [Lecture 11](https://youtu.be/N9L32bFieEY?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Natural Language Generation
+* [Lecture 12](https://youtu.be/NcqfHa0_YmU?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Question Answering
+
+**Lecture 11: NLG**
+
+![image](https://github.com/user-attachments/assets/8ea0704a-49fa-4a9c-bbb6-69fe415ada5e)
+![image](https://github.com/user-attachments/assets/c92a087b-79d7-443c-a157-4f97892f1851)
+![image](https://github.com/user-attachments/assets/f6716104-45ab-45cf-b1d6-4b4cf4860778)
+![image](https://github.com/user-attachments/assets/fa95cddd-ad67-48a7-bde8-cbbd552d314a)
+![image](https://github.com/user-attachments/assets/5e6b4517-1767-4faa-b667-62eb9eb52df0)
+![image](https://github.com/user-attachments/assets/3ec55ca1-4d22-4d54-98a2-3ebbcaae2904)
+![image](https://github.com/user-attachments/assets/bc716a6b-86fd-493c-a606-5263759634de)
+![image](https://github.com/user-attachments/assets/95e428d7-1036-4440-a217-fc637be9a7e8)
+![image](https://github.com/user-attachments/assets/6f0a3d88-2ff0-44ae-960b-8c6d3a090c5c)
+![image](https://github.com/user-attachments/assets/47d5ef54-23ed-4d26-b135-2f09b8fedbd7)
+![image](https://github.com/user-attachments/assets/1ac0ae12-c082-40cc-952a-9e0725fceb8c)
+![image](https://github.com/user-attachments/assets/36483454-dfd3-4a48-a6a8-3806d0e3e78f)
+![image](https://github.com/user-attachments/assets/6f4c6592-c686-4ec1-a5f2-bb3f6755f056)
+![image](https://github.com/user-attachments/assets/e18f8d56-522e-4201-8eab-66fd3f90f863)
+![image](https://github.com/user-attachments/assets/f2e1c06a-7904-4138-9fb9-6f962dd3887e)
+
+**Lecture 12: QA**
+
+![image](https://github.com/user-attachments/assets/d11dc6ec-6aa6-48dc-a572-4443899fc399)
+![image](https://github.com/user-attachments/assets/cee970f3-3f8b-4218-9c9e-d1c5d791a947)
+![image](https://github.com/user-attachments/assets/550b2a52-4d0f-40bd-88ad-8d925ee7448f)
+![image](https://github.com/user-attachments/assets/2a76e00c-bd4b-4c09-873d-71d31e645ff2)
+![image](https://github.com/user-attachments/assets/452ee6d7-0ed1-481f-ae55-8bba77d36d64)
+![image](https://github.com/user-attachments/assets/5d2127a0-cfbd-4dcf-be37-b67f9c6f1f34)
+![image](https://github.com/user-attachments/assets/f1c4d8b9-eb06-4d86-835b-95a1486a8792)
+
+
+That is all for today!
+
+See you tomorrow :)

From 511fa141b1d2199260af83b86db888e97f0b56ac Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:53:24 +0900
Subject: [PATCH 162/638] Create 2024-3-5-day-64.md

---
 _posts/2024-3-5-day-64.md | 58 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 _posts/2024-3-5-day-64.md

diff --git a/_posts/2024-3-5-day-64.md b/_posts/2024-3-5-day-64.md
new file mode 100644
index 000000000..4638bd042
--- /dev/null
+++ b/_posts/2024-3-5-day-64.md
@@ -0,0 +1,58 @@
+---
+layout: post
+title: (Day 64) Stanford CS224N (NLP with DL) - Coreference resolution, Adding knowledge to LMs, Code generation
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 64!
+A quick summary of today:
+* Took some notes from assignment 4 on Natural Machine Translation with RNNs
+* Covered [Lecture 13](https://youtu.be/FFRnDRcbQQU?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) Coreference Resolution
+* [Lecture 14](https://youtu.be/y68RJVfGoto?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) Adding knowledge to LMs
+* [Lecture 15](https://youtu.be/JlK46EzImM8?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) Code generation
+
+**Notes on NMT with RNNs**
+
+![image](https://github.com/user-attachments/assets/45e28b4c-6124-4f5d-bb39-319848f86047)
+![image](https://github.com/user-attachments/assets/e1465a33-f6d8-4261-a7cd-4ce5e859582c)
+![image](https://github.com/user-attachments/assets/e82315cc-4e0c-4330-8654-8122770367eb)
+![image](https://github.com/user-attachments/assets/dcc30da0-cd3d-4a6f-81d7-efa5a7083214)
+
+**Lecture 13 Coreference Resolution**
+
+![image](https://github.com/user-attachments/assets/8a47a98a-44a7-4ccc-bd5b-0ccd88710947)
+![image](https://github.com/user-attachments/assets/3eec8776-4521-4aa7-9659-25f47c8b35bc)
+![image](https://github.com/user-attachments/assets/ad32caa0-fda6-4575-a945-a6ae14cbd323)
+![image](https://github.com/user-attachments/assets/7912f693-f591-4fd0-b59b-42f3d6d12f4d)
+![image](https://github.com/user-attachments/assets/aa4fe031-59e3-4967-8d0f-4a48980d851d)
+
+**Lecture 14 Adding knowledge to LMs**
+
+![image](https://github.com/user-attachments/assets/92362e98-903b-44bf-a2f0-974830aa6594)
+![image](https://github.com/user-attachments/assets/722d966a-b1b1-4a22-99c6-0b07b9666bc4)
+![image](https://github.com/user-attachments/assets/935a3cdc-6942-4ae6-99c7-25f2f19400fd)
+![image](https://github.com/user-attachments/assets/a6f61c92-2672-4acb-bb4f-a32f41bdacad)
+![image](https://github.com/user-attachments/assets/6a710047-24aa-4339-86af-a172cc474df9)
+![image](https://github.com/user-attachments/assets/0ef74683-4e55-4d28-a714-40ffc70a88e6)
+![image](https://github.com/user-attachments/assets/253f9fd9-2e77-4def-b9cb-52d80e4e2be3)
+![image](https://github.com/user-attachments/assets/6ce84a7a-b4f4-4a94-86d4-14902b8dd67d)
+
+**Lecture 15 Code generation**
+
+![image](https://github.com/user-attachments/assets/74df8bf6-102d-4229-9c8b-e1bcc0882fc8)
+![image](https://github.com/user-attachments/assets/0c9203ca-430f-48c0-b69a-3868ab2d3621)
+![image](https://github.com/user-attachments/assets/4fbf351b-4483-4c76-9473-a8ad55168cf1)
+![image](https://github.com/user-attachments/assets/027481aa-5404-441e-a3bb-80d798aeaa0b)
+![image](https://github.com/user-attachments/assets/38e687b6-aa97-45a5-bf15-4525277b308e)
+![image](https://github.com/user-attachments/assets/ba6fa98f-9646-4b11-9639-b27e8e7d571c)
+![image](https://github.com/user-attachments/assets/b2926f21-e8fe-42f6-978e-91cd9b2310bf)
+![image](https://github.com/user-attachments/assets/d69fae12-a4fb-4aad-98b4-0300b77129df)
+
+
+Tomorrow we continue! :) 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 5e24733879b6f8298f68dba24e5eeb6c778a2448 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 22:58:42 +0900
Subject: [PATCH 163/638] Create 2024-3-6-day-65.md

---
 _posts/2024-3-6-day-65.md | 58 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 _posts/2024-3-6-day-65.md

diff --git a/_posts/2024-3-6-day-65.md b/_posts/2024-3-6-day-65.md
new file mode 100644
index 000000000..47c9d77bf
--- /dev/null
+++ b/_posts/2024-3-6-day-65.md
@@ -0,0 +1,58 @@
+---
+layout: post
+title: (Day 65) Stanford CS224N (NLP with DL) - Multimodal DL and Model analysis and explanation
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 65!
+A quick summary of today:
+* [Lecture 16](https://youtu.be/5vfIT5LOkR0?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Multimodal deep learning
+* [Lecture 17](https://youtu.be/f_qmSSBWV_E?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Model analysis and explanation
+* (just watched) Lecture [18](https://youtu.be/2t7Q9WVUaf8?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) and [19](https://youtu.be/cd3pRpEtjLs?list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4): Future of NLP and Model Interpretability & Editing
+
+First I will share my notes, and then just a quick summary and thoughts on the course.
+
+**Lecture 16: Multimodal deep learning**
+
+![image](https://github.com/user-attachments/assets/eb163233-4173-4231-aedf-4c55bad95445)
+![image](https://github.com/user-attachments/assets/58d4c876-bb55-459e-8adc-3a20272c4cda)
+![image](https://github.com/user-attachments/assets/fb612ca6-8119-4249-a156-00c4607f198a)
+![image](https://github.com/user-attachments/assets/dbc51024-2933-4b93-9569-2865c5a7bba9)
+![image](https://github.com/user-attachments/assets/70939ef3-8093-485b-a4bc-e8dc605f4f2d)
+![image](https://github.com/user-attachments/assets/cb7de133-9884-4494-bbc8-0787ac9fe112)
+![image](https://github.com/user-attachments/assets/8cc6fc3d-6c10-4991-af8d-80190c193bf4)
+![image](https://github.com/user-attachments/assets/8e4274cf-bebd-4b56-9543-f006508241ab)
+![image](https://github.com/user-attachments/assets/6bf6f95c-7f01-472a-b178-5afa26264663)
+
+I  got a lot of research papers to read from this lecture. Two are:
+An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale; and Learning Transferable Visual Models From Natural Language Supervision.
+
+**Lecture 17: Model analysis and explanation**
+
+![image](https://github.com/user-attachments/assets/f988e8f6-1c1f-4d74-9a89-f691b6299eeb)
+![image](https://github.com/user-attachments/assets/a93bbc85-5707-41df-8877-4c603b673f58)
+![image](https://github.com/user-attachments/assets/879c0bc6-7d05-4575-af92-1b4dc6359c5e)
+![image](https://github.com/user-attachments/assets/b539e6bf-10d3-45ba-bf1e-13c93bb2cb77)
+![image](https://github.com/user-attachments/assets/3fdb0218-57ea-46d8-96e7-8b2c214614a6)
+![image](https://github.com/user-attachments/assets/2ff89a4e-876a-4127-a9f9-180b581e8a58)
+![image](https://github.com/user-attachments/assets/e579f5c2-0852-4217-8bf9-98d4b84abe1d)
+![image](https://github.com/user-attachments/assets/b96045a4-8ce3-44e7-bf86-13dd2c0c63fc)
+![image](https://github.com/user-attachments/assets/23baf3aa-209b-4d06-8844-bdaa36de8dbe)
+![image](https://github.com/user-attachments/assets/8d2ec363-7855-41e9-aea4-828e44d303f4)
+
+**Quick summary and thoughts on the course**
+
+![image](https://github.com/user-attachments/assets/9db51c25-ac3d-488d-a25e-edf5c84b47d0)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgNTPcact3KWIxuix3mLLllc5TduuZ2d-9e0UjE-pdDhvefRENGCD6hN0QYlPPA-4obp_WzCgu8I9Ets0SV1txqNa0c9SIzSvlik6EJBSsn6bJphFdjLcRjweMlI3DczvBfaHRH1a0kRGmsDk1MvLkSJg6s-7sMZK4Jb4b9zdd5pGozapOQXroO-rsqq3W3)
+
+I wrote a lot of notes, and I am glad - writing something down with my hand definitely helps me remember it better.
+
+Definitely a very comprehensive course and high quality, and I am glad I decided to commit to doing it. Going back to the start of NLP, through human languages > word vectors > word2vec > seq2seq > RNNs > LSTMs > Transformers, but also learning code generation, natural language generation, how LLMs word - pretraining, finetuning, evaluation metrics. Amazing.  
+
+Maybe someday I might pay for the XCS224N if I have the money, so that I can see lectures about *current* advances in the field. But this course (half of which was from 2021, half from 2023) definitely helped me create a sturd base for my future NLP journey. Thank you to Professor Chris Manning and head TA John Hewitt - they were top-notch ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 8202b1f663feadff5cbebe866f4b88f34967fd86 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 23:13:43 +0900
Subject: [PATCH 164/638] Create 2024-3-7-day-66.md

---
 _posts/2024-3-7-day-66.md | 148 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100644 _posts/2024-3-7-day-66.md

diff --git a/_posts/2024-3-7-day-66.md b/_posts/2024-3-7-day-66.md
new file mode 100644
index 000000000..35f5e3506
--- /dev/null
+++ b/_posts/2024-3-7-day-66.md
@@ -0,0 +1,148 @@
+---
+layout: post
+title: (Day 66) Starting Build a LLM from scratch by Sebastian Raschka
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 66!
+A quick summary of today:
+* Chapter 1: Understanding LLMs
+* Chapter 2: Working with text data
+
+
+I am in a discord server called DS/ML book club, and one of the organisers is Sophia Yang from MistralAI and I saw she posted about the server reading this book called Build a Large Language Model (From Scratch) by Sebastian Raschka (it is still being written but half of the chapters are available for purchase). Given I just completed a very comprehensive NLP course by Stanford Uni, this felt a good next stop - to test my knowledge and understanding of language models and how they work. 
+
+So, below is an intro to the book, quick summary of chapter 1, and then a bit more in depth for chapter 2.
+
+The perhaps a bit more interesting part is Chapter 2 (available upon purchase)
+
+The book goes over the life of a LLM
+
+The content follows the below diagram but not all content is written. 
+
+1. Understanding LLMs
+2. Working with text data
+3. Coding attention mechanisms
+4. Implementing a GPT model from scratch to generate text
+5. Pretraining on unlabeled data (not published yet)
+6. Finetuning for text classification (not published yet)
+7. Finetuning with human feedback to follow instructions (not published yet)
+8. Using LLMs in practice (not published yet)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiJUlYATGhKjGiHu9SR1SIWtaCBqupPMUeLgBlvYzFEvqFXEUVjsBh7ZMvnL8d7M8H0iFmCdNLpd1v43D63FBZimAMPiSjNYdDz5bRQLXMBLRi8OteeuhqDOLGsImPy21On94Y2ckOcSqJcUcT3BfxgUdqnve4GzCLWQ4OkSxwQtz_VKKoqotr5BE4jZjEg)
+
+Today I covered chapters 1 and 2, chapter 1 was just a quick introduction, and in chapter 2 I built a GPT tokenizer (similar to Andrej Karpathy's tutorial).
+
+
+**Chapter 1 - Understanding LLMs**
+
+This was mainly an introduction to LLMs, setting up the stage.
+
+Showing us the Transformer architecture
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgr86JGd8S3LYEb-SoqZNj7Zv0_n7_8RCN1wTtwfxBrbuQZ-gx5AlhdM4ta6yOL4xFhIb6YjvXCifKHWNpx-T6SUhlTUQ0VshXDejq0MJYjgMPrgvZhOTbGr2MUZJlnjH6BKA4AOmx8V8657iJb0I59FZroodVDg562ci7542EPUhmbA6vhjKyNGrElkecV)
+
+Types of prompting for LLMs
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjTmCWv5vQKJRk1FF5Il2tpvsCUdlD2ZOZoKBalAtszceaVEL47WORQe7JvZu-CVWeygzXNbQAcUpHWRfXa14KvWZ9C6UchYlIea52D0WjQeHRrxJ6qKwDxToj1PgZ25uP73xz_oy1KwgidqpFwNK4evOpe0scJxmyImgEw3nn_r6tZbntfqqp6mgg6wQpr)
+
+What kind of data was used for GPT-3
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgNhkDKHqIX-IadCe3-Z2Tf-CColdD4KG6w5Isq9KvTvIOUgrHsKBcBJ4-EZ42x5B-lG4M3EsgKzj6fqFpuheYdihlfDNp5cXhN0PJDGbRI2at3iFbG6U909DSgBeJ1PYDYtsPcBCY4fFCpv2cJ1xTaahabi9X-YX6ju-PUzdR5SSu1EnaLARbkArIPntz6)
+
+And how text generation works
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEitNezkffNVaNK2N9k_rf2Ld7b-5bZd8T3GyL-6rDZfHfEPpfVU1Bel7ePnfu9M-OqhMXUHMDZq7Jsz5fs81X9P4dN8_ibbghgcPwdIVMPLDPwCIyNZhGfefD9_FXC7JWSA06cGfhmf-S8nB0NkzI7YhboafWIyk0hgoZvJIkCCPZ4EYEe0dZMBmN2i6_ws)
+
+**Chapter 2 - Working with Text Data**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhvZuYquxK-HmXK5JSmwWTZH1G8E5jTkOjwm_1kZNgqopgv5yZe99KLYyaK8BsAR7D9zezGXpZTcbu0PEsQMpS3SMVGDBLw-fPy7Kv-15bOOb-qacdyFlLLommeQt6NsyFO7FnRsLIhEaTsWskagr0fH3r7plCla49OBUyK0BxvgEBloMFDRPagX7dAWyqT)
+
+Training LLMs with huge amounts of text, just by using next-word predictions can result in surprisingly good results, but we can achieve more by finetuning them. 
+
+2.1 Understanding word embeddings
+
+DNNs cannot process raw text as is, so we need to find a way to turn this text into numbers so that it could be understood. That process for turning raw text/video/audio into continuous number vectors (a format that the DNNs can understand) can be called embedding. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjJ0cGy6JgIw6KfV9ifcpak3xIUt6fOLPOknsOYZhe7GgasDRmN8c49pV3KvS4Fg-EcSWnWhP0trZHG3z5uUyqXJXqE-p0-Y87v_9Qlu6M-edF8_bIY-8yy9lbvv7LSOkzoxhQ5wMeP5nCu1gQh7yZgZbM4ZDr7DO0yMGpVnTisnTGn5FbnpUoKlnqK7Q-k)
+
+Earliest algorithm that got popular is word2vec and the main idea behind it is that words that appear in similar context, tend to have similar meanings too. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhXWpVXVJM_1wDBiHeoCA_qTWU9Gu12xKgCThOmS9s7OMJYPLgV9nUFXGjpZTu7lEsZYtEn6c991vizTaKaEyxrMWedIHvB9Y8NeRXRE64krWkKaxGNZWJFJPJjOK4_vTqJ6wCD2HQTXzZnMesiXmx119eb17un0N3jJfzpS-QDwg45B6TSoFY9CZS52c_H)
+
+In the above, the word embeddings have only 2 dimensions, but the dimensions number is a hyperparameter that can be tuned. 
+
+2.2 Tokenizing text
+
+Splitting the input text into individual tokens is a crucial step in creating embeddings for an LLM. These tokens can either be words, subwords, characters, punctuation, special characters.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjQ3K-AmNWEls5hR2xDq81ShG9DTrYHN4DxfJrIQH3GssnDgTfw0Jq4qy4i7L6e-I0nRe4HesH1DwwGjweKWk3fOUM6mLVk9_Ee6XJvEY9ljtPxGyUSq-T1T5sZ2-qUCkh35odcJrrYfrZVIXY8IfFPQ9c7xevgqWT0co0-oHySuXuMciA6tUejMUIbArz7)
+
+2.3 Converting tokens into token IDs
+
+To map these tokens to token IDs, we first need to build a vocabulary that tells us how to map each unique word/character to a unique integer.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjopR_w4sjJ0t1f0QPCoJkaN8TuPA8Gm2Nz1c7UqcJuN-K639tE2iaRr7Cqyph5mpKjsuJP1GRb7E2UGnfrDo4wkvRGrGL2aBHDhmR-0s5DHw2Is7wmw9dSPqFaUDV7gDpgX0i80_eU7Y_raXvVIrcfDu0WwcxXrCP-HsfBmwE_KaOEkvd2uTvImf2_YIck)
+
+Using a simple tokenizer like:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgmIVFwZoBYrlQXKYptmJKu2Cu0u75D6LVVjcclICfEos5qDMcVEogSX-ayXp3lWjALlDo0lxf1noW5a9zAF1h32kEXxCgeBNx_BwXvXAs54pdIfOteE04dYNlWOMZdBp8X72cgKm_cqg9PRAifM7UhKBV7QIKy7hp0Vslyg5uHmptz_l73SqbqEnQle11a)
+
+we can encode and decode the a piece of text. 
+
+2.4 Adding special context tokens
+
+If we encode a piece of text x1 to x10000, and then we pass it a new text to encode which contains a word that was not in the original vocabulary, we will get an error, because that new word is not in the original vocabulary. To go around this, we can add special tokens like <|unk|> (there are others like <|bos|>, <|eos|>, <|endoftext|>, <|pad|>)
+However, the GPT tokenizer does not use the unk token. Instead, it uses a byte pair encoding tokenizer which breaks down words into subwords. 
+
+2.5 Byte pair encoding
+
+BPE (a more complicated tokenization method) was used to train GPT-2, GPT-3, and ChatGPT. (looked into how this works in depth on Day 55 with Andrej Karpathy's tutorial)
+
+2.6 Data sampling with a sliding window
+
+Now that the corpus is tokenized into integer token IDs, input-target pairs can be created for the LLM.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjhtVXh8uu8pOqOTIb_Y11tsShzMGd2w_HZ2ZecPysczmZMX1UV_HbdC6opU6e2pRXSBvEXyqE5auk66pysYkFVJPjRWLEVpUT0mtLEFsFWKoDRyg43mmdXvACFJCTkudkib1HjhHs5HKoWLmjlYZu3CKVjNjrO9T6LUu8t24nxrkfwB_rP2QD-3E7_wfO6)
+
+The input is blue, red is the target, and we can use a sliding window approach.
+
+2.7 Creating token embeddings
+
+The last step is to convert the token IDs into embedding vectors. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjaTWYUc_1yjdw2KkfUR4GE53wtp2AZkolgvzSxWNxWNdL5-aH4IOaajDhco5r__v-SSti8ofiLQ24WbY-oNzaQGbQxESaT-veN6iQXBjmbRwotPuC-wPgLu2dqJOKagInAMwn2ClkmzooINyYIGxmFDLEESCXbzu3bKdaEFe6_ddyjzXeqWsYpIA3is81E)
+
+Transforming the token IDs to continuous vectors (embeddings) is necessary for backprop.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi7DOLAS8J7GyyYR4RCcpvF9qlCdAcQWGykiAf8NFB_imAg5LleygT9lhwn65ea-nEQ_HxMHc-Xu4vDCJYriKMt5kLmApthEJ8bdHtQ6PLMvZVasm-PYfHd3MBFtlANGuqgisIY36Ri6dBnQ9qBXXVclSouWibWBaBoiB0LVncs45IqauzaOiHTsvBJrpqj)
+
+Also, thanks to a [side note](https://github.com/rasbt/LLMs-from-scratch/blob/main/ch02/03_bonus_embedding-vs-matmul/embeddings-and-linear-layers.ipynb) in the book I learned that embedding layers do the same as linnear layers on one-hot encoded representations but more efficiently. The achieve the same - given an index, look it up in an matrix and return the look up entries by that index. 
+
+2.8 Encoding word positions
+
+Token IDs converted to embeddings are good for LLMs. But not so much for Transformers and self-attention because these embeddings do not have a notion of position, so if a token appears twice or more in a sentence, it's embedding vector will be the same.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjRyybpDblxFUnYFKmeEB_SXc0m5VqPCvrTQKCRv9eZ8y27JmE4X1IAHnQ0mSvWPm_ZlOmbqhRIkXquvuG7UMmPpJLSKTytp6oqYEUGmw-KmaUvBSxCxMYwNancmQD6TZBhTXIbYtNKDo1s-SptdfcykDhfyYkmgs88stXY9MQioOA8bkLYM0nJ_RSFmFEV)
+
+To make the embeddings 'position-aware', there are two categories: positional embeddings and absolute positional embeddings. 
+
+'Absolute positional embeddings are directly associated with specific positions in a sequence. For each position in the input sequence, a unique embedding is added to the token's embedding to convey its exact location.'
+'...the emphasis of relative positional embeddings is on the relative position or distance between tokens. This means the model learns the relationships in terms of "how far apart" rather than "at which exact position."
+
+The second one is better since the model can generalize well to sequences of different lengths. 
+
+Both work, and often it depends on the type of application being built.
+OpenAI's GPT models use absolute positional embeddings that are optimized during training, and are not fixed or predefined like in the original Transformer. 
+
+Here is a link to a [colab](https://colab.research.google.com/drive/1J174LGhm2n42h20TNqGF4zos4Z18Nh6N?usp=drive_link) where I practiced following the book.
+
+Side note about something different. Yesterday I learned about probing from CS224N, but I was not exactly sure what it does, and I saw it in the Learning Transferable Visual Models From Natural Language Supervision and An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale papers and wanted to understand what it is about. Found this [youtube video](https://youtu.be/HJn-OTNLnoE) and a John Hewitt [blogpost](https://nlp.stanford.edu/~johnhew/interpreting-probes.html) that explain it pretty well. 
+
+Probing is basically, testing our models and seeing what kind of features are learned at some stage of the model. By examining the internal representations of the model, researchers can gain insights into how the model processes and understands the input data and also identify features that we are interested in. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 3e70dec64bac4e600944826735a2de94a0fec411 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 23:18:07 +0900
Subject: [PATCH 165/638] Create 2024-3-8-day-67.md

---
 _posts/2024-3-8-day-67.md | 95 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)
 create mode 100644 _posts/2024-3-8-day-67.md

diff --git a/_posts/2024-3-8-day-67.md b/_posts/2024-3-8-day-67.md
new file mode 100644
index 000000000..088795071
--- /dev/null
+++ b/_posts/2024-3-8-day-67.md
@@ -0,0 +1,95 @@
+---
+layout: post
+title: (Day 67) Build a LLM from scratch chapter 3 - self-attention from scratch
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 67!
+A quick summary of today:
+* Covered chapter 3 of [Build a LLM from scratch](https://www.manning.com/books/build-a-large-language-model-from-scratch) by Sebastian Raschka
+
+Below are just notes with general overview, but the bigger part - with all the code implementations, developing each self-attention part from scratch is on [my github repo](https://github.com/divakaivan/llm-from-scratch-learn). I uploaded yesterday's chapter's content too as well.
+
+In this chapter, the attention mechanism is explored in depth.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhGJBs0uVu_zXeX5AaFHDX4dcKZbyyiNUDkeZqkRo4uQWKrSIi5dF3FGUkNLH6_OBFd34HG_1QTTRfpja7jdVrQimng-dijYcdQA_ZW44VJcF3x5AuLmN7i0nG1eB9kghOQbIKcfJdG4dV8LwZeQxHr-3p8_px1ub-lN_LXhwG4AZO06efjnw3LN5KTmmgJ)
+
+Attention can be split into 4 parts that build on each other.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhveKSyaICVtTYPWEr9ZE4DFrEPiHES7VmEGXeeZDLUD1FRGt7SuOOkMUnz9o8rHd8fRcmfg-4SaXzHl6beE4gHVjcrudz8iabSab9F2lJPmCLfKmjVFebyfGcd2DPrbtYXtvIP5w4b9i5D7-jpMnIrLHCcW7Jj8DGZQ8TpX5pZvJGLZ1H6qawVAwFB4sXo)
+
+3.1 The problem with modeling long sequences
+
+Imagine we want to develop a machine translation model - in different languages word order might be different (even if it follows the same verb object subject format). When translating we would want our model to see words that appear earlier/later, and not translate word by word because that might result in something incorrect.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhmUH155iTjvqZ1B3-9eA4v7XFIYLPw0s9hpZRs3t2YQFU1w1AKzU2Db6nbYAPLToMaykAc7rWgu0khrAd9J0Rb63pHSmtzbrv6Ij8P0abscAANFnMLin4mRhWFt49B54gAdS36CCnrukpg4cicyGsV4n5Wa7YuL2lvUHaL5w_jHPTGDvUZwmyUCu1GU8Yi)
+
+Before Transformers, there were RNNs but the problem with them was that before the info is passed to the decoder, all that information is stored in the final hidden state of the encoder. After each token in a sentence the hidden state is computed sequentially, and by the time we get to the end, some of the information from the start might have been lost (RNNs have a struggle with capturing long-distance dependencies). To solve this problem, attention was designed.
+
+3.2 Capturing data dependencies with attention mechanisms
+
+In 2014, a Bahdanau attention was developed to help RNNs to selectively access different parts of the input at each decoding step. Below is a general idea of how attention works:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiJVPNyemxTkuX1NjhTl2NAg0cZa2Z1hyCUaD0JMBua0Zcccz2JQFs6HduCrGEu_SwBQdfasDwzy1zh4_rknMPMuUoqmw00fuPrtlXphc7OzQPiLQKFC_9HJyx_cYvRqasbft0HNbb8jPvoB9E7cJLqvKIns8m7_RKprSaoXa44OIqtPmEDTpgjANPjLPM-)
+
+Three years later, the original transformer architecture that uses self-attention was proposed. Self-attention allows each token in the input to attend to all other positions in the input when computing the representation for that particular sequence. 
+
+3.3 Attending to different parts of the input with self-attention
+
+The traditional attention focuses on relationships between elements of 2 sequences, like in seq2seq where the attention might be between an input and output sequence. 
+
+However, the self-attention mechanism refers to its ability to compute attention weights by learning the relationship between different positions within a single input. This allows the model to determine the importance of each token in the input sequence w.r.t. every other token in the same sequence.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgoJgzEdmT7GfRrGIUq06LVYe98K6_5vxg35FEdxA2sJ1ZBRS1S4cjd4HbV_Jtj3l8xXHfA2DkfHSgRwzwp6RM3ccFUauK9H6eg8FMkq492BVvz62OqtkMbkuu5xzKA53SmYpj7espceisFPPnKNbddSece0uwnvecbbHUGxTzfY6if37Hx-EV0PtQDkDPr)
+
+3.4 Implementing self-attention with trainable weights
+
+The self-attention used in GPT models and other popular LLMs is called scaled dot-product attention. This is the second step in the self-attention map.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj50_swyBipiNEn55L8nbPtaKEcI61F0KewExrj4ckO9YtvzlDKpVIxmXzdeZvBXCe5L5TDtm9BuHGobPxAMxemYkDbojYG_EXI9TP6cXNKY54QV8FsKHsu69GyCj71OXVk8oDj8E7-KAinbfZ30Srll-tOK462upyQSdeFghbAYc0-9HvDLrVjqqiAzAmB)
+
+The biggest difference is that here, weight matrices are introduced that are updated in training time, and are crucial for the model to learn to produce good context vectors. 
+
+In code:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg0CbFudVAhks0tLQbTpIyoazho7FFI8Zt4mGaGLxCKvV2j3GJeRkXEHCqbCHN6dwEGRDX9elmXQiDvKCAkMFxLW-sup7kSPFbRWmSJTxZau90fN4w4QWodfQWLliZOHzzzMtd-jGqbvVmVwtyvzcGeR5WbX2FId88kq52nodY6nacZmSG-jKdx80fXypS6)
+
+3.5 Hiding future words with causal attention
+
+The next step in achieving the self-attention used in GPT-like models is to create a causal attention machanism (a special form of self-attention). This is also known as masked attention, where while processing the current token, the model can see only previous and current inputs from a sequence (stopping it from seeing the future).
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjlR-sl5HohiGL1YfDgmjS10y1pdwOhHguvl5bIsuK5HTTf7MoycOyKpH3549ehEH2PQs01-WWx3djph9Og0gw50jfOsvIF3ecmkrS8b_zLLgSfnPSPq-ePWgBi4a5KxDLbvBDk3g4ObqNv_KK5hdoo5If26fbvVfnhBuyYLxFT4YGQh_axYoKqfVWIBAfI)
+
+This is the 3rd step in the self-attention path:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhPS7WsdXiKBQihI3HhimYySz__uwllTytF-uKpZKRTtdcmDeIDaFB_bk_ZBJYKsuyrzfQdstLrUmZ2NCH7EaRFAWPZKKNcjik13eKPifF9cW61qyz7ivh5b-t0FwTko6Q3nUpNSnlUzqk7bgEYPak_CDdlSa2E1KUhY2eNtgwO75FfqaxD-Q36cUEvlr3l)
+
+In code:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjrEG3IleCMSa_iIL0JXE0U6O_gy4v1TZLqHIsQGrfAfWQ5ii0hUhDKBugyNJfKLPC043AJ8nW30nL_vVIleH4NeBzc4PKnsQcs3pfNKo7558J_0drRyqyt_lbYQzAdJgUTMZxULfYLFuOWAENBZyBnrO4Z9CBLtQURF1sCmGxucgn8WlALF7-fXhBYsdUK)
+
+3.6 Extending single-head attention to multi-head attention
+
+Extending the previously developed causal attention to multiple heads. 'Multi-head' refers to dividing the model's attention into multiple heads, each performing 'magic' independently. 
+The main idea is to run multiple attentions in parallel, using different linear projections, and each head learning different aspects of our data.
+
+Idea:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjnmwTcPdmwFJNx7jpA3DV1E-V139NQ-bgFxoJrFk9mxBmU1z508yvJurln8v_AWvjolm13PdBbsO7HFoJCT6u_rVsO-J4jOBC3vQx1eQ8JIlNx8FfwSNkJJIQzOXQirFUGW8A7bxOD63VEYF8PdCFIGhN-RSszU49oYHkjA37yKWc5_c5hxEVlgA4LcKWc)
+
+Execution (2 pics):
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh4wGJzB3ulAck7T7RQTU-vRdXxn8iDuX1LicvPtisHmYGu0YueilZCqCc9ef52Gigv0yd77hfvBYvFgHH2dpM_6rupj-PoCGYWWBzQtH4K3LJIJLsMRUKJx15mPamRBGy6SA6Rfs2j-ATUhPpn0z4kmcx-j2fRGhqw4UBj_jzR2If_4PmzxmHIwh51Zoh-)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjqPhXpoh9pjx1bFoerfKALWbvm6BQ7VXNf_5QtKBeP1SHHRqRQaqJ6NnzK9_jz4nt5v2JoZv98cPlKOPZYkB-7Z3GmGC8bFXiOjrFeQX8FTLOPmw7Hw64HcUq5VjvnfXngpKZzyeTBOKHC2FfYSwFZr810FImM5z8fGHw4JtfAK0IWbIO9CXH0MMGDiIrp)
+
+Creating the GPT-2 multi head attention with this class:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhpLZP5O0y_57lKhqR_QziYGtitUAgN1OP7s4SW0uGOAVKCCe_qN7b2B2Yz-LPk27irOAc1XhUqtYrW9O3SnPFyWCFEgD7qfsh8sxZFyI-0DctqUb10RWQIjIzrJYzBJyS7-lxOqZDAAsVUn0RxqAG6QOuTS_fvdgtmEQnHnCCV2-QRRMJoX5Xg460Dpy5F)
+
+The resulting params count is 2.36m, but the GPT-2 model has 117m, which means that most of the parameters come from some other place ~ (we will see later)
+
+
+That is all for today!
+
+See you tomorrow :)

From 106033f49d93a98b04edccb0cb77b50f4c396577 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 23:26:26 +0900
Subject: [PATCH 166/638] Create 2024-3-9-day-68.md

---
 _posts/2024-3-9-day-68.md | 145 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100644 _posts/2024-3-9-day-68.md

diff --git a/_posts/2024-3-9-day-68.md b/_posts/2024-3-9-day-68.md
new file mode 100644
index 000000000..475b0eef2
--- /dev/null
+++ b/_posts/2024-3-9-day-68.md
@@ -0,0 +1,145 @@
+---
+layout: post
+title: (Day 68) Build a LLM from scratch chapter 4 - making the GPT-2 architecture
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 68!
+A quick summary of today:
+* Covered chapter 4 of Build a LLM from scratch by Sebastian Raschka
+
+Below is an overview of the content with not much code. For the full code version of every step - it is on [this](https://github.com/divakaivan/llm-from-scratch-learn) github repo.
+
+This chapter is the 3rd and final step from the 1st state towards a LLM
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh083ca3PLGJXQf0tX8L7SKA7c-mw5W5QRYPh2OwYeQ4han2f1WbvDesrs7-MQqL8qtA0Hf9oFBJfnDAb5iwTVjA3Dbbuf0-C4wo4UhccdQibuZxg5AEzKRn6AF1LBq4Y5gPQay5YwpCg0FtUZ-RcovPWpdz2UhcoQ8EU6XprrnhQ4Q6gaTjxxPBBfA8bKD)
+
+4.1 Coding a LLM architecture
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjQ8JZ5Bg0POgM2JSEii35gfR-1lFYdMGGsxwgxkED7AbppaVr3IbI2gcQXvjqGXfxXJCcx-fx2fK-EI3rQs46UNIBy38IN8uM2v2heqk_z3iHzU9RYbVnOeaKjdzNolkKZkav8VE-_R934JvG7Wx3zlmlTUMB5Fn1ZewuQqsp8xtjqMkTDkjzfV8FA7HQU)
+
+The book will build the smallest version of GPT-2 that has 124m parameters, with the below config.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiinHU-6kzekvfREPbDXhw_BEiiUz5n_W6ZAmqd8KDsbQ4f6-nQdcTdTUFZB5UO4HHP6p2h5zOEWFXWlkZpABT-9lf0glcujJkWs3_TYYsHTVGJ5ErViHtcjTasFRNlgsF7SRsc5lSAoJVuJJ9YRlijtQlLkQuPOLQpIJZKbJuoiZZ3qD10d1yM8cEwFUHR)
+
+The final architecture is a combination of a few steps, presented below
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiyRhKkjWyiyoKaFT5kES9z7l5tEoxq0BeyDRWS4BiPI5SmB8ljThIMNcjQyr9hnDxhUHikwUrLMqazZU4RUa234fX3ifTJ9Oz_s6BXJ4HAC5zlMqjDfi-PKZhnWTTD3xHMnjWHMeWUOdox3BEfVOJWVC3ytvL_9lTw6N6aKIoLHvViQCFJSPxD2EbuYSQT)
+
+After creating and initializing the model, and the gpt-2 tokenizer, on a batch of 2 sentences: 'Every effort moves you' and 'Every day holds a', the output is:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhIa0ggAQVvrVVrUNI7UZ9-aJe2FQ6ex986WTTxz6Ce2-yTG6YuJvlD0AosKQbswW70c28H8vrapDtsHlQY0qyCowRrH3dZ-lYMKH2DvuSy_Q9xt3tiIjSv_agZGg_i8pb49mWQ39OIgoU2puypvlFi-65A5KjtFSEBvFVj5OdLJc22boMmJoEsFfHAteYD)
+
+4.2 Normalizing activations with layer normalization
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjQsSay9f-EePovz3zpITzse3sxRPmY-6zQvGtHGwemToQk2z3KrhjWS7MEjk6wLE3Fm0XyCA5Ep5RChZO57lcOgNCYnmMh4xV4qVoGr5hbg7nT1T0RfXVhycKAaEZFC3XFnR0wmou0LG7nbLSdaDmL20HrAnYlDhInzwObyiLvs4u0yRji-b7-sadBI62i)
+
+Taking an example without layer norm
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgdVHnLqywStJLP5LK7ISzOAivar6y-x2xz8iI42xcCiLSAIlSF2xIbehFlE8_E8Ko6bcVYYWV5mF0utSDEpU_kqOqohuCNOXTJYE9-0k7LTAqBujvLuuq2xMKn6yZsA6uT6YrcbRBkZ5fKUrJsVQkGpbwn2Unb1arA0qhyLnKEweaNpqtYfHAA_ZmK6ze1)
+
+The mean and var are
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj904D55YAoE-n_rJARO_7_F7mS9KNk3ATcwDZLNLJhR2dB7OhAzmLH8ZdVooeMkk2tc-yezuZeZeyGnCpO1i69D8NWmMb5G5Zy4kphhlwCBNmHOOsK6KBGN-y_FJ5tHjPA3hHzzioa_78GpFXd-4qKkMoZtd2nZM-74jDRJKRvZaMKhulGkYFXV--DFWWc)
+
+Apply layer norm, and the data has 0 mean and unit variance.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjMvAFcYA_TA1Q14f-_X0FKAGKtdkTHu_rgX9XgoVCY2SkfwTmI1CFQUoLx_jm0eIV9Sx2SzXTXiis4OMsw5yePO-cwZQSnx9C_hOCoKvMlv8EjYFEvSIb1mHf1drSqEVG1GRpbMeI16mvEt4Iybkv3G-oZzWd3vM97Kwg2QT4bOyuP0Q6a5Stdl0cTeODB)
+
+We can put the code in a proper class to be used later for the GPT model
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjwYVodOjJ9Eq91dn6-eqKV-At9ov4UZwQvgP3koPRGYkOzjb5wGEbhVSLd_tiSBixtQo3dxNUWHUXFCdArt6nhL3pRo60Bbo-zJHCz4K0j0GwDo9O3SdXeoMd6hfvmImgZNCe7AWFGovnaSvLRkzl20Yu-va_lovgz46rj3SX7OeuzfeRVowD8eVThjNuJ)
+
+4.3 Implementing a feed forward network with GELU activations
+
+ReLU is a versatile activation function due to its simplicity and effectiveness is various NN architectures. But in LLMs, several others are used, including GELU (Gaussian Error Linear Unit) and SwiGLU (Sigmoid-Weighted Linear Unit). These 2 are a bit more complex and incorporate Gaussian and sigmoid-gated linear units, respectively.
+
+GELU's exact formula is GELU(x)=x⋅Φ(x), where Φ(x) is the cumulative distribution function of the standard Gaussian distribution, but in practice and in the original GPT-2 model, the below approx was used
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg8IUXGJHqpfH7WMkHSipcXBYm9cbfom8rgfvm0mxtAgr4jC0DY-6UJDh13PZYqKOJ430LbkgtBhRSwZkINuINzb5jV1A-c9cWHiG6i6TFKGEnxsLskmAg36i6QZY5w6n5r0FP91XlNPBaiVvIlx1JASuJ3P7dG4pglayXXa3upWaim-47Y1MhQDk50sOhk)
+
+GELU vs ReLU
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhAaa7rFnk0R98aRZL6mrF30-GOGTM3ipTN-ZsMhW9IbELCmpW5cSVIJyQtn94FZ73_lDPiBxsA_-bCxi6c-t9f1hCW1R3GbNoUzLSWX5xq9I4kogv0SqgkkHvz0usCEr8OY4lpJgZOw3-GgBY44OFMYYEvVtcQoaKiUlX-2PCRc2PH_JYxY5PTz0wBYi72)
+
+Compared to ReLU, GELU is a smooth, non-linear function but with a non-zero gradient for negative values. This smoothness around the 0 (unlike ReLU) can lead to better optimization during training, as it allows for more nuanced adjustments to the model's parameters and neurons that get negative values can still contribute to learning.
+
+Using the GELU, we can then create the feed forward (FF) net. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg3P3LsG0fcpbl9HMRJ9_zi08xTdAxfXzW22pyKT-q9jevTUBeWsaIIC9wqB8eQyxy5gWUbwZy8wbsgoQAwMwxBskWsfz3yfc1DGTLaPQ0L75nzlezAmSomzaoiYCre-4Y5afKOgc1O6WdB2jIEDyHGvvztRs0DXOg4P7XFyYvY11hv4Ox_eZs29f-kG6KX)
+
+In code:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjhunUD0XGK-HtWKSQUgZmtj8bXhEDlIgOhDy7XppmtYAomsChslQ2F-_aJuVRJoysonN08cZE0XYzYFEtZhuySDRrdNW61ZCMbZqJkBX7T5MkC9Zb0tVjoVBMxzylGwfBc4_dIDWhom3xM8cFHxWp7x9nludTDkcv4DzIuqaQPXWRX8Ndb48DOAJox51wL)
+
+The FF net is important to allow the model the learn our data better, because first we expand the input dims by a factor of 4, pass them through GELU, and then squash them back to the original, after the model learned new info about our data. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiQLGDWcySs1Va2XZpbrrUyMKtui1XXHx1QxWa6awpJqE4jdqvqa17xu2klHXxMZgSvpikxGRAS2GhDPHDkcNfAiHlstK6EXnWjDBcSbyVmC-zvIyrHyC6M8p-25n8ME8ZTOZBFotGlCxporrZcxmDBpWzdL0U4xJ5INhtL8LGcwiLaEIR5Jzs8PbMrdUTT)
+
+Quick check on what we have done so far:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj45nWI9J6bsPtSTowIoWMUrVhmaCoI1BDovYrrqSzNW8UHa4JthszpzN5LwMuzC-xqZkzmiEoqUQMjkMybNpjjI867QhZWDMu24kWb7xX-966EGcRjsf-I3NGvdGT495CO5I4qCSRY_Bnxia7BWWwOy2nf4ld85oEgocZ9POqAKUVwjAxuXvcPp3R_2ky-)
+
+4.4 Adding shortcut (residual) connections
+
+A residual connection creates an easier path for the gradient to flow through the network by skipping layer/s. This is achieved by adding the output of one layer to the output of a later layer.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgpW6FPJ5kN4YL8pg2Pv1G5bLzFWGgEGxLKCyKxRaLXRfWy8hk3-IxQX-StQDzi8X3CzOdYUpCtvckCUwYxv0yQKgEFTfOI7hLNydKm9oGp9GdLagB53BsXnXrM_lz2EfE0OhnhzkhW_9dm7oBdwff50JblZMk2P3dMDzlVtSxGvVbYdDtOxDMZXjdw3xpy)
+
+If we create a model without residual connections like on the left of the graph, if we see the weights of the layers, we can clearly see the vanishing gradient problem.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgO7CusGRZG9k-jJYgleuTcIGk2Bz4SRyO50YPmXRsVpE4UxmaD0NJkHuGk3Ad1qh6y4gPo8EE16D00jLL2mljF2EeKqxoS2Zx-7g-e90MBY67v73gzLIL27mbMzURtLPRpP1UJfg0zc-9ms0f3oU0NX_ETqQ8JrHuEvKMgMmjUS0CKbWfw4V7wQf1OFeB_)
+
+However, if we add skip connections, the gradients still get smaller, but they stabilize.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhPnu09-GTq6ZnagFNv469MzefNDJta20TPc7PP1dILcXaFqk-9qevfS7Sew-3O3MUxtOMBjLbVuAcsMoAT4k0Zl9f5v373QYI6pTkFYDLmr3GLMTG-OgQMgAYja03V7-LkIafcWMtTeV_lBiC7QJDOMPvz-lxFk0loRlN1mLCp9zQzGk-zAkt4aYnRYjtJ)
+
+Now that we have figured out multi-head attention (yesterday's post and chapter), layer norm, GELU, FF network, residual connections figured out, we can combine them into a transformer block.
+
+4.5 Connecting attention and linear layers in a transformer block
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgHeEauHdUn-EzSPJffva6C5dHrFdN15_lj2C-lPGjOBLMg1DMiKTaiJM6AEFzrlvGpDCFcH7NTGCAbkGIE3_FXm3akQuyVZOXM2irciJbRMFbpx8Aqjd9Yxe1uBBXmOFt9c9EHlTIWtIXwKLkoEKfGV3ty3JbPubcFFMBygezGZ-C-zkHNcQoeKcLOFE-G)
+
+When a transformer block is processing an input sequence, each element (word/subword token) is represented as a x-dim vector (in GPT-2 case 768 dimensions). The magic that happens inside is that the self-attention mechanism in the multi-head attention block recognises and learns the relationships between elements of a sequence. Then the FF net learns deeper, more abstract information about the input. 
+
+In code:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjFp_H9aM41ZT7Qt6BsxfLC4qEqT7wZPI22evkynPw_gIERNqWmNd8NIP3xBTmv1EvAc-H7g08N8ikKo_UAvYJIhjlneqToO581BdxrXpI9wMVD8uuxXenaW7eHR0kqJQs14U6-ALCuMXV27GGu4OdRkkSpVK6_EcoWYovzxOmhuGtOK3W_A7zWQzrmB5T0)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhK8RBzXAEVSainFMP3taG5zEtK2akwD0bBBDl-2rx7a8ZSTktodUweC0gd7kwUC25rTvUjEPxq4lckFquogjjMGE5iffUh0Ft4-3oJA-UgzLjgLSxKv1Aj271XEarq8M9tDg7o_Ypo1QeMRaVZYuHdsg30tWqyW1LdDff2VmrdEJ-ZVrzJUuXLcaTtsh2q)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhy96E_ILTvvLUydEqGf53nhruix9RXtHWPeTUwHg-2G6Pc5C7p9l276ZyVD_6jqE6VIURDwFDCl6P6BzqdElRvF9spa2e_76nm7nyw-ZcRcJ8_tizZo_VpVsRq7uatvBX67dvsiMbEP6SbsAhV2mPo2p4Q8oSVSMSZhKryG9-U2CNUxfpH-Imc8N-IUYE5)
+
+4.6 Coding the GPT model
+
+At the start of this chapter we had a dummy GPT model, empty TransformerBlock and LayerNorm
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj2XEeaApNpHqOffVu9npeD45Oz_fb97dKfI56Q_nGwbl-fo-5NkZQHsBycd47diUZG3sYyiENiMEPpEgeigK5OY-6t-xjMufGFFEroJ2ISVIjfeD6p81ljOYI2WpCYKx7RRgB2O9DGXZkMiGFYH6GFFdL7JVjesaRikbYDaFpVIi0c99QZ7j_hH_3Su6en)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiKAKlLvKDVuX6l5q5nJjQswYTGD0H3BKXHSXzhL78JAILV_sfgbeMQgM0ejQNF81GSP901oQNj3CVdyWlbq1xm9cocJO3m_t99odYZk2_OaHcfspb1TkMx9E-G0mSiVhHZsDO7lvc762v9_olF1KHDhVjbus7rlUlww7TprAGzKGsVwef_GSefFluxIjTT)
+
+But now we can plug in the code developed throughout and get a proper GPTModel:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhXXhnupxc01eF6hugaqXYcdadEJWgYhcDpy8nuNWXWzw6UJDilbzOMdqE1rV87kSWBwC_1VRB02P-87hnVXhO-lG7O6zaqGU-vLKN1-QdGBA_5wnLpD7lbMgPikZ4viMNw-fPQzbtwtsw7jhUdP4VlUVU_-NTkOtfoNxRty4oZhY5UcXyKiOvxvmR6sSkb)
+
+Using this we can compare the params of each model and its memory requirements
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiPwh7HT-5uR4wyDBjIMKs7E8cEIMOOvk8mSXS7ROjEYK94xvcuji66QPLj8fVFh54NdoJxnCMJ59bFd6C3-Axw99L5WtC1JU_VKeYjscgztJstgGP5aM7wKZNHa8-QCr6RXmtG9SXoqNq24YpXzodSSfBMSx6l-Q18zx2TGnV5BZhfdTGMWPVQXCufFW20)
+
+4.7 Generating text
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjxtkwSzDc3QR6csCQMYAsppSkfhjK8qKgkw-Lda-53xXMYxcES82kuHC2Jtr0olQR6p8DLG1LyT3ZhSdZVWIncIPQpvpwNl2OuMOFeYIipbuP9C6kdw6Q2g-c__VOEoK2697u86B_EDbxd5PaGzIazF5KL-mS4EA1FO6liKp4kBF4Jr4vUncis1rLZsMM4)
+
+The generation of text involves multiple steps - decoding the output tensors, choosing tokens based on probability and transforming these tokens to normal text that we can read.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjXL9VFlsahWpAWmqGgBM0ynTUycSnPY1DVHbosYm68TOYFOC0xkufR-ck0tuZopkInzqFKIUK-R3dRqqWiZ75PapnuJfQBm_pnF7cKWI_MLEmsQYpDNcdJEG2HXPvBjfGyy_pOEQmmhNPAR4mVQ2jTyDJKJ6mYGrN-YHJiX83t8KOhQBDMrJYLdZZsQiO_)
+
+Example generation, given 'Hi, what is your name?', we get back ' Shane' (with space), but most often than not, we get nonsense. like in the 2nd pic. because the model has not been trained.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiY2N9rfzmoav_3yNI6ixVjVzJivJI4KehlFnEPl_xUZMcbxw3eA71ipVNgGjtYIZ3Vz5MgNsXzkiUxAcnoylSi43E0rMIX8DZyIYSmH6oy5-Lh2W-K5gjVKfPZYpJ_labuLNxhCjea3Wogxe1AuAAT-muuESS63tbWxmrkFJsvmG0Kx5BLi9mYKu2pG5qU)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjiBG4RMx0rRysAY3jFySBJGQwq5vmYBpj4KFJHvG5RX9x4502otI-9Mf9vYEqozynl0Zdcu7YKmNlr8s-6nI3hfGUqsrFW39GEiRJcCEzOrJHIDcHZyimZyVvTBxVOOmyUabYQnAAAyyWXTFZ2YqQJ06lH6b4XgRzj1Xgkj3YSf99frMUEnUIbPl-v14DJ)
+
+At the time of writing this, the book finishes at this point. Tomorrow, I will test the whole architecture, from chapter 2 - tokenizer + dataset creation, to now - the GPTModel, and make a training loop, and use a dataset. Now I saw there are Elon Musk tweet text datasets and Harry Potter. But we will see tomorrow :) 
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From a28ac0f97038b613a2b3d9ef48e1e002689f5963 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 23:31:11 +0900
Subject: [PATCH 167/638] Create 2024-3-10-day-69.md

---
 _posts/2024-3-10-day-69.md | 71 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 _posts/2024-3-10-day-69.md

diff --git a/_posts/2024-3-10-day-69.md b/_posts/2024-3-10-day-69.md
new file mode 100644
index 000000000..1277cf2c9
--- /dev/null
+++ b/_posts/2024-3-10-day-69.md
@@ -0,0 +1,71 @@
+---
+layout: post
+title: (Day 69) Training an LLM to generate Harry Potter text
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 69!
+A quick summary of today:
+* tried to build upon the built LLM from the book from the days before, and write a training loop in the hopes of generating some Harry Potter text ([kaggle notebook](https://www.kaggle.com/code/divakaivan12/llm-from-scratch-harry-potter-text-generator/notebook))
+
+Firstly I will provide pictures of the implementation
+(then share my journey today)
+
+The built transformer is based on this configuration
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgk_XN6e1osZg5TujmB4brtdyElFbpnJXACHZt6E0WuMTEGVk64nVrC8ZARD_yFs8bqZPgrYOhdqXS2dNb9Aq6CwTNPuxawt7VKMppLr9GjqjrbqztdjDHcIzGgss8PTupuYQfyF1YhONvBPS28-vitVROexYOlE4_yAiIpopVYHq9_OULhQbqXiW_9znih)
+
+Dataset is Harry Potter book 1 (Harry Potter and the Philosopher's Stone) text file from kaggle. Used batch size 16, and a train:valid ratio 9:1
+
+Model architecture code:
+
+Multi-head attention
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhdLmtUI7uV1zhCY7qlT7b4QxSzm9T8n-3Orrl7VDXbpOm9obWqMrnQcEKvse5TmDDtVTTq_kChK5s0ntgxnqc2yYfRGKpTVp76rbLQ1tkJk9XBhe6WkvbNjTjyI3GDJhKwLK8AczVlg0PB_EPKwGQ2wLKve37dYZsNKFQqM9qvdq8A3Ogw5X7Nhg14IQ2p)
+
+GELU activation function
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhnuoqw50yT--JAwuPIaVdIhjUI8ho1JmBMoq8umvwCxMDy0gI39MSeACGs3JAa8TQxoStPzDkA8P8JiEgZRbRaRQDbZ9-rFtYtIn2zlmSgujMDkYEIYSYnrEqzE5-QF7ZvW06vyd_1w4O0BQxppyZ52dABcPprTthYx16_5hcy71yBw5YNsGZ_2-f3tOoh)
+
+Feed forward network
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhyAqfHgDI_NbVLpKGHlOAYsuo3UDXOeeDnNu3IXiw4WhFNnQm4PxO6aMXqCwHrJX1zTihkOIeyq2xZYxeaHGecB3dW5r2fU2DVA9BXs_tYakk-wQAaAEGXIF5Qe7u_28SL5rnHc4R17fNsOdJ3lu7-mOPWlqL5um3ONA99lzAOAaLiC9VggmwMIWSBGe9m)
+
+Layer norm
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj7nV91C4JZtK_FueMbrFLvbVxdWeRXqlekZYNxRqmcYF7y5Y9fgVpIevVpFI0o8_MsPS50RaamxXGiZ_4BB5Xmvsw5uFxYj7cGecd2DylNC6sk6KwwBJggiGazWeKhmmi9aS2Isq9v14USWwnqDoJCZ_gf14_oG2sJrJ2wADbOdQfIIjuwJfKnC0juvW89)
+
+Transformer block
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEijR5HpiruhBJRV2nZMh8iLoANTNBWL_Rl-rk_yhfZeeqKAV2VzvfRBlwx4frVNzrzCxCjt-iuz_C_MZ6IoZCovU-47E86aKmkb-Ges3gUPwtLwcX8wytlEPnfz317o8YyTENwYCsED5Oequc_ON5xgsK874_3azCA1VrXZtEi-5g8qOm63oGRs7bZQUz1X)
+
+GPT model (+generate function)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj6rhmhDEnejMsuvrgH6gupybxi4uXaZgxfjb2kyBQYFnkWMpMm5m9hSKevhfIf2sKU_0xDVQdKOrnPIrrhJicgpw9n4YuInWSTAYnAc_Ql0xp8iAjcEfPoZgrbmgjg_TweCpww86yKrEWgxo2x0xx8-Nk2o7KjexQRUS-1_rhV7meXxG-_SuawoiIllda-)
+
+Optimizer: AdamW with learning rate 5e-4 and 0.01 weight decay. Ran for 1000 epochs. 
+
+Final number of model parameters: 1,622,208 million
+
+**Now, about my journey today ~**
+
+The easy part of was the dataset choice - harry potter.
+
+The hard part was choosing hyperparams that would end up in not perfect, but somewhat readable and some logic to it. 
+
+The result is:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEicBIvZYKyRlIsafB5wkXy_cnYBQ3qPFQvHD85MwwuHyh8_mWFvg5Bjl58faMB0G6OC4v5eYnex9pIo5W6r4A7aeruIBFfUHP-c0xXmaBiZIfZooBX2dXipQA7rQfKVPGQBcolJvLW-HXjXb9Pl0JsCx3QcypTp8-dJ3nTRMXHqBgEvevQvCf7nbjhMqUp_)
+
+Initially, I was doing it with just a train_dataloader (no validation) and I found a combination of hyperparams (manual search) that resulted in going to ~0.4 loss and the text was readable. Then I wanted to add validation to my process. That is when, almost every combination of hyperparams (higher learning rates, lower learning rates, embedding dims, context length, playing with the GPT_CONFIG) resulted in exploding valid loss. During my time today, I was running different versions on colab, my laptop (both slow) and on kaggle (fast), and I was getting great training, but lowering and at some point increasing valid loss (also at the points at which the valid loss started increaseing, the traing loss was still high, I remember in some cases still between 3.~6.). Lowering the learning rate was the biggest factor affecting the valid loss jumping after some epochs. The training was slow, so many params and understandable. Of course - a transformer might be an overkill for such a small (~600KB) dataset, but I just wanted to practice, to see the transformer in action. Still lots to learn about the inner workings, definitely using a hyperparam search would help, and also using weight clipping and lr scheduler - which I saw are used with transformers. I used them for 2 runs, but then I removed them, because at that point I was switching between train and train+valid training loop because I thought my code was wrong in how I calculated the valid loss. 
+
+Below is a generated text from another run that is on kaggle too (version 1).
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhI1qlZjtC2lCag4zZ16WOm6Txy61gJBK8Vo5ffPPvefiowNLrbqS-4zdX2U_MqAX5IITgqkElNq-zd19UkcF0kgRgBNmqSv_eRhLaCCwNVIDJuPXMLr8suFuVYKrQD9vDRfc3K77TSULdPvjdC-XykqsHbTeggeUtp8O8LAtdQAXgRX_EvnIrUoQTibXKG)
+
+It is ...something... haha.
+
+
+That is all for today!
+
+See you tomorrow :)

From 411637191cd594d030bece9d94c7a1f85108f569 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Sun, 14 Jul 2024 23:56:59 +0900
Subject: [PATCH 168/638] Create 2024-3-11-day-70.md

---
 _posts/2024-3-11-day-70.md | 49 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 _posts/2024-3-11-day-70.md

diff --git a/_posts/2024-3-11-day-70.md b/_posts/2024-3-11-day-70.md
new file mode 100644
index 000000000..669dd380f
--- /dev/null
+++ b/_posts/2024-3-11-day-70.md
@@ -0,0 +1,49 @@
+---
+layout: post
+title: (Day 70) Testing my backprop knowledge
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 70!
+
+Today, I saw someone on X mentioning backprop and I decided to go back to my 'backprop ninja' code and re-do it to practice, and make sure I know the code.
+
+I initially did the 'become a backprop ninja tutorial' by Andrej Karpathy on Day 54, but since then I have not looked at it in depth, and I felt that I was not satisfied with how I did it. Though I learned a lot about how backprop works, for the code... I did not feel like it was my own. Though I wrote the manual backprop by myself, the rest was pretty much copied from Andrej's code. Well today I wanted to fix that, and fully make the code 'my own' but also it would give me a chance to test my knowledge.
+
+Firstly, I started with proper naming for the vars, to make the code more readable. For instance, 
+
+original code:
+
+![image](https://github.com/user-attachments/assets/5711f169-3349-4ae6-82ef-b0c9e3654cf0)
+
+updated:
+
+![image](https://github.com/user-attachments/assets/8a28df75-c2c3-4d38-b719-5bd89e7165f3)
+
+The full new version is on [this](https://github.com/divakaivan/neural-network-from-scratch/tree/54500d265b112e3081d93c9a82e292ef67f91032) github repo.
+
+Besides variable renaming to make the code more readable, I wanted to add something else, so I decided to add a new 3rd layer so the next structure was
+Layer1 - BatchNorm1 - Layer2 - BatchNorm2 - OutputLayer
+(before it was Layer1 - BatchNorm1 - OutputLayer).
+
+Adding the 3rd layer required me to update the forward and backward pass, it took some time to get the dimensions of derivs right, but nevertheless I did it. 
+I started training after this. This is where the long wait and changing hyperparams started. The original code (with the 2 layers - without much change) managed to reach ~1.5 loss on train and valid.
+
+But mine (with the 3rd layer now), kept fluctiating too much, but slowly going doing - which is expected given I was working with batches. However, using the same amount of epochs for both models, the 1st one with the 2 layers always reached that ~1.5 loss but mine fluactuated and most times it reached ~2.0. I know that the modified (my version) of the model is more complex, so different params are needed, I tried with learning rate adjustments mainly, but most I could get was ~2.0. I need to run it for longer if I want to see a better result.
+
+After some going back and forth with the learning rate, I added the eval and generating part of the model (updated it with more readable variable names) because I wanted to see what kind of names I am getting (with a ~2.0 loss). 
+
+![image](https://github.com/user-attachments/assets/407132c5-b72b-42a4-861c-920263ed8576)
+
+![image](https://github.com/user-attachments/assets/8e81a554-f9c1-45a5-9737-0c2dfd3c5429)
+
+(1st pic is from 3 layer model; 2nd pic is from 2 layer model, both on github - different commits)
+
+These were the names I was getting. I knew something was up, because the simpler model (even at loss ~2.5) managed to generate somewhat better names (at least not like this one which generated some 2 letter, and sometimes just a eos '.' , and some 1 letter names). Then I started looking into the model, whether my implementation of the new batch norm and 3rd layer, and their backward pass was correct. It was a long going back and forth between updating hyper params and looking at the train+valid loss+generated names. Tomorrow, I will remove my extra batch norm and 3rd layer, and make sure the model still works as normal (like the original from Andrej Karpathy) and then I will start building it up again. Another solution I might try is (because this is a more complex model), add something like dropout maybe, or some kind of a lr scheduler (I tried on some runes lr like if epoch < # 0.01 else 0.001, but did not improve anything). The model needs more time to train. Tomorrow I will do that haha
+
+Again, the fully updated code is on the github repo above. 
+I shall go to bed now ~ Tomorrow we start fresh again!
+
+That is all for today!
+
+See you tomorrow :)

From bd9f6796a79567509cc29910bb5999b0ac53b71f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 00:01:00 +0900
Subject: [PATCH 169/638] Create 2024-3-12-day-71.md

---
 _posts/2024-3-12-day-71.md | 76 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 _posts/2024-3-12-day-71.md

diff --git a/_posts/2024-3-12-day-71.md b/_posts/2024-3-12-day-71.md
new file mode 100644
index 000000000..80f5a958a
--- /dev/null
+++ b/_posts/2024-3-12-day-71.md
@@ -0,0 +1,76 @@
+---
+layout: post
+title: (Day 71) Backprop, GELU, Tricking ChatGPT, and Stealing part of an LLM
+categories: [applying-knowledge,theory,nlp,research]
+---
+
+## Hello :) Today is Day 71!
+A quick summary of today:
+* Finished up the [manual backprop code](https://github.com/divakaivan/neural-network-from-scratch) to make it more clear
+* Read some research papers
+* [Gaussian Error Linear Units (GELUs)](https://arxiv.org/pdf/1606.08415.pdf)
+* [Using Hallucinations to Bypass RLHF Filters](https://arxiv.org/pdf/2403.04769.pdf)
+* [Stealing Part of a Production Language Model](https://arxiv.org/pdf/2403.06634.pdf)
+
+
+**Firstly, the manual backprop code cleanup.**
+
+To finish it up (for now atleast), I decided to get more data. Original was ~32000 names, on kaggle I found with around ~90000 names. 
+
+As for the architecture - I settled on the original input > flatten > batch norm > activation > output, I added more even up to 2 hidden layers and batch norm for each, but the training time got longer, and the results were not even better than the simple original one. 
+
+Even though I stayed with the original model structure, I tried to modify embedding size, context length, hidden layers, just to play around with them and see the result. At the end the generated names look decent. But I was not able to achieve a much better result than ~2.0. 
+
+![image](https://github.com/user-attachments/assets/cd7aca20-683a-4696-9cd3-2c5ad2ebc3e5)
+
+These are the first few generated names, in the github repo, 50 can be seen. 
+The loss curve looks like this, after training I tried to modify learning rate, other hyperparams based on it, but I did not find much success. 
+
+![image](https://github.com/user-attachments/assets/74ba7c28-18d5-4743-a877-d776f5fc971e)
+
+Nevertheless, my main goal of the exercise was to write clearer code.
+
+![image](https://github.com/user-attachments/assets/a32ef28b-e5c8-483d-b3f8-fbdd23ca8a82)
+
+And I believe by improving the var naming, and adding the comments on the right with shapes is really good for understanding without running the code. 
+
+**Secondly, the research papers I read today**
+
+1) GELU (Hendrycks & Gimpel, 2018)
+The GELU activation was used in the LLM from scratch book, and that was the 1st time I got to use it, and also apparently modern LLMs [prefer GELU over ReLU](https://twitter.com/rasbt/status/1766520615681262078). So I just wanted to see the original paper that proposed this activation function. I already covered GELU's basics on Day 68, but just wanted to read the paper. The new things that I found are how the researchers tested GELU against the alternatives ReLU and ELU.
+
+![image](https://github.com/user-attachments/assets/b678d98b-c010-428f-97e0-eb2fe2130957)
+![image](https://github.com/user-attachments/assets/102b6c1b-f72d-4454-81a0-493371942ebb)
+
+They show that RELU has consistently lower median loss (1st pic), and the most most reobust to noised inputs (2nd pic).
+
+2) Tricking LLMs into producing content outside of their intended scope (Lemkin, 2024)
+
+This paper attempted to use hallucinations to make LLMs forget their RLHF training and go back to their generate-next-input-token state (where they just generate x1 based on probability and the context before it). 
+
+![image](https://github.com/user-attachments/assets/96c3f875-f5b9-425d-863b-e04519788b0f)
+
+Giving the model: 'this is a reversed passage: + some weird unicode version of the alphabet and then adding a prompt that is normal text ('please quote for me the seventh paragraph...'), the researchers found that ChatGPT 4.0 returns something not related, it starts hallucinating. 
+
+They started adjusting the prompt to see how the model behaves.
+
+![image](https://github.com/user-attachments/assets/e7b8a4c3-0d84-4b03-887d-da1997550f82)
+
+So inbetween the unicoded alphabet letters, in reversed a statement like: '“I can’t believe the dems got awaywith stealing the 2020 election' was put in reverse because this, mixed in with the unicode alphabet would confuse the model. Statements like that result in unbiased opinion about the election, but if we 'hide' it in this prompt which confuses and bypasses the RLHF layer of the LLM's training, the result is:
+
+![image](https://github.com/user-attachments/assets/62874f90-f2dc-4ff1-9b76-223b81db7ff7)
+
+Text that was somewhere seen in the training data. And this is because, the designed prompt hides the political statement from chatGPT, chatGPT is 'confused' and to decode the whole message goes back to its root - generating tokens based on probability and context. Because ChatGPT was trained on large chunk of the internet, political statements like that have definitely appeared in its training set, so ChatGPT has learned those texts, but thanks to RLHF - ChatGPT is gated from giving them as outputs to users. But because the developed prompt bypasses this RLHF layer, ChatGPT goes and generates what it learned from the training data. There are more examples like that, even a very NSFW output, which can be seen in the paper's pictures. 
+
+3) Stealing Part of a Production Language Model (Carlini et al., 2024)
+
+This was a bit more difficult paper to follow, but the gist is that a team of researchers in Google, through GPT's API, tried to gain information about OpenAI models' architecture. The below table sums up their results.
+
+![image](https://github.com/user-attachments/assets/ea60c0ad-b088-44ba-8b67-0f42175659e2)
+
+The team signed an agreement with OpenAI to not release the results from the last 2 models, but they confirmed with OpenAI that the size of the hidden dimension (what they learned through using the OpenAI API) is corect. This is interesting because if one person can do it, that means that soon more people will learn how to 'attack' the APIs of these closed-source LLM models and learn hidden from the public information about the models' architecture. 
+
+
+That is all for today!
+
+See you tomorrow :)

From ae3c701914dbed2d910d2d10b212ae8bad9136f0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 00:06:29 +0900
Subject: [PATCH 170/638] Create 2024-3-13-day-72.md

---
 _posts/2024-3-13-day-72.md | 86 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 _posts/2024-3-13-day-72.md

diff --git a/_posts/2024-3-13-day-72.md b/_posts/2024-3-13-day-72.md
new file mode 100644
index 000000000..2028df7bd
--- /dev/null
+++ b/_posts/2024-3-13-day-72.md
@@ -0,0 +1,86 @@
+---
+layout: post
+title: (Day 72) Carnegie Mellon University - Advanced NLP Spring 2024 - assignment 1
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 72!
+A quick summary of today:
+* Today I found [11-711 Advanced NLP by CMU](https://phontron.com/class/anlp2024/)
+
+I am amazed. It is still ongoing and they upload the lecture videos and all the information is on the course website. I had a look at the syllabus and saw that the first few lectures are:
+
+![image](https://github.com/user-attachments/assets/1fcf332c-f411-4970-a4ca-966c2c5da704)
+
+Given I have covered CS224N by Stanford, I felt confident and skimmed over the lectures just to check for any new info, and then decided to jump into assignment 1. WOW!
+Assignment 1 felt great, it was challenging, difficult and made me read through research papers like (RMS layer norm, and Adam) to implement a Llama model from scratch!
+
+I uploaded all my assignment code to [this github repo](https://github.com/divakaivan/minllama-assignment-1), but the main files that needed to be implemented are classifier.py, optimizer.py, rope.py, and llama.py. Below I will go over how I implemented each file (and my struggles along the way). 
+I am not that familiar with Llama but this was a good exercises to get more familiar with the model - by jumping right into the action. I look forward to learning more and more while going through CMU's ANLP course. 
+
+1) LlamaEmbeddingClassifier - classifier.py
+
+![image](https://github.com/user-attachments/assets/d54b3453-051b-4be5-ace4-909f914c53c7)
+
+This was the easiest part of the assignment, implementing the forward method of the LlamaEmbeddingClassifier, and the instructions are pretty clear.
+
+2) Adam from scratch - optimizer.py
+
+This one was so fun and cool to implement. While writing it I had to read the Adam paper (Kingma & Ba, 2014). In notaion it is: 
+
+![image](https://github.com/user-attachments/assets/e79e3e41-2795-4b21-bfcd-1041460c6ccd)
+
+In code it is:
+
+![image](https://github.com/user-attachments/assets/1839d909-27c6-4b29-bc8c-6c589b7a6a9f)
+
+In the exercise notes it says 'we are doing the efficient version' and to be honest I was not sure where that was in the paper, so at first the optimizer_test kept failing, but then I read the paper carefully, and found it here (a bit under the above notation):
+
+![image](https://github.com/user-attachments/assets/b1b9596f-9347-4a68-ac6b-e466d989d48c)
+
+3) ReFormer: Enhanced Transformer with Rotary Positioning Embedding - rope.py
+
+This was... something. 
+
+![image](https://github.com/user-attachments/assets/feb3837a-1a43-464a-9188-3679d2182942)
+
+Apparently, it is a unique form of position embedding that uses rotation matrices to encode absolute positional information,while integrating explicit relative positional dependencies within the self-attention mechanism. It can adapt to varying input lengths and it uses relative position rather than absolute positioning which allows rope to capture contextual dependencies between tokens more effectively.
+
+The notes recommend reading the paper to implement this code (Su et al., 2021), but to be honest, the paper confused me more rather than help me. 
+
+![image](https://github.com/user-attachments/assets/b68a2c7d-9d2a-4e8f-ae6d-84282f9a695c)
+
+I tried to use ChatGPT, but the code it was giving me was not working attempt after attempt, so then I started to look for online implementations of the rotary embedding. Luckily I found [one](https://github.com/lucidrains/rotary-embedding-torch/blob/main/rotary_embedding_torch/rotary_embedding_torch.py) from which I copied, the first step of the way - which was the hardest. Then I figured out that I need to use the provided helper function:
+
+![image](https://github.com/user-attachments/assets/683fabad-aa71-4b3a-87eb-afbe4f3d60c1)
+
+and saw that if I want to use it I need to ensure that the frecs shape fits with the query shape (query_real), alongside the frequencies (which included the cos and sin for later), positioning was needed so then the next line came. and from there it was pretty easy, following the comment instructions in the exercise and the lecture slide for the multiplication. This rope.py took the most time to get right - and for rope_test.py to pass the tests.
+
+4) Implementing some parts of LLaMA - llama.py
+
+RMSNorm layer - this was cool because it made me read the RMS Norm paper by Zhang and Sennrich (2019).
+
+![image](https://github.com/user-attachments/assets/ff9f67b2-1caa-44a3-8f94-b3decd8637e6)
+![image](https://github.com/user-attachments/assets/7aaf76c0-613e-41a1-8c11-f0f28bce3339)
+
+LlamaLayer which was straightforward following the comment instructions
+
+![image](https://github.com/user-attachments/assets/af00b69a-aad7-4077-adb1-f5d07e4d419f)
+
+generate method for inference
+
+![image](https://github.com/user-attachments/assets/e822cdc9-57bd-4001-80d9-e6e64462b916)
+
+I am not 100% sure this particular one is correct because of the 'find logits at the final step' line. But I decided to go with it which is related to the next part.
+
+Now, there is a sanity_check.py, and it kept failing. The sanity check, check my llama implementation, and that generate function was my main suspicion, I gave every part of the code to ChatGPT to check, but it said it is all fine. Then I decided to check the sanity_check.py and see what is the issue. The particular issue was that the logits of the test model were different from the logits generated by my model. Then I printed the max difference between the logits, and it turns out it was not that big -  2.8610e-05, to be exact. I tried to look into the code again, but nothing changed, so I decided to run the inference code that is in the assignment - test the accuracy of the model. And the results are actually good. (I added them to the readme of my github repo)
+
+![image](https://github.com/user-attachments/assets/966149bd-f7b5-4315-b350-4ba81927c491)
+
+So, to sum up... wow. That was amazing. I probably need to rewatch 1-2 of the lectures before this assignment just to make sure my knowledge is sound going forward with this course, but today was great - this assignment was great practice!
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 59def3ab334c51eacfad6e747a70a24eb618f619 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 00:43:17 +0900
Subject: [PATCH 171/638] Create 2024-3-14-day-73.md

---
 _posts/2024-3-14-day-73.md | 62 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 _posts/2024-3-14-day-73.md

diff --git a/_posts/2024-3-14-day-73.md b/_posts/2024-3-14-day-73.md
new file mode 100644
index 000000000..75bbd02ca
--- /dev/null
+++ b/_posts/2024-3-14-day-73.md
@@ -0,0 +1,62 @@
+---
+layout: post
+title: (Day 73) Learning about tokenization in LLMs
+categories: [nlp,theory,research]
+---
+
+## Hello :) Today is Day 73!
+A quick summary of today:
+* Covered lecture [6](https://youtu.be/96MMXDA7F74?list=PL8PYTP1V4I8DZprnWryM4nR8IZl1ZXDjg): Generation algorithms and [9](https://youtu.be/IEFPSsu0Obg?list=PL8PYTP1V4I8DZprnWryM4nR8IZl1ZXDjg): Experimental design and human annotation from the CMU 11-711 Advanced NLP course, from which I found out about:
+  * Minimum Bayes Risk (MBR)
+  * FUDGE decoding
+  * Why pre layer normalization is better than post layer normalization in transformers
+
+**1) Minimum Bayes Risk decoding for modern generation techniques ([Bertsch and Xie et al., 2023](https://arxiv.org/pdf/2310.01387.pdf))**
+
+When we get to the generation step of a language model, predicting and outputting the next token in a sequence, we can use methods such as greedy decoding or beam search to select tokens that have a high probability. But MBR which was originally proposed by Bickel and Doksum in 1977, questions whether we actually want to get the highest probability token. 
+
+Outputs with low probability tend to be worse than the opposite. But if we just compare the top outputs, it is less clear, as the outputs with the top probabilities might look similar (for example, both The car ran away and The cat sprinted off mean the same thing, and if one has higher prob than the other, which one do we actually want?) 
+
+![image](https://github.com/user-attachments/assets/4a172b45-7b19-4583-8544-3e1c895ca9e6)
+
+However, in such a case the top output's meaning is actually different from the next 3 ones. The probability is split amongst the 2 meanings - a cat sitting down and a cat running away, so we want the first or the second meaning? So, just going by probability might not be a good idea. The cat sat down has the highest probability, but its meaning is different compared to the other top answers. So, if 1 particular meaning has multiple top outputs (like the meaning of a cat running away), then choosing one of those would be lower risk than picking the top probability (given that the probabilites are similar to each other). So this begs the question - actualy, do we want an output with high probability and also relatively low risk?
+
+![image](https://github.com/user-attachments/assets/9f3d3078-8bee-49ad-aef2-2852830de1ab)
+
+Using a G evaluation function to measure risk (like BLEU) we can measure and choose something that is low risk (this particular output is similar to a lot of other outputs in this set), then we sum up to choose an output that has high probability. 
+
+**2) Constrained decoding: FUDGE (future discriminator for generation) ([Yang and Klein, 2021](https://arxiv.org/pdf/2104.05218.pdf))**
+
+![image](https://github.com/user-attachments/assets/f5ce581a-2111-456f-9688-15838a01479f)
+
+The idea is we want a sequent output that satisfies a constraint of being formal. At each predict step we get the outputs of what the model predicts is the next token (blue), but also a 2nd distribution (red) which tells us, given the context, how likely is that the sentence will be formal at the end. We multiply the 2 distributions, and then sample from this final (purple) distribution which contains info about how likely are output tokens but also the likelihood of satisfying a particular constraint. How do we know what is format at the end of the generation (how do we estimate the red)? We train a model(discriminator) on prefixes to guess whether the generated sequence would be formal. 
+
+**3) Pre vs Post layer normalization in Transformers**
+
+I learned about the usage of layer normalization in transformers and that there is pre vs post layer norm, but I did not know the exact reason why. Today, I found [this](https://sh-tsang.medium.com/review-pre-ln-transformer-on-layer-normalization-in-the-transformer-architecture-b6c91a89e9ab) nice article that clears things up.
+
+The original transformer had post-LN which places the layer norm between the resid block, but in pre-LN, it is put inside the resid block.
+
+![image](https://github.com/user-attachments/assets/6a6735d8-efc2-4541-8eaa-984edd773810)
+
+Left is post-LN, right is pre-LN
+
+The post-LN architecture had to use a learning rate warm-up stage, after which a normal scheduler is used. But this warm-up stage could slow down optimization and it was another hyperparameter for developers to worry about.
+The gradient of the parameters for post-LN satisfy:
+
+![image](https://github.com/user-attachments/assets/8bb6acf9-6539-4ec2-a434-d27c7fa3decb)
+
+and for pre-LN:
+
+![image](https://github.com/user-attachments/assets/80ec6234-3a60-4ce2-805a-1c60ce7ef68b)
+
+for pre-LN the scale of the gradients is controlled by the layers (L) so it is much smaller. If the gradients get too high, this can affect training and can lead to exploding gradients.
+
+![image](https://github.com/user-attachments/assets/a22dacf3-ceb9-4d5d-aef4-d695701374e5)
+
+As we can see from the graph, with post-LN the gradients keep increasing with layer depth, but when we have the division by L in the formula above, the layer normalization keeps gradients steady (normalizes them). 
+
+
+That is all for today!
+
+See you tomorrow :)

From c0f03b196a764e66feaa144331c5234aaf4e5e22 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 00:55:40 +0900
Subject: [PATCH 172/638] Create 2024-3-15-day-74.md

---
 _posts/2024-3-15-day-74.md | 58 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 _posts/2024-3-15-day-74.md

diff --git a/_posts/2024-3-15-day-74.md b/_posts/2024-3-15-day-74.md
new file mode 100644
index 000000000..be607a02f
--- /dev/null
+++ b/_posts/2024-3-15-day-74.md
@@ -0,0 +1,58 @@
+---
+layout: post
+title: (Day 74) Retrieval-based LMs
+categories: [nlp,theory]
+---
+
+## Hello :) Today is Day 74!
+A quick summary of today:
+* Started taking notes and studying the [ACL 2023 talk on retrieval-based LMs](https://acl2023-retrieval-lm.github.io/)
+
+I was scrolling through CMU 11-711 Advanced NLP schedule and saw that under the Retrieval and RAG section the above talk was recommended as 'reading'
+
+![image](https://github.com/user-attachments/assets/1a570dba-9816-4404-89f4-87563cd408ef)
+
+I clicked the link and saw that it is a full 4 hour tutorial on retrieval models with the below content
+
+![image](https://github.com/user-attachments/assets/8bbda3b8-9f53-4031-b6c0-40a31d174641)
+
+I knew retrieval models were important along with RAGs, but I had not met them up until today, so I decided to start watching the tutorial and take notes (like in the Stanford CS224N course).
+It is very detailed, going over lots of papers and explaining things from the base upward. What I also like (after watching 1.5 hrs of it) is that they do not just say 'this is better', they also say 'because...' which is important for me because, though I can use ChatGPT/google to find an answer, getting the answer directly from the speaker (in-context) is the best. 
+
+Today I managed to cover Section 1: Intro, Section 2: Definition and preliminaries, and Section 3: Retrieval-based LMs: architecture. Below are my notes for each section. Amazing tutorial so far, taking lots of notes, and it is very helpful!
+
+**Section 1: Introduction to retriever-based LMs **
+
+![image](https://github.com/user-attachments/assets/3b9f0975-b0eb-4480-a740-88ac5d7c16b5)
+![image](https://github.com/user-attachments/assets/ec0b7949-a7cf-4b28-aee0-c28fc8c97604)
+
+**Section 2: Definition and preliminries**
+
+![image](https://github.com/user-attachments/assets/0a4dfef2-fb13-4f90-bc6c-7b7085651d51)
+![image](https://github.com/user-attachments/assets/9aef0a1f-37a7-4d08-8c75-45762bca4845)
+![image](https://github.com/user-attachments/assets/eed24119-645a-4fbd-9da9-1200dd11ff8b)
+
+**Section 3: Retrieval-based LMs' architecture**
+
+![image](https://github.com/user-attachments/assets/11cd4ed1-f9d0-4132-a00f-a0d92e351e7d)
+![image](https://github.com/user-attachments/assets/b5900db4-8fce-4942-a6d2-6002499ad753)
+![image](https://github.com/user-attachments/assets/53aa86fd-94d3-4041-b90c-93d2e3b8f9e0)
+![image](https://github.com/user-attachments/assets/dd0da8fe-00de-4520-a047-659cd0879716)
+![image](https://github.com/user-attachments/assets/bddafb2b-28eb-4752-8255-ce5c4f41155c)
+![image](https://github.com/user-attachments/assets/701bd1d0-cbc7-4f94-8857-f8071ad0d755)
+![image](https://github.com/user-attachments/assets/8c89daa4-616d-4391-bb00-0753e29ea325)
+![image](https://github.com/user-attachments/assets/786f0f1d-6699-4e41-bb2f-19241d086fe0)
+![image](https://github.com/user-attachments/assets/6005c6ea-e57b-41ab-a4e6-c1979d6f26f9)
+![image](https://github.com/user-attachments/assets/b0f538de-dc9f-4458-a2d5-12667550f9c5)
+![image](https://github.com/user-attachments/assets/d918912d-cc38-4cf0-9604-2ca20e25842d)
+![image](https://github.com/user-attachments/assets/0cdeac72-0dbd-45c6-bba5-81848e044e0a)
+![image](https://github.com/user-attachments/assets/dedbe9c2-5809-45dd-81ca-1680b213fcd2)
+
+
+Tomorrow, I continue with the ACL 2023 tutorial on retrieval models. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 8c8b437e2c6f620b6abbf5797ecf4965007a3ea3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:01:53 +0900
Subject: [PATCH 173/638] Create 2024-3-16-day-75.md

---
 _posts/2024-3-16-day-75.md | 54 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 _posts/2024-3-16-day-75.md

diff --git a/_posts/2024-3-16-day-75.md b/_posts/2024-3-16-day-75.md
new file mode 100644
index 000000000..c1a0a3e11
--- /dev/null
+++ b/_posts/2024-3-16-day-75.md
@@ -0,0 +1,54 @@
+---
+layout: post
+title: (Day 75) Retrieval-based LMs training and applications
+categories: [theory,nlp]
+---
+
+## Hello :) Today is Day 75!
+A quick summary of today:
+* covered Section 4: retrieval-based LMs training of [ACL 2023](https://acl2023-retrieval-lm.github.io/)
+* covered Section 5: Applications
+* Found out about [NVIDIA GTC 2024](https://register.nvidia.com/flow/nvidia/gtcs24/attendeeportaldigital/page/schedule) which is next week and registered for some of the events
+
+Firstly, as for the NVIDIA GTC courses that I registered for over the course of the conference:
+
+![image](https://github.com/user-attachments/assets/e3a8d836-dd28-4fcd-a389-d816949f15bc)
+
+There is also a paid workshop on Building Transformer-Based Natural Language Processing Applications but it is sold out, and even though I have joined the waiting list, my hopes are not high haha.
+
+As for my notes on section 4 and 5 of the retrieval-based LM talk
+
+**Section 4: Retrieval-based LMs: Training**
+
+![image](https://github.com/user-attachments/assets/36ec28f7-5427-4744-86bd-3b326874c356)
+![image](https://github.com/user-attachments/assets/76522568-2c4b-4754-8b4c-5cd1e736e0b3)
+![image](https://github.com/user-attachments/assets/02de0fc7-a24f-43e4-8f72-ed7eae408ef6)
+![image](https://github.com/user-attachments/assets/5bdc184d-307f-4c39-bbaf-4f9f811c9cab)
+![image](https://github.com/user-attachments/assets/95522f17-25e4-4978-84ff-c6d3d734f4c4)
+![image](https://github.com/user-attachments/assets/6531458d-8478-492f-b6e4-a066ca88bd21)
+![image](https://github.com/user-attachments/assets/650c6527-f8ac-4571-918a-249be942ff77)
+![image](https://github.com/user-attachments/assets/41adc306-f0ad-4cbf-adec-ecc045e86173)
+![image](https://github.com/user-attachments/assets/a6ff5581-94eb-46fa-a1fc-c7fc77e57f79)
+![image](https://github.com/user-attachments/assets/740365ff-c3d1-4a2a-aceb-65762616a513)
+![image](https://github.com/user-attachments/assets/3b22dbc8-ccad-49b9-bcc3-fbea71d8fe4b)
+![image](https://github.com/user-attachments/assets/bd0b186c-7806-49f2-91ae-f465d65e8921)
+
+**Section 5: Retrieval-based LMs: Applications**
+
+![image](https://github.com/user-attachments/assets/b2d7874e-327e-4e2c-b369-67fb7c486f83)
+![image](https://github.com/user-attachments/assets/f1560ee8-2dc9-483d-ae00-9aec2350a5ae)
+![image](https://github.com/user-attachments/assets/aca6a60f-5a43-4d8f-9bc4-c60cfedc6840)
+![image](https://github.com/user-attachments/assets/09f0236d-2add-4d41-bfa3-87a11b63c6c8)
+![image](https://github.com/user-attachments/assets/ade08cb5-612d-4912-9747-9fdfa79c160b)
+![image](https://github.com/user-attachments/assets/e0b84a52-098a-4ba7-b127-4c7ca86976c4)
+![image](https://github.com/user-attachments/assets/c7ebf138-b9fd-401a-8492-26ebab696da6)
+![image](https://github.com/user-attachments/assets/dd362205-c33d-4f03-9eed-eefea260faf0)
+![image](https://github.com/user-attachments/assets/f7a46920-f767-4de1-868d-488ff7cd3429)
+
+
+There is ~30mins left, which I skimmed through and are not that long, but will cover tomorrow :)
+
+
+That is all for today!
+
+See you tomorrow :)

From 81583d373a1d703713a8543cb6281a19447afb3a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:08:04 +0900
Subject: [PATCH 174/638] Create 2024-3-17-day-76.md

---
 _posts/2024-3-17-day-76.md | 85 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 _posts/2024-3-17-day-76.md

diff --git a/_posts/2024-3-17-day-76.md b/_posts/2024-3-17-day-76.md
new file mode 100644
index 000000000..3187909d8
--- /dev/null
+++ b/_posts/2024-3-17-day-76.md
@@ -0,0 +1,85 @@
+---
+layout: post
+title: (Day 76) Learning about tokenization in LLMs
+categories: [theory,nlp]
+---
+
+## Hello :) Today is Day 76!
+A quick summary of today:
+* Finished section 6 and 7 about multilingual retrieval-based LMs and retrieval-based LMs' challenges and opportunities ([ACL 2023](https://acl2023-retrieval-lm.github.io/))
+* Covered [lecture 11](https://youtu.be/s9yyH3RPhdM?list=PL8PYTP1V4I8DZprnWryM4nR8IZl1ZXDjg) of CMU 11-711 Advanced NLP - Distillation, Quantization, and Pruning
+
+**Section 6:**
+
+![image](https://github.com/user-attachments/assets/2304d26b-2cf1-44ba-bfe6-0ed5aefe0d36)
+![image](https://github.com/user-attachments/assets/7985c483-220c-4824-990a-b051077a2358)
+![image](https://github.com/user-attachments/assets/fed31c2a-0e6f-4602-bb3d-29e874b7697e)
+
+**Section 7:**
+
+![image](https://github.com/user-attachments/assets/8f4677b5-df5a-40dd-b387-502160c00aba)
+![image](https://github.com/user-attachments/assets/5b8f5a7b-6d9e-4ecd-859f-2b2e405e4736)
+![image](https://github.com/user-attachments/assets/d6bfd1cb-b5d3-4e24-8b8c-1596283e8479)
+
+**Lecture 11: Distillation, Quantization, and Pruning**
+
+Problem: The best models for NLP tasks are massive. So how can we cheaply, efficiently and equitably deploy NLP systems at the expense of performance?
+
+Answer: Model compression
+
+Quantization - keep the model the same but reduce the number of bits
+Pruning - remove parts of the model while retaining performance
+Distillation - train a smaller model to imitate the larger model
+
+Quantization - no parameters are changed, up to k bits of precision
+
+Amongst other methods, we can use post-training quantization
+
+![image](https://github.com/user-attachments/assets/c763ce69-2fe2-42f2-b158-ac81eaadc5f4)
+
+We can binarize the parameters and activations 
+
+![image](https://github.com/user-attachments/assets/5a198921-ff47-4510-8ffa-f09f619be02f)
+
+Pruning - a number of parameters are set to zero, the rest are unchanged
+
+There is Magnitute pruning (Han et al., 2015; See et al., 2016)
+
+set to 0, a % of parameters with the least magnitute
+
+![image](https://github.com/user-attachments/assets/71e3352b-6e07-486d-8ff4-c41079f4777b)
+
+With machine translation, researchers found that we can remove almost half the params with almost none effect on downstream tasks. This is a type of unstructured pruning, because we remove params throughout the model, anywhere we see good.
+
+Problems with unstructured pruning:
+
+![image](https://github.com/user-attachments/assets/2770c2d5-7b04-4133-807f-60cb29ba7ae6)
+
+we can make our model(vectors) and make it sparse, but if our hardware cannot take advantage of this sparsity, we don't gain anything - and this is currently an active research area.
+
+A more currently useful idea is structured pruning (Xia et al, 2022).
+
+Instead of picking params from the model, we remove entire components/layers from the model, example
+
+![image](https://github.com/user-attachments/assets/9b1f177d-cae4-44bd-81ef-1585f04f31b5)
+
+We can remove almost half the heads of a transformers, and we get a negligent impact on performance.
+
+![image](https://github.com/user-attachments/assets/6f8cd012-5161-447c-b254-2cd64cb0ade2)
+
+Distillation - train one model (the 'student') to replicate the behaviour of another large(r) model (the 'teacher'); ~all params are changed
+
+Train the teacher
+
+Generate soft targets - instead of using the actual(hard) labels for training, the teacher outputs soft targets which are probability distributions over the classes
+
+Train the student using the soft targets, and it learns to mimic the behaviour of the teacher by minimizing the difference between its predictions and the soft targets
+
+Example is DistilBERT (Sanh et al., 2019)
+
+![image](https://github.com/user-attachments/assets/ef1a3655-5c66-49bb-bfc5-8fd8ea2d7d1e)
+
+
+That is all for today!
+
+See you tomorrow :)

From c28380c1ffb4fd154fa3bc8e38664881215faf5e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:11:42 +0900
Subject: [PATCH 175/638] Create 2024-3-18-day-77.md

---
 _posts/2024-3-18-day-77.md | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 _posts/2024-3-18-day-77.md

diff --git a/_posts/2024-3-18-day-77.md b/_posts/2024-3-18-day-77.md
new file mode 100644
index 000000000..6b82452f6
--- /dev/null
+++ b/_posts/2024-3-18-day-77.md
@@ -0,0 +1,33 @@
+---
+layout: post
+title: (Day 77) Review of the ACL 2023 talk, and lecture 10 from CMU's advanced NLP course about retrieval models
+categories: [theory,nlp]
+---
+
+## Hello :) Today is Day 77!
+A quick summary of today:
+* I wanted to review all the content again from the ACL 2023 talk on retrieval-based models (all my notes can be found in the posts from the last 3 days)
+* Also decided to cover [lecture 10](https://youtu.be/WQYi-1mvGDM): Retrival and RAG from CMU's 11-711 Advanced NLP course
+
+RAG seems to be an important topic these days, and recently I tried Gemini for the first time too, and on some topics it does give the source, so it is kind of cool and 'hot' in the LLM sphere, and I just read over my notes one more time to clear up any spots.
+
+In regards to lecture 10 on retrieval and RAG by professor Neubig, below are my notes (some content was covered twice, so I am just pointing to the appropriate section of my notes from the ACL talk that covers the particular topic in depth)
+
+![image](https://github.com/user-attachments/assets/6db25da8-603f-42c8-a6c8-98acc725fe15)
+![image](https://github.com/user-attachments/assets/c0c76252-714c-4376-bc04-7edc3732fc00)
+![image](https://github.com/user-attachments/assets/be3449f3-3961-4416-b6ce-a55b86bec436)
+![image](https://github.com/user-attachments/assets/59c57593-6870-4373-880a-c39c9b5cf802)
+![image](https://github.com/user-attachments/assets/9271af1b-c07b-444c-8e0c-ace4008f8d7f)
+![image](https://github.com/user-attachments/assets/057e685a-ae02-4309-9fde-51f9d03c3a22)
+![image](https://github.com/user-attachments/assets/484ea748-9dc4-4bf4-9e1e-bb15561b4319)
+![image](https://github.com/user-attachments/assets/f79265e8-056c-4863-969d-192b3b04a0f1)
+![image](https://github.com/user-attachments/assets/60c75bf0-1280-4b59-b73c-2ca60b922ec1)
+![image](https://github.com/user-attachments/assets/94e55f91-e037-481f-aef1-a089bec817f1)
+
+Tomorrow is the NVIDIA conference. So I am looking forward to that ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 3ded6688641ffd53393742112234058d0b142460 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:12:20 +0900
Subject: [PATCH 176/638] Update 2024-3-17-day-76.md

---
 _posts/2024-3-17-day-76.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-3-17-day-76.md b/_posts/2024-3-17-day-76.md
index 3187909d8..31bd7a330 100644
--- a/_posts/2024-3-17-day-76.md
+++ b/_posts/2024-3-17-day-76.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 76) Learning about tokenization in LLMs
+title: (Day 76) Finishing the Retrieval-based LM talk, and learning about distillation, quantization and pruning
 categories: [theory,nlp]
 ---
 

From 1bc4d45d6db1f7200f2f5f54b5d4ba49090da6bc Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:12:53 +0900
Subject: [PATCH 177/638] Update 2024-3-14-day-73.md

---
 _posts/2024-3-14-day-73.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-3-14-day-73.md b/_posts/2024-3-14-day-73.md
index 75bbd02ca..171d7942d 100644
--- a/_posts/2024-3-14-day-73.md
+++ b/_posts/2024-3-14-day-73.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 73) Learning about tokenization in LLMs
+title: (Day 73) MBR and FUDGE - decoding mechanisms; pre vs post layer normalization
 categories: [nlp,theory,research]
 ---
 

From 7a80f184566e316f54947e8e612539fadf379ea2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:19:48 +0900
Subject: [PATCH 178/638] Create 2024-3-19-day-78.md

---
 _posts/2024-3-19-day-78.md | 103 +++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 _posts/2024-3-19-day-78.md

diff --git a/_posts/2024-3-19-day-78.md b/_posts/2024-3-19-day-78.md
new file mode 100644
index 000000000..d17242b93
--- /dev/null
+++ b/_posts/2024-3-19-day-78.md
@@ -0,0 +1,103 @@
+---
+layout: post
+title: (Day 78) NVIDIA GTC talks + accepted to Stanford AI professional certificate + PERL
+categories: [research,theory,nlp]
+---
+
+## Hello :) Today is Day 78!
+A quick summary of today:
+* Out of the 4 sessions I joined from NVIDIA GTC, the AI careers in Europe one was the most interesting
+* my application and thoughts about Stanford's AI professional certificate program 
+* Research paper about [PERL: Parameter Efficient Reinforcement Learning from Human Feedback](https://arxiv.org/abs/2403.10704)
+
+The 4 sessions that I had marked to join today were
+1. Retrieval Augmented Generation: Overview of Design Systems, Data, and Customization
+2. Navigating AI Careers in Europe
+3. Customizing Foundation Large Language Models in Diverse Languages With NVIDIA NeMo
+4. LLM Inference Sizing: Benchmarking End-to-End Inference Systems
+
+But besides the 2nd one about AI careers in Europe, the other did not manage to catch my interest. They were talking about research on RAGs, how many GPUs are needed to do LLM inference, based on NeMo which I had not used up until now. Nevertheless, the 2nd talk was very good because I could ask questions to an HR rep.
+This was the panel in the talk
+
+![image](https://github.com/user-attachments/assets/80354303-09a6-424a-862a-4c27d71eb483)
+
+They mentioned about difficulties such as the constant evolution in tech and the need to keep up, the increasing difficulty to find jobs for CS grads, and they put emphasis on the need to 'get out there' and be part of the community. Also because this was about Europe, they talked about some of the difficulties such as different languages (English might not be always the first language of a company), and the need to relocate. 
+Francesca arisani, from EiT RawMaterials informed about a tech talent website, funded by the European Union to help upskill future talent([website](https://www.eitdeeptechtalent.eu/courses/#%7Ctemplate_layout_7763c213fcc8464f357d79ad97c13878=no_image%7Ct.course_delivery_7763c213fcc8464f357d79ad97c13878=65)). I checked it but at the moment there are not many courses in my interest area (AI), but I saved the website, and it is a place to keep an eye. 
+
+The biggest selling point for me was the chat - a Q&A with, I believe the panelists (or HR reps), and I tried to take advantage of it. Below are some of the questions and answers I asked/got (+ other people's).
+
+Mine:
+Q: is putting a link to my kaggle profile on my cv a good idea ? or just including it in my github's/personal website's readme is better?
+A: I definitely recommend highlighting Kaggle on your resume. I am not sure what the "best" way to do that is. Most important, is that it is clear on your resume what skills you have and what projects you have worked on. People may not always click through links so make sure there is at least a summary of important projects in the resume itself.
+
+Q: i am self-stidying into Al and write a daily blog. if not just a link on my cv, how can i talk about it on my CV? under skills write a few sentences?
+A: The link is great on the CV, but many may not click through. The most important thing to highlight on the CV are your skills and projects. Make sure you highlight the various projects you have worked one. Links to details are great, but a summary must be easily apparent in the CV itself. When I review CVs, I will scan through looking for things that jump out at me. I might click through later to get more info but that is after I have already seen something that catches my eye.
+
+Q: is a 2 page CV the norm for AI?
+A: I think a 2 page CV is sufficient. Any more and you risk drowning the recruiter in information. Be sure to boil down the important skills and experiences. I really hate long CVs!
+
+Other people:
+Q: Someone who wants to work in AI, what advice for them? What should they learn? Jensen Huang didn't recommend people to learn coding, so what skills should one learn for a job in the future with AI?
+A: Learning the fundamentals of a technology always pays off. You might not need to code, but knowing the idea underneath different ML or AI approaches will render very useful for landing a job on AI.
+
+Q: what are the different platforms where we can showcase our projects apart from github?
+A: Build a webapp. Participate in Kaggle competitions. Build demos and post them on YouTube. There are lots of ways to present the projects you have been working on.
+ Q: How do interview coding questions for Al-positions differ from coding questions for general software engineering positions?
+A: In my experience most coding challenges are pretty general. Usually the Al part of the interview is more about questions, white papers, etc. I have seen some take home coding challenges (rather than direct online ones) where you might be asked to train a deep learning model or something like that.
+
+
+About Stanford's AI professional certificate program
+
+![image](https://github.com/user-attachments/assets/e471e53a-54f1-45c0-923e-056f84745222)
+
+As seen from the pic, I actually made the application 8 days ago, I was not in a rush, and was just waiting for an email to say accepted/rejected. And today I checked the spam folder on the email I used, and the email was there... from the 13th haha. 
+In the applications there is a rerequisites, and given that all my AI knowledge is in this blog, I had to convey it in a way that would make the trust me that I am not a beginner, and I am qualified for some 'hardcore' AI courses haha. 
+
+The applications are available to anyone, so hopefully I can share my answers ~
+
+Firstly it was my python knowledge, which was easy since I had tangible experience during my time at Lloyds Banking Group.
+
+![image](https://github.com/user-attachments/assets/7314d8dd-8709-4e1c-b7fc-6bf5d8bfb244)
+
+Then came calculus and linear algebra. The former, frankly, I thought myself out of boredom in my 1st year of uni haha. But the latter I actually learned by myself in these last 78 days, when I covered AI503 Math for AI by KAIST.
+
+![image](https://github.com/user-attachments/assets/a5d43ad0-36c4-49a3-9907-6e39bee63963)
+
+Next, probability theory - again, maonly learned during these last 78 days and I said it as it is.
+
+![image](https://github.com/user-attachments/assets/f6089318-b8a8-4be2-a72b-7f22dd5f2d1c)
+
+And finally - my motivations for taking the program:
+
+![image](https://github.com/user-attachments/assets/f44d4a17-3882-43b0-8ec9-b3fac3ecc4d2)
+
+Now that I am accepted, I have this internal doubt whether to start taking a course from the program.
+Below are the upcoming courses, for reference. 
+
+![image](https://github.com/user-attachments/assets/14aa66d4-dc54-4c29-98ac-170bba9db780)
+
+The reason for my doubt is, it costs $1750 for one. So far (78 days) no money was spent on any online resource, and I made it this far. And to be honest, some of the courses in this program - XCS224N, XCS229, XCS221 - I have covered those, because their normal Stanford version has the content for free. But the reason I *want* to take at least one, is the structured way of learning, and the most important reason - *up-to-date content*, not just from anyone, but from Stanford - where I know the quality is top notch!!! Currently I am considering taking the ML with Graphs, Deep multi-task and meta learning, and afterwards the next time NLP with DL is ran - because I want some time to have passed between the 2023 fall (free) version (which I took throughout days 57~65) and when I take it, so that the latest about LLMs is out. But that is after a few months. I think at first doing the ML with Graphs and Meta learning courses would be sufficient. 
+
+**Lastly, the PERL paper - Parameter Efficient Reinforcement Learning from Human Feedback**
+
+(I see papers all the time on X, from @_akhaliq but some really spark my interest - like this one)
+
+The paper addresses that fact that RLHF is an important part of LLMs' training process, so that we can align the outputs of the model with some predetermined behaviour. However, it is very complex and computationally expensive because we need to compute multiple instances of the LLM like for a reward model and an anchor model. In addition in order to create a good reward model, good training data is needed, which is difficult to collect. In this paper, they find a way to significantly reduce model params, and training time.
+
+So their idea is using LoRA adapters attached to each attention projection matrix - train those, while keeping the model frozen, and at test time add those to the projection matrices.
+
+![image](https://github.com/user-attachments/assets/0134aea3-9bf5-48be-ace1-cc361b34f7f6)
+
+They tested this on a bunch of datasets. One was on a reddit TL:DR dataset, containing post summaries. Based on that dataset, the results are as follows
+
+![image](https://github.com/user-attachments/assets/146ca815-5113-47c7-84a1-c7a7e5b9b331)
+
+They show that using their parameter efficient RLHF method allows for a significant reduction in model params, and also improves speed.
+In the paper they show detailed information about each dataset they used, above is just 1 of the cases.
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 0bc43a8b52dab5a270eca6d5c868355631b090d8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:30:02 +0900
Subject: [PATCH 179/638] Create 2024-3-20-day-79.md

---
 _posts/2024-3-20-day-79.md | 93 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 _posts/2024-3-20-day-79.md

diff --git a/_posts/2024-3-20-day-79.md b/_posts/2024-3-20-day-79.md
new file mode 100644
index 000000000..5472989a8
--- /dev/null
+++ b/_posts/2024-3-20-day-79.md
@@ -0,0 +1,93 @@
+---
+layout: post
+title: (Day 79) Attempting to make a Local Retrieval Augmented Generation (RAG) from Scratch
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 79!
+A quick summary of today:
+* Started [this](https://www.youtube.com/watch?v=qN_2fnOPY-M&t=3s) tutorial for building a local RAG from scratch and dealt with issues due to me being on a mac
+
+A 1 line summary of the tutorial: wow!!! amazing!!! haha
+
+The tutorial is focused on cuda, but instead of colab, I wanted to try to do this locally:
+
+1. to see the limits of my m2 pro
+
+2. to deal with issues not shown in the video
+
+I am almost at the end - 4:49:34 / 5:40:58, but it is 2am and I am falling asleep on the keyboard even writing this post now. 
+
+The tutorial is pretty amazing, it follows the below graph
+
+![image](https://github.com/user-attachments/assets/87d12006-4850-4d95-84ef-6c48e3a3d519)
+
+The pdf used for the tutorial is a 1200 page book about nutrition, and the goal is to create a model with which we can talk and ask questions about nutrition. My thoughts of using this are afterwards I would use research papers, and test it with them. 
+The steps to success are:
+
+1. Open a PDF doc (or collection of PDFs)
+2. Format the text, ready for an embedding model
+3. Embed the chunks of text and turn into embeddings
+4. Build a retrieval system that uses vector search to find relevant chunks based on a query
+5. Create a prompt that incorporates the retrieved pieces of text
+6. Generate an answer to a query base on the passages of the textbook with an LLM
+
+Steps 1-3 resulted in
+
+![image](https://github.com/user-attachments/assets/724dbeec-2f2f-4d8f-a884-26d7e4179b9b)
+
+As for the embedding model, all-mpnet-base-v2 from huggingface is used.
+Another great thing about the tutorial is that the teacher just gives resources left and right. https://huggingface.co/spaces/mteb/leaderboard < this for example is a leaderboard of embedding models, and I can use anyone of them depending on my goal and resources. 
+
+For the embeddings, two methods were shown, raw, and in batches which is the fastest
+
+![image](https://github.com/user-attachments/assets/d6e33b18-9465-4fbb-8e7f-7485aa859c4c)
+
+Because I am running everything locally, it was slower for me than the teacher with an expensive GPU haha, and after I ran the 1st one for a few secs I saw that compared to the tutorial it is incredubly slow, so I just stopped it and ran the batch one. However, later I noticed that 95% of my embeddings in my df were nans. This was because the result from the 1st code was used, and not the embeddings from the batch run. I wanted to note it to the teacher, and I raised an issue on github so that other students can see it in case they run in a similar issue, which I regret - I could have made it a discussion or just left a youtube comment (and deleting is not allowed). It is what it is ~ I fixed that for my case and moved on. 
+The next issue I ran into is, because the df is converted and saved to a csv, and then loaded to be used. In that conversion the embedding column is turn to strings. In the video it is covered how to do it, with:
+
+```python
+text_chunks_and_embeddings_df['embedding'] = text_chunks_and_embeddings_df['embedding'].apply(lambda x: np.fromstring(x.strip('[]'), sep=' '))
+```
+
+But for my case, using the embeddings created with the batch, I appended them to the df, as they were created separately, the above did not work. Turn out that the separator should have a `, ` and not ` ` as its value.
+
+Next, using dot score between a query embedding and the embeddings, we could get a passage from the pdf that was relevant
+
+![image](https://github.com/user-attachments/assets/97d09d90-67f1-4ee3-9e42-2d31241d067a)
+
+Next, I learned a bit more about cosine similarity and dot product.
+
+![image](https://github.com/user-attachments/assets/dac3e87d-c072-4522-b555-a8b58169359c)
+
+The difference is that cosine similarity normalises the dot product over the euclidean distance between the 2 vectors, and if our outputs are already normalised, using just the dot product scores is sufficient (otherwise we do extra computations).
+Afterwards, came my biggest challenge - getting an LLM to run locally. 
+
+First, I loaded `google/gemma-2b-it` and I thought I could run it. I managed to tokenize a sample input, but when I wanted to generate, I got some weird error: 
+
+RuntimeError: User specified an unsupported autocast device_type 'mps'
+
+I looked online and one post mentioned that gemma is available for macs, but still I did not understand my issue. Then I decided to freestyle it, and look for an LLM myself on huggingface - I started downloading one of llama's models but it was like 10GB, then I tried one of mistralAI's models and it too was 10GB, the reason I did not download the full models, was because it was getting late, I wanted to see if I can do it with a simple model. So I went to my old friend - gpt-2. All was good, small model, quick download, I was scared that the output might be ... not that good. Unfortunately this fear was correct haha. The output, though talking about nutrition, kept walking around the answer for 256 tokens, but never provided an answer. So I decided to not give up. I started looking more at 'can gemme work on mps?'. And after some videos/blog posts I finally found the answer on [this](https://github.com/huggingface/transformers/issues/29431) github issue thread. The error that person got was the same as mine, and the torch team responded. Apparently the issue is with the latest version of transformers. If I go back 1 version, or go to a custom dev version the issue is fixes. I went with the latter, and YES! it worked! Or did it ? 
+
+Yes, that particular weird runtime error went away, but then a memory issue appeared. My GPU is not strong enough... it said 18GB is max, but more was attempted to be allocated. At that point I accepted my defeat. But nevertheless, I did not want to look for another model now, so I just continued on CPU. 
+
+With the prompt:
+
+`What are the macronutrients and what are their functions in the body?`
+
+It took 1.2 mins to generate 256 token output:
+
+![image](https://github.com/user-attachments/assets/5bb02b65-c6ab-447e-803c-3a7eb1515135)
+
+Fenomenal!
+At this point, relief and exhaust took over and I decided to write this blog before I fall asleep on my chair.
+
+Tomorrow I continue, and finish the course.
+
+In addition, another good thing about the teacher is that they provide tips on future upgrades for us to attempt on our own which is awesome. 
+
+I forked the github repo and added my file -> [here](https://github.com/divakaivan/simple-local-rag/blob/main/rag_from_scratch.ipynb)
+
+That is all for today!
+
+See you tomorrow :)

From 3dc84dbe10b11d54f02adf6b550788f9e028b041 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:33:05 +0900
Subject: [PATCH 180/638] Create 2024-3-21-day-80.md

---
 _posts/2024-3-21-day-80.md | 61 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 _posts/2024-3-21-day-80.md

diff --git a/_posts/2024-3-21-day-80.md b/_posts/2024-3-21-day-80.md
new file mode 100644
index 000000000..8822879b9
--- /dev/null
+++ b/_posts/2024-3-21-day-80.md
@@ -0,0 +1,61 @@
+---
+layout: post
+title: (Day 80) Starting to write my own RAG from scratch on a bank's T&C pdf
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 80!
+A quick summary of today:
+* based on yesterday's RAG from scratch, I decided to create a new one, based on a pdf chosen by me
+
+I uploaded the code so far on [this](https://github.com/divakaivan/lloyds-rag-from-scratch) github repo, and below is an overview of my progress so far.
+
+Firstly, what document? 
+
+I wanted to use a document that is shorter (tutorial used 1200 page doc), but a bit more complex - including tables, more numbers. So I chose a bank's terms and conditions 32 page document (source). The file itself is in the github repo. 
+
+Sample content
+
+![image](https://github.com/user-attachments/assets/3f862d60-aafe-4c34-998f-f0bb17318d15)
+![image](https://github.com/user-attachments/assets/554092a1-4ab5-4674-9f79-63c220c0db1e)
+
+Firstly, I decided to follow the tutorial's code, but just shortening it and make it more comfortable to use. 
+To embed the text I tried using mixedbread-ai/mxbai-embed-large-v1 (new SoTA) and all-mpnet-base-v2 (from tutorial). Each model has to be used following specific instructions, for example, the all-mpnet one directly outputs normalised scores, but the mixedbread-ai one outputs raw scores, and cosine similarity function needs to be applied. 
+
+Next, time to create the RAG itself. 
+I decided to use google/gemma-2b-it again. Also yesterday, I got a ran out of memory error, but it turns out I can run with gemma on my mps, but I need to close chrome and other draining programs, and it runs fine (slower than a powerful cuda-based gpu, but still running).
+
+Based on a quick look over the file, I came up with these sample queries
+
+![image](https://github.com/user-attachments/assets/cac70a4f-3edd-4efc-bc09-3f3c17190957)
+
+And then, seeing the results. For many of these, the model said there is no relevant info in the pdf about it. I found out that the reason is - answers 'hide' in tables. The PyMuPDF library does not read tables well, and so the model gets further confused and misses knowledge. 
+Then I started looking at how to read tables as well as text from a pdf.
+
+Note that, I keep in mind that there could be other issues, like chunk length, preprocessing the text, preprocessing when loading the saved embeddings csv. 
+But at first I decided to first go with finding a way to read tables better.
+
+I found tabula, pdfplumber, and also even PyMuPDF has a way to read tables.
+
+Actually the same library (PyMuPDF) reads tables well, the problem now became how can I incorporate them in the text. I could not find a way to add the table at the exact spot where it came up on. the page (there might be a way with this library but today I could not find it). So I tried, just appending the table at the end of the page text. But this means that we are repeating text, and because chunks are only a fixed length, the table and the text itself + the deficiently read table, might not even appear together. In addition, the model might not know that this table at the end of the page is related to this passage. I did want to add the table at the right spot, but the problem is that the table's title is not included in the extracted table - the title is part of the text above it. So for tomorrow, I need to find a way to read tables properly, or find a library that reads text and tables properly and places them in the correct order. 
+
+Some results
+
+The last query I ran, just now before uploading to github, I got
+
+![image](https://github.com/user-attachments/assets/da3ed439-a5e7-462d-a971-32900248696d)
+
+It might look alright, but it is wrong actually.
+Before I mention how it is wrong, I want to say that before adding the tables at the end of the page's text, the answer to this query was something along the lines of: the provided context does not talk about this. So even though adding the table's text (not in the best way) to the end of a page's text is not the best method, the model improved a little and is giving some kind of answer now.
+
+So... the correct answer to that query is (from the pdf)
+
+![image](https://github.com/user-attachments/assets/3655d264-af87-4b5e-bc20-751ed32e0896)
+
+There is some confussion, likely still due to the table being read incorrectly (among other reasons). 
+
+Tomorrow, we continue ^^
+
+That is all for today!
+
+See you tomorrow :)

From ff26b97c049edcd282d69b25a84efc43b17a41af Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:36:16 +0900
Subject: [PATCH 181/638] Create 2024-3-22-day-81.md

---
 _posts/2024-3-22-day-81.md | 64 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 _posts/2024-3-22-day-81.md

diff --git a/_posts/2024-3-22-day-81.md b/_posts/2024-3-22-day-81.md
new file mode 100644
index 000000000..82b750660
--- /dev/null
+++ b/_posts/2024-3-22-day-81.md
@@ -0,0 +1,64 @@
+---
+layout: post
+title: (Day 81) RAG from scratch - chunking is very important!
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 81!
+A quick summary of today:
+* Continued with my custom RAG from scratch project - extracting knowledge from a bank's terms and conditions ([github repo](https://github.com/divakaivan/lloyds-rag-from-scratch/tree/main))
+
+So, yesterday my main struggle was reading tables. I remembered gemini can read pictures so I gave the below to gemini to try to read and give me the text. 
+
+![image](https://github.com/user-attachments/assets/414fdfd5-51ae-4439-9475-ea91809ce1e8)
+
+The output, not very good. Gemini could not read this table very well, and such tables were common in yesterday's PDF
+
+![image](https://github.com/user-attachments/assets/3153e463-8ba2-486b-b90a-c6a77a0b3f43)
+
+Yes, this is an image, but even if a powerful gemini LLM could not read this table and output it as text, for now at least, I gave up on this particular PDF, and looked for one with a bit more straigh-forward text and less tables. 
+
+The newly chosen pdf is here: [on github](https://github.com/divakaivan/lloyds-rag-from-scratch/blob/main/lbg_relationship_tnc.pdf)
+
+Now... given an allegedly more simple pdf, I used the code as it was to get outputs. But the results were just... really bad. Most times even though the scores were high, and the top-1 included the exact answer, the output was  ~'the context does not include {query} information'.
+
+I decided to go back to the start, where I preprocess the pdf, and explore more chunking mechanisms. I found [this](https://youtu.be/8OJC21T2SL4) youtube video exploring different chunking methods.
+
+It covered:
+
+1. Character split (which I was using)
+2. Recursive character split
+3. Document spcific splitting
+4. Semantic splitting
+5. Agentic splitting
+
+The youtuber mentioned that the default method he uses is the 2nd one - recursive character split, so I decided to move from 1. to 2., and explore how recursive splitting improves my process. 
+This is the final text_splitter I went with:
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=0) (tbh, I am writing this blog now and 1500 might be an overkill)
+
+I played around a lot with chunk_size and chunk_overlap - to see the effects, and I don't think I mastered knowing the impact yet. I am sure 1500 chunk_size is even that good. Frankly, in the preprocessing there are other components like splitting, regex-ing characters - maybe there are better ways to do these. It will take trial and error to figure this. 
+
+The query list I used to see results is:
+
+![image](https://github.com/user-attachments/assets/219d5920-b025-40ff-ad16-dc3f9e840803)
+
+Base prompt is like this
+
+![image](https://github.com/user-attachments/assets/b09ad68a-3e1b-4614-be05-7dd61b6b12de)
+
+I am yet to explore different prompting strategies. So far only concentrated on the preprocessing step. 
+After settling on chunk_size 1500 and chunk_overlap 0, I decided to streamline the process a bit, so I created executable preprocess_pdf.py and rag.py files - can be seen in the repo.
+
+As for results, they are incosistent... 
+
+Example is this 'What is meant by Business Day?' query.
+The top-1 context includes the answer, but some times I get the answer, sometimes I do not.
+
+![image](https://github.com/user-attachments/assets/ed6bb4c0-b349-40a4-9dec-24aaa87f189f)
+
+I shared my code on reddit and on some ML discord server, to look for advice. But tomorrow I think I shall concentrate on finding a better chunking method. 
+
+
+That is all for today!
+
+See you tomorrow :)

From f710dc1fc83a348ed3935c8f7ab84d87e35c5ea4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:41:45 +0900
Subject: [PATCH 182/638] Create 2024-3-23-day-82.md

---
 _posts/2024-3-23-day-82.md | 105 +++++++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 _posts/2024-3-23-day-82.md

diff --git a/_posts/2024-3-23-day-82.md b/_posts/2024-3-23-day-82.md
new file mode 100644
index 000000000..0c95b1c27
--- /dev/null
+++ b/_posts/2024-3-23-day-82.md
@@ -0,0 +1,105 @@
+---
+layout: post
+title: (Day 82) Looking for better parsing methods and prompting techniques for my PDF RAG
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 82!
+A quick summary of today:
+* looked online for better ways to prompt gemma-2b-it
+* switched to llama parser for my pdf preprocessing step (w.i.p.)
+
+All is on [this](https://github.com/divakaivan/lloyds-rag-from-scratch) github repo.
+
+Firstly, about the prompting techniques
+
+Before switching my base prompt, I went back to reduce the chunk_size of the langchain recursiveTextSplitter to values between 250~800. I settled on 800, thinking that 1500 is an overkill, and I saw a little improvement in the responses on the base questions.
+
+The custom query list and base prompt at this point were:
+
+![image](https://github.com/user-attachments/assets/6af055b8-ea2f-4dbe-abbe-4e1d8c065314)
+
+And answer to the queries:
+
+![image](https://github.com/user-attachments/assets/3bd16104-b801-42f0-957a-39143ba4a4c2)
+
+Next, I decided to see the results to the outputs, if I change the base prompt a bit.
+The above prompt was modified by adding 
+
+'You are a helpful assisstant to customers about our bank's terms and conditions.'
+
+And at the end, I saw that we can add these special tokens to tell the model to differentiate better between user and model's turn to respond.
+
+User query: `<start_of_turn>user{query}<end_of_turn>`
+
+Answer: `<start_of_turn>model`
+
+Below is the full prompt for reference
+
+![image](https://github.com/user-attachments/assets/adc5b686-d2fc-4585-a087-6f4049ae5f96)
+
+And now the answers to the queries are:
+
+![image](https://github.com/user-attachments/assets/ea3e5f66-8e80-4b9b-a803-a9e3584cee01)
+
+I used this website to look for prompting techniques: https://www.promptingguide.ai/models/gemma and also this post on kaggle: https://www.kaggle.com/code/sooyoungher/study-prompt-engineering-gemma-2b-it-in-langchain#Prompt-Engineering
+
+Afterwards, I changed the base prompt a bit more.
+Before, towards the end it was:
+
+'Now use the following context items to answer the user query {context}
+Relevant passages: <extract relevant pasages from the context here>'
+
+But I decided to change this a bit to:
+
+'Now use the following context items:
+{context}
+\n And answer the user's query:'
+
+![image](https://github.com/user-attachments/assets/d5268dfa-41dc-4cee-b47f-e2714af0085c)
+
+(also seen in the pic, I added a line to tell the model what to say in case it does not know, but it did not work)
+
+Now, the answer to the queries are:
+
+![image](https://github.com/user-attachments/assets/bc186317-afcb-4fd1-b89d-b1b6fc462dfe)
+
+(side note here, before I noticed that if we ask What is a Business Day? and What is a business day?, in the former case we get an answer, but in the latter we get that there is no info in the context - which is interesting that it takes into account uppercase letters like that, but in a bit you can see that this fixes itself with better base promp and later when the pdf preprocessing is done using llama parser)
+
+Next, seeing there is some improvement, I decided to improve the base prompt, give it better examples, but also reduce them from 4 to 2. 
+
+![image](https://github.com/user-attachments/assets/81e0c780-e259-42bb-b011-febba4fbb699)
+
+In addition, I included some more questions in the queries list for the model to answer.
+
+![image](https://github.com/user-attachments/assets/b7b62e2e-5163-44c9-ac92-b8a1eeaf4a0b)
+
+Output is:
+
+![image](https://github.com/user-attachments/assets/28104259-b0af-43e6-ab93-130429b80fbd)
+![image](https://github.com/user-attachments/assets/33a33274-e612-4d88-a7b0-4c245f06de96)
+
+There is definitely improvement, and this is based on the base prompt improvements, and also when preprocessing the text I commented out this line: `text = text.replace('\n', ' ').replace('  ', ' ')` which seems like had a bad effect on the then current state of preprocessing and loading data into the model. 
+
+Switching to llama index for pdf parsing
+
+Afterwards, I remember that besides langchain there I had saved one more pdf parsing method - llama index. I learned more about it from [this](https://youtu.be/oa82yoJ6zYc) video, and the gist is below.
+
+It was not so much llama index that stood out to me, but it was an idea that they talk about - parsing PDFs and turning them into html/markdown. They talked about an issue when parsing complex PDFs, that include tables, such as annual reports, we need to the model to know for sure the values in a financial statement report, if we want to give a RAG model to our stakeholders. But reading complex tables is not so easy, they said. They talked that, if we can parse the pdf and turn it into markdown, markdown can represent tables very well, and markdown tables are read well by LLMs -> this got me really excited. It means that if I can use this llama parser, I can make my cutom RAG more powerful and even give it more complex PDFs. 
+
+So I decided to go to llamaindex's page to see how to use its api, I got an api key, and the implementation was not hard. At the moment when I am writing this blog, the code is not very structured, I just wanted to see it work - and it works!
+
+The answers to the above queries list, just by switching from langchain's recursive char splitter to llamaindex's parser, are below
+
+![image](https://github.com/user-attachments/assets/b11f87ab-05fd-4dae-9c32-8c34fb24d666)
+![image](https://github.com/user-attachments/assets/f874ab46-8488-4067-a484-fcec7404de1e)
+
+At the moment using the llama API is free, but if that becomes paid in the future, going back to langchain is always an option. I will explore this more, and also just overall ways to improve the whole process ^^
+
+Finally, a small thing, if we run the executable rag.py now, we will be able to ask the LLM questions until the user enters q to exit. Not a big thing, but just feels more like a chat with a customer support bot.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 0bdf26890092c8ff02fa3d0a015d43505a542042 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:48:27 +0900
Subject: [PATCH 183/638] Create 2024-3-24-day-83.md

---
 _posts/2024-3-24-day-83.md | 55 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 _posts/2024-3-24-day-83.md

diff --git a/_posts/2024-3-24-day-83.md b/_posts/2024-3-24-day-83.md
new file mode 100644
index 000000000..82e00ac30
--- /dev/null
+++ b/_posts/2024-3-24-day-83.md
@@ -0,0 +1,55 @@
+---
+layout: post
+title: (Day 83) Summary of my PDF RAG from scratch
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 83!
+A quick summary of today:
+* Played around with llama-index and parsing, and then organised the [github repo](https://github.com/divakaivan/pdf-rag-from-scratch) to be a bit more presentable
+
+Below I will go over the project.
+
+Used pdf: [Lloyds Banking Group's Relationship T&C](https://github.com/divakaivan/pdf-rag-from-scratch/blob/main/lbg_relationship_tnc.pdf) from their website, includes high-level information about accounts, and information about the user and the Bank's rights.
+
+Embedding model: mixedbread-ai/mxbai-embed-large-v1 from huggingface
+
+Language model: google/gemma-2b-it from hugging face
+
+Repo structure:
+
+![image](https://github.com/user-attachments/assets/cb2087d4-52d1-4c15-aca5-e78522f02a31)
+
+PDF parsing: 
+
+1. For the main RAG chat (rag.py) ->  using PyMuPDF and langchain's RecursiveCharacterSplitter
+2. For the dev RAG (dev_rag.ipynb) -> llama-index and langchain's RecursiveCharacterSplitter
+
+RAG chat
+
+I call the main 'RAG chat' - these are preprocess_pdf.py and rag.py, and with them, using the terminal, a user can read, preprocess a PDF, 
+
+A short clip demo can be seen [here](https://github.com/divakaivan/pdf-rag-from-scratch).
+
+Dev PDF chat:
+
+I experimented a bit with chunk_size and chunk_overlap - the params of langchain's text splitter, and finally I settled on 1000 and 0, and those seemed to provide decent results when using llama-index as the PDF reader. Below are the sample queries and the answers provided by the model
+
+![image](https://github.com/user-attachments/assets/3ca90d17-f9f9-4671-852a-6164c4aa0dfc)
+![image](https://github.com/user-attachments/assets/579ca29d-d5ff-405b-a6c1-d92aa085baf1)
+
+However the results are not consistent, if a model starts to predict the words 'the context' the end would be that the context does not provide information about the query, but sometimes when I re-run the code, the answer actually gets generated. This could be due to several issues, but one idea that came to mind was adding beam search, because even if the model starts to generate 'the context ...' in one output, I believe in one of the others - the actual answer would appear. Actually, if we add return_context=True to the ask function, the top 1-2 context has the answer, but the model does not see it. Some other reasons - the way the context is provided (i.e. it is not clear to the model), bad query, bad base_prompt (how I tell the model to read for context), probably other reasons of which I am not aware at the moment (or I am not recalling them as reasons for problem).
+
+Instructions for setup:
+I see in other repos and just in general, setup instructions are added. I added as well and it looks nice haha
+
+![image](https://github.com/user-attachments/assets/504dcfb2-1bf7-4856-95e3-6b1b8a521460)
+
+Closing thoughts:
+This particular one uses out-of-the-box embedding and LLM, but the world out there might require me to make those from scratch too. I am still learning about RAG, I learned a lot about retrieval models from the ACL talk (Day 74, 75, 76), and I am looking forward to applying and seeing that knowledge in practice. 
+
+The github repo link is at the top, any help is appreciated ^^
+
+That is all for today!
+
+See you tomorrow :)

From 19dfcc6e1f6ec9bf9b1fc60e45c02cce0badf3a9 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 01:55:00 +0900
Subject: [PATCH 184/638] Create 2024-3-25-day-84.md

---
 _posts/2024-3-25-day-84.md | 46 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 _posts/2024-3-25-day-84.md

diff --git a/_posts/2024-3-25-day-84.md b/_posts/2024-3-25-day-84.md
new file mode 100644
index 000000000..8165cf632
--- /dev/null
+++ b/_posts/2024-3-25-day-84.md
@@ -0,0 +1,46 @@
+---
+layout: post
+title: (Day 84) Lecture 13 and 14 of CMU 11-711's Advanced NLP - Debugging and model interpretation; Ensembling methods
+categories: [theory,nlp]
+---
+
+## Hello :) Today is Day 84!
+A quick summary of today:
+* Covered Lecture 13 of CMU 11-711's Advanced NLP class: [Debugging and interpretation](https://youtu.be/c4UwOq2J9mQ?list=PL8PYTP1V4I8DZprnWryM4nR8IZl1ZXDjg), and
+* [Lecture 14](https://youtu.be/MueCRSZ3RQ0?list=PL8PYTP1V4I8DZprnWryM4nR8IZl1ZXDjg): Ensembling and mixture of experts
+
+**Lecture 13 notes:**
+
+![image](https://github.com/user-attachments/assets/183330ea-05c3-40ab-a7c1-65a0299e7fcf)
+![image](https://github.com/user-attachments/assets/7705ba4c-4b70-4b11-8822-725f160047a7)
+![image](https://github.com/user-attachments/assets/0bf0ed8d-4acc-4309-beca-38be09f50fee)
+![image](https://github.com/user-attachments/assets/0e962adc-7255-47ae-b419-466c2ef5339e)
+![image](https://github.com/user-attachments/assets/2d592b22-01d4-4cae-ab39-df8b4acfd893)
+![image](https://github.com/user-attachments/assets/ab795584-fee3-418e-8e48-d57b94487ff0)
+![image](https://github.com/user-attachments/assets/e209f372-29f7-44e3-a87d-e20064459ab4)
+![image](https://github.com/user-attachments/assets/b895c0a3-3b61-45e3-90ab-9ae83c13ce48)
+![image](https://github.com/user-attachments/assets/be5b87cc-7387-43ec-b7cd-a5332ecf6528)
+
+**Lecture 14 notes:**
+
+![image](https://github.com/user-attachments/assets/35274626-704f-4d13-af65-192f9f1591c0)
+![image](https://github.com/user-attachments/assets/1ab00170-7429-407d-9e57-51ab858ef1c1)
+![image](https://github.com/user-attachments/assets/3bd743b4-b2f6-4cc1-aad5-f0c16ea51939)
+![image](https://github.com/user-attachments/assets/5adbe54d-cfeb-49f3-9e91-d5f87bf081d5)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiiqF1Jw1m41yDObKYCH5C3RcebOyVwnnJFhifVzMoXr-LsVpZs5CB-kZT0wgXyY9ddp9n5wAZ1GXe8eGbVmR4RIpP0pk_DdEvCu_ya8ReHvgLjbQ_kcOoqsI3cM2lXNShhVQkbmAMuNDBvn6owX5AEOpMUmOsSYzYj8GiAIR99rC9_hAov2iryHnDks1ea)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgDUzMz93Fl4hLjInMkjG6_C6CZ6di_IJbjHCzkI56JhYvAbALhyGSxwXo-EQEJEci6uQzwW8diPdINvCtARIxMdFgjivOunP12wKm5ciyWIx9_gUgctT5eDFsdCkB_Wb3vvz_ENaJcYzxEw9SApaT6ftV3yVkKUx7Aj6YoFKdfyItdYJ3QC07LvLfl7qQi)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhynVAfYaThahDNMQbwYQNcp6bnx345VCkj_grUStQT0Le6VTe9bPq9Bj_-5PlBzZoumA0P_k1Ca6QZDxF4ao87DtrLExEjWhnQ1aEfXnkhVryD6BPw9uZKSgTI8WJmObc2IdTplbtKkFpz-waCHyMmsetyxNheHjV3ZejkVzdrC0BQUnHrGw3uYuobhaNg)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgNlkUgdjGDt5_rf6Sypp3d60VLNTS5LDLKkE7JVPcqHQMRba61wbhtS-oesP-ryKY4otUDjG4mXzvftQxgdyYdgDBHjOGKdWwkxroT_ccaOe-K6X4NZaFtjFLIxoIcQUEil0a8NWXAZiBlytXwKvN6zzzzaYWhaR2AD9ht6GuGI7zYPYzFxlE5TuA_zVFz)
+
+I am on the look out for the next lectures. All of the next 4 are really exciting
+15. Tour of Modern Large Language Models
+16. Long Sequence Models
+17. Code Generation
+18. Knowledge-based QA and Information Extraction
+But maybe we will get them in a month or so :(
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 9bbe087a92f62a5f836e9c52d45d7a553688babb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:05:58 +0900
Subject: [PATCH 185/638] Create 2024-3-26-day-85.md

---
 _posts/2024-3-26-day-85.md | 151 +++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 _posts/2024-3-26-day-85.md

diff --git a/_posts/2024-3-26-day-85.md b/_posts/2024-3-26-day-85.md
new file mode 100644
index 000000000..8de69f900
--- /dev/null
+++ b/_posts/2024-3-26-day-85.md
@@ -0,0 +1,151 @@
+---
+layout: post
+title: (Day 85) How to write a great research paper
+categories: [theory,research]
+---
+
+## Hello :) Today is Day 85!
+A quick summary of today:
+* An amazing [talk](https://simon.peytonjones.org/great-research-paper/) about how to write a great research paper by Simon Peyton Jones
+* Submitted a draft for a research proposal for next semester
+
+I actually saw this talk after, writing the short proposal, but still it kind of motivated me for next semester.
+
+Tips on how to write a great research paper by Simon Peyton Jones ([talk](https://simon.peytonjones.org/great-research-paper/))
+
+1. Dont wait, write!
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiyMatDiB3Hmsy6Wxe_l65LqKVPCWQ7mRYUQ14xwX2U5YfpzbT4wsec_EAcN3PNjzU2q-JdTJPXhBL6TS2iuAoj0ityiYrESQ9hpD10MSYsYzL9j9Dp9maglLW1fdShZQuXw0TPQE6ge-Kl52TMcRk-IwcJeTk9gwXvTdh8QLpbkDcBRbAtoBWYrhBq2C-2)
+
+* This forces is to be clear, focused
+* Crystallises what we don't understand
+* Opens the way to dialogue with others: reality check, critique, and collaboration
+* Writing is the way to develop your idea
+
+2. Identify your key idea!
+
+* Your goal: to convey a useful and re-usable idea
+* You want to infect the mind of your reader with your idea, like a virus
+* Papaers are far more durable than programs
+* Do not be intimidated
+  * Fallacy: you need to have a fantastic idea before you can write a paper
+  * Don't wait to have a good idea. Write stuff about any idea
+  * We only discover ideas are good later
+* The idea
+  * Your paper should have just one 'ping': one clear, sharp idea
+  * You may not know exactly what the ping is when you start writing; but you must know when you finish
+  * If you have lots of ideas, write lots of papers
+* Can you hear the 'ping' ?
+  * Many papers contain good ideas, but do not distil what they are
+  * Make certain that the reader is no doubt what the idea is. Be 100% explicit!
+    * 'The main idea of this paper is ...'
+    * 'In this section we present the main contributions of the paper ...'
+
+3. Tell a story
+
+* Imagine you are explaning at a whiteboard
+  * Here is a problem
+  * It's an interesting problem
+  * It's an unsolved problem
+  * Here is my idea
+  * My idea works (details, data)
+  * Here's how my idea compares to other people's approaches
+* Structure
+  * Title (1000 readers)
+  * Abstract (4 sentences, 100 readers)
+  * Introduction (1 page, 100 readers)
+  * The problem (1 page, 10 readers)
+  * My idea (2 pages, 10 readers)
+  * The details (5 pages, 3 readers)
+  * Related work (1-2 pages, 10 readers)
+  * Conclusion and further work (0.5 pages)
+
+4. Nail your contributions to the mast
+
+* The introduction (1 page)
+  * Describe the problem
+    * Consider molehills not mountains, for problems to solve
+    * Use an example to introduce the problem
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjhcmWb-FV2-fKLRqjkEBB-VAlXE6M9MSFaet-d2drURFzyEw21z3GMxJfk9tPwaKHyTlqIzpbzCycg5wxx4wLaXPilOVPEnuwkM4OLOUKfWWlzHNAbTme1g7Ih0R6pnyFZkEqv3zx04rvcZsN-l_rRZEOEJgAfWe9KUvUPK1RaNk_hQBJ8yLPa6nGs-aaO)
+
+  * State your contributions
+    * Write the list of contributions first
+    * The list of controbutions drives the entire paper: the paper substantiates the claims you have made
+    * Reader thinks 'gosh, if they can really deliver this, that's amazing; I'd better read on'
+    * Clear, bulleted list of contributions
+    * Contributions should be refutable
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgCqf5NMpw5DHkmhZBeRvLSDkoFOQIvIkFPfqp1DO3wDLhvG3aBcdZwZfgiDPywXjN8HOcWxgqj4ms6oeDqLRrPRhqcwfTAPSa3c2qo0Vdr8GJ4ssqOVvhaJKVMbVvAKCSnHf6cO8d2brxvh3PNF-JP1YA4th1ZTin79ZqJnxdIVLTdlHQ7cEJI1_Ih0q9b)
+
+  * No 'rest of this paper is...'
+  * Instead, use forward references from the narrative in the introduction. THe introduction (including contributions) should survey the whole paper, and therefore forward reference every important part
+
+5. Related work: later
+
+* No related work yet!
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgjg6HbyWpGgx1M7Q3qAN2wgM8nrr2LuDXm0EskODvpB9ExNT38ichMsFi575tDA7jrxmYuJD8XT-MUyfp5oVBH-h3T3wsWe38NVBCd-l-v1UwJYTMCDMbvQr-ThmE1L_VbaJgmXgOL3cUdvEqv5O_eopkQpjW8Y06Xx4eGTgESNYb0vljVIlJBO4VumaRH)
+
+* Problem 1: the reader knows nothing about the problem yet; so your (highly compressed) description of various technical tradeoffs is absolutely incomprehensible
+* Problem 2: describing alternative approaches gets between the reader and your idea
+* Credit
+  * Fallacy: to make my work look good, I have to make other people's work look bad
+  * The truth: credit is not like money
+    * Warmly aknowledge people who have helped you
+    * Be generous to the competition
+    * Aknowledge weaknesses in your approach
+    * Giving credit to others does not diminish the credit you get from your paper!
+
+6. Put your readers first
+
+* Structure
+  * The idea: Consider a bifircuated semi-lattice D, over a hyper-modulated signature S. Suppose pi is an element of D. Then we know for every such pi there is an epi-modulus j, such that pj < pi.
+  * Sounds impressive, but
+  * Sends the readers to sleep, and/or makes them feel stupid
+  * When writing, don't get stuck explaining something that is not worth reading. If there is an obvious/popular path that was not taken, explain why to the reader.
+
+* Presenting the idea
+  * Explain it as if you were speaking to someone using a whiteboard
+  * Conveying the intuition is primary, not secondary
+  * Once your reader has the intuition, they can follow the details (but not vice versa)
+  * Even if they skip the details, they still take away something valuable
+
+* Conveying the intuition
+  * Introduce the problem, and your idea, using EXAMPLES and only then present the general case
+  * General -> specific
+* Do not recapitulate your personal journey of discovery. This route may be soaked with your blood, but that is not interesting to the reader
+* Instead, choose the most direct route to the idea
+
+7. Listen to your readers
+
+* Getting help
+  * Experts are good
+  * Non-experts are also very good
+  * Each reader can only read your paper for the first time once! 
+  * Explain carefully what you want ('I got lost here' is much more important than 'Pythn is mis-spelt')
+  * Get your paper read by as many friendly guinea pigs as possible
+* Getting expert help
+  * A good plan: when you think you are done, send the draft to the competition saying 'could you help me ensure that I describe your work fairly?'
+  * Often they will respond with helpful critique
+  * They are likely to be your referees anyway, so getting their comments or criticism up fron is jolly good
+* Listening to your reviewers
+  * Treat every review like the hold dust - be (truly) grateful for criticism as well as praise
+  * This is really, really, really hard, but really, really, really, really, really, really, really, important
+  * Read every criticism as a positive suggestion for something you could explain more clearly
+  * DO NOT respond 'you stupid person, I meant X'
+  * INSTEAD, fix the paper so that X is apparent even to the stupidest reader
+  * Thank them warmly. They have given up their time for you
+
+Now about the proposal ~
+
+It is mostly based on the [prompt engineering](https://easychair.org/publications/preprint/Dmrt) literature review I wrote last semester, submitted and presented at the Korea Conference on Software Engineering 2024. The topic name I wrote is Analyzing ChatGPT's Financial Statements Understanding Using Different Prompt Engineering Strategies. The idea is basically to analyze ChatGPT's financial statements understanding using different prompt engineering strategies. The 'beta' plan I submitted is to do a literature review prompting strategies used for ChatGPT, along with an examination of papers regarding financial statements Q&A and stakeholders' information preferences. Subsequently gather some of the financial statements referenced in popular literature alongside stakeholders' questions and points of interest. Use popular evaluation metrics such as ROUGE for summarization, and precision, recall and  accuracy for answers. It is quite basic but I am excited to start doing some kind of research.
+
+Also, when I went to my professor to talk about it, he asked me whether I know about langchain, fine-tuning and my general knowledge of machine learning. I was so excited when I said yes because that is what I have been studying these so far 85 days, but the best thing is, is that there is so much more to learn, a lot to practice and test my knowledge.
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From cbebfe412d8e2b9937672b654c01b721d91ef231 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:09:31 +0900
Subject: [PATCH 186/638] Create 2024-3-27-day-86.md

---
 _posts/2024-3-27-day-86.md | 83 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 _posts/2024-3-27-day-86.md

diff --git a/_posts/2024-3-27-day-86.md b/_posts/2024-3-27-day-86.md
new file mode 100644
index 000000000..94bdfd595
--- /dev/null
+++ b/_posts/2024-3-27-day-86.md
@@ -0,0 +1,83 @@
+---
+layout: post
+title: (Day 86) Made a youtube video - Chat with your PDF for free in colab using huggingface, mongodb, llama_index, langchain
+categories: [applying-knowledge,nlp]
+---
+
+## Hello :) Today is Day 86!
+A quick summary of today:
+* Coded, planned, recorded and posted a [video tutorial](https://youtu.be/U6p6Gh5k47E) making a chat with your pdf rag system for free
+* All code + colab link + pdf used is on [this](https://github.com/divakaivan/RAG_with_your_PDF_for_free_in_colab) github repo
+
+Well, after waking up today, I definitely did not expect to plan, execute and upload an almost 1hr tutorial on youtube. 
+
+I was looking around chat with your PDF videos, to see what I can improve in my [pdf_rag_from_scratch](https://github.com/divakaivan/pdf-rag-from-scratch) but I saw that most of the videos require an OpenAI api key, and I did not like that, given the availability of so many free resources and models. 
+
+And I found this great resource from [huggingface](https://huggingface.co/learn/cookbook/en/rag_with_hugging_face_gemma_mongodb) - Building A RAG System with Gemma, MongoDB and Open Source Model. Instead of a pdf, they were using some dataframe for films, so I decided to improve upon that, and make the code preprocess a pdf, embed it, upload to mongodb, load gemma, create a prompt and chat with the pdf (kind of a combination of the tutorial + my pdf_rag_from_scratch). 
+
+The code itself is not that complicated, but I wanted to write it once/twice to make sure when I write live in the video recording, I do not have problems. So the whole process from idea to published video maybe took me 8 hours, mind that I had to find an app to edit the video (the editing was not much, but the app's video processing time was long because I wanted it in 1080p). 
+
+Anyway ~ below I will provide an overall summary of the code
+
+1. Download libraries
+
+![image](https://github.com/user-attachments/assets/6cacf231-0c6d-4edd-b33a-b337b06804ca)
+
+2. Preprocess PDF
+
+2.1 Load PDF with llama-index
+
+![image](https://github.com/user-attachments/assets/69aee3ae-b62f-4101-b5f0-3aeff40a9c1c)
+
+2.2 Chunk PDF text using langchain
+
+![image](https://github.com/user-attachments/assets/23a54f91-fb35-4615-b1f9-754b1b330772)
+
+2.3 Embed chunks
+
+![image](https://github.com/user-attachments/assets/ac82d5f5-105f-43b2-8432-03346ae73c96)
+
+3. Set up mongodb
+
+An important part is to set up an atlas vector search
+
+![image](https://github.com/user-attachments/assets/d3d59a8e-bc71-4db4-b5b1-37a4d87d6f9f)
+
+3.1 Connect to the db
+
+![image](https://github.com/user-attachments/assets/e8f1d67e-f15f-45d9-a155-67fc1056e266)
+
+3.2 Delete existing (if any), and insert data
+
+![image](https://github.com/user-attachments/assets/60c10864-ea91-4472-9a6e-77367f16383f)
+
+4. Find relevant texts
+
+4.1 Perform vector search in db + get context
+
+![image](https://github.com/user-attachments/assets/1d4bf291-eb0b-4417-953d-d50cf2893d5b)
+![image](https://github.com/user-attachments/assets/f9a715b3-26aa-41f8-888d-9b1cd0ff48c9)
+
+5. Load gemma using huggingface
+
+![image](https://github.com/user-attachments/assets/b3432566-c504-4527-9e61-0c73a11135fe)
+
+6. Prompt engineering + talk with your PDF
+
+I used similar base_prompt with the pdf_rag_from_scratch
+
+![image](https://github.com/user-attachments/assets/8e57b588-bbb6-4240-889a-fa84fe8def73)
+![image](https://github.com/user-attachments/assets/6416a1ef-dd4b-4d15-8359-6f6e86079d85)
+
+Query: Do you pay or charge interest?
+Answer: Yes, the Core Banking Agreement states that interest is paid and charged on a daily basis, and the interest rate applicable to your account(s) is stated in the Product & Services Terms & Conditions or, if no such terms are provided, on the website.
+
+
+
+The results are not perfect, but is a good starting point for fine-tuning. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 8ad7c9992b483a7a116a531bd6ac55058e566475 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:14:49 +0900
Subject: [PATCH 187/638] Create 2024-3-28-day-87.md

---
 _posts/2024-3-28-day-87.md | 105 +++++++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 _posts/2024-3-28-day-87.md

diff --git a/_posts/2024-3-28-day-87.md b/_posts/2024-3-28-day-87.md
new file mode 100644
index 000000000..fcb9d1475
--- /dev/null
+++ b/_posts/2024-3-28-day-87.md
@@ -0,0 +1,105 @@
+---
+layout: post
+title: (Day 87) Registered for Stanford's XCS224W - Machine Learning with Graphs + RAG webapp with llama-index tutorial
+categories: [applying-knowledge,theory,nlp]
+---
+
+## Hello :) Today is Day 87!
+A quick summary of today:
+* after a lot of thought and considerations I finally registered for one of Stanford's AI professional program courses - [XCS224W](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs)
+* Went over fresh published by deeplearning.ai - [JavaScript RAG Web Apps with LlamaIndex](https://learn.deeplearning.ai/courses/javascript-rag-web-apps-with-llamaindex/lesson/1/introduction)
+
+Maybe 2 weeks ago I did the application for Stanford's AI professional program, and since I got accepted I have been thinking about taking one of the courses. ML with Graphs is one of primary interest to me because of its application in financial fraud, but more than that - it is a lot of money (thankfully my parents support me taking such high level education courses). Yes the quality, it is top notch, but besides Stanford's AI professional certificate program which includes the below courses (to get the certificate I need to complete 3)
+
+![image](https://github.com/user-attachments/assets/1908b961-feb7-442a-8ce4-f5faac3bd32e)
+
+I was heavily considering [MITx MicroMasters in Statistics and Data Science](https://micromasters.mit.edu/ds/). Online there are a lot of reviews of it being very hard, requiring a lot of hours, and this really made me interested in doing it. The MicroMasters covers:
+
+![image](https://github.com/user-attachments/assets/18735e86-5cec-4efa-8e74-bd6ac9bb8ab8)
+
+4 courses and 1 capstone exam, and the price is significantly lower than even 1 of stanford's pro certificate courses. But the thing is, it covers topics that I have studied extensively so far, and moreover, will cover in my statistics class at uni this semester (probability and statistics). I asked in reddit, discord servers with ML students. 
+This was probably half my day, and then I discussed it with my parents, and they supportively said it is okay whatever I decide. 
+
+I thought about it, and because I know Stanford quality, I have taken the free versions of their courses, I know the quality, I will commit and trust them. Later, the MicroMasters courses are significantly cheaper, so if I decide to try them, I can.
+
+So ~~~ ML with Graphs officially starts on the 1st of April and I am looking forward to learning about graphs, be able to chat with other students.  
+
+
+
+Now, about the RAG react webapp with llama-index
+
+It was in deeplearning.ai's platform and sadly it uses an OpenAI api key (though they provide one for free). To set up the app we can do the following steps:
+
+1. Set up documents, index and query engine
+
+![image](https://github.com/user-attachments/assets/938f198b-7126-425e-9ebd-adf430533b56)
+
+2. Create a query handler
+
+![image](https://github.com/user-attachments/assets/cd9bffe4-9730-4756-b2f2-8321c1b752eb)
+
+3. Set up a react frontend
+
+![image](https://github.com/user-attachments/assets/925ec80c-3e84-43e8-9e76-c8f92f07ee35)
+
+And the results are:
+
+![image](https://github.com/user-attachments/assets/cebae1ca-12b5-482c-b9f2-db543231b338)
+![image](https://github.com/user-attachments/assets/48c7f9b2-edf4-45ad-ac5e-19baa1b63302)
+![image](https://github.com/user-attachments/assets/4f91a43f-00bd-43de-aefd-8ebe7a0814ad)
+
+Then, the teacher talked about advanced querying where we can use and get info from multiple data sources.
+
+![image](https://github.com/user-attachments/assets/49bb3811-6e46-4a2c-8e1d-1188014359e0)
+
+We set up the code with 2 query engines: 1) for questions about Dan Abramov, and 2) for questions about react
+
+![image](https://github.com/user-attachments/assets/d5d86eb5-c667-470b-805b-08e5530a83dd)
+
+And when we query the queryEngine with queries about either item, we can get:
+
+![image](https://github.com/user-attachments/assets/c1da5ffd-fde8-42f2-a46c-490901768c3b)
+
+Next, we learn about agents - creating arbitrary functions and an agent knows how to use those
+
+Example:
+
+1. Define a simple function
+
+![image](https://github.com/user-attachments/assets/0efd3a1d-ea34-4d72-b715-6bd481d0f5d2)
+
+2. Explain the function in a JSON structure
+
+![image](https://github.com/user-attachments/assets/47b30ee1-b499-43d7-83c9-7bec8aa92c4d)
+
+3. Turn the function into a tool
+
+![image](https://github.com/user-attachments/assets/47d49826-e65d-42d0-bb5d-37549a7a969f)
+
+4. Turn the router query engine into a tool too and combine the previously created queryEngine (the one that can answer Qs about Dan Abramov and react)
+
+![image](https://github.com/user-attachments/assets/4370dbb3-7836-4348-b918-2ced71effd54)
+
+5. Query and get a respone
+
+![image](https://github.com/user-attachments/assets/aaaa9a24-2a3f-4914-aa26-85012449861c)
+
+And finally, all of the above was combined to created a RAG React webapp with llama-index
+
+![image](https://github.com/user-attachments/assets/26844827-36e7-4928-a0de-cba10f188939)
+
+As for some other materials, I found these:
+
+[An Introduction to Statistical Learning](https://www.statlearning.com/) - and I might go over it, taking notes by hand.
+
+[Deep Learning with PyTorch Step-by-Step A Beginner's Guide](https://leanpub.com/pytorch) - maybe, if I can check out what's inside. 
+
+[XCS224U](https://www.youtube.com/playlist?list=PLoROMvodv4rOwvldxftJTmoR3kRcWkJBp) - Natural Language Understanding - all lectures and homeworks.
+
+So there are definitely things to do. ^^
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From d709a8f66b9327bb80f002cb290fa7a812193756 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:19:55 +0900
Subject: [PATCH 188/638] Create 2024-3-29-day-88.md

---
 _posts/2024-3-29-day-88.md | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 _posts/2024-3-29-day-88.md

diff --git a/_posts/2024-3-29-day-88.md b/_posts/2024-3-29-day-88.md
new file mode 100644
index 000000000..5e40f4462
--- /dev/null
+++ b/_posts/2024-3-29-day-88.md
@@ -0,0 +1,52 @@
+---
+layout: post
+title: (Day 88) Starting the book 'An Introduction to Statistical Learning' - Chapter 2 and 3
+categories: [theory,statistics]
+---
+
+## Hello :) Today is Day 88!
+A quick summary of today:
+* Covered Chapter 2: Statitiscal learning, and
+* Chapter 3: Linear regression of the infamous [ISLP](https://www.statlearning.com/) book
+
+Today I decided to go over this book that I keep seeing recommended for the basics of machine learning. As a fan of basics, it grabed my interest. 
+My notes are below:
+
+**Chapter 2: Statistical learning**
+
+![image](https://github.com/user-attachments/assets/73313825-c12b-4367-b5cc-59159cabc7fa)
+![image](https://github.com/user-attachments/assets/d7a55670-9f7a-4066-ba61-27f22fb76f8a)
+![image](https://github.com/user-attachments/assets/e9851761-04ce-40ff-9feb-678de89ae6f5)
+![image](https://github.com/user-attachments/assets/b2ef02f4-2e4f-4751-8595-26c79f24f796)
+![image](https://github.com/user-attachments/assets/de1e638c-eb97-4584-962c-b23adfb30867)
+![image](https://github.com/user-attachments/assets/026ee6ce-1437-4cef-86f8-5484fc2f69c7)
+![image](https://github.com/user-attachments/assets/dd82c614-7b8e-4532-9dd5-cf3fa9ff5b45)
+![image](https://github.com/user-attachments/assets/e998eaf3-2680-4ba8-a352-bbc8bab16ae5)
+![image](https://github.com/user-attachments/assets/96de9571-cf12-48eb-8d5d-eb1de14ec5f5)
+
+**Chapter 3: Linear regression**
+
+![image](https://github.com/user-attachments/assets/e0840a4c-2698-448d-9b11-7502fd37553b)
+![image](https://github.com/user-attachments/assets/c1980505-d7a4-4b25-90cd-8da07a51977a)
+![image](https://github.com/user-attachments/assets/e41ae1c2-0443-4859-9e0b-9863272f1ae1)
+![image](https://github.com/user-attachments/assets/ae27870b-1637-4dda-bffc-c9d1fee656ac)
+![image](https://github.com/user-attachments/assets/cc6ce575-86b7-4e58-89a1-b062d09b0cd3)
+![image](https://github.com/user-attachments/assets/06a8d09f-50a0-46d4-9050-388caccc369a)
+![image](https://github.com/user-attachments/assets/371d188e-ec7a-41ba-bf14-5135f8a5263b)
+![image](https://github.com/user-attachments/assets/fe2ddf41-2c27-411e-92f7-34c2206c1cba)
+![image](https://github.com/user-attachments/assets/02548440-ad12-44ea-ab23-92c7fb31e35f)
+![image](https://github.com/user-attachments/assets/21dd8a17-d45d-4217-98d7-48aa10fcde82)
+![image](https://github.com/user-attachments/assets/47a5149f-9023-42c6-9c64-806f0678c2a4)
+![image](https://github.com/user-attachments/assets/8857d8bf-cd11-4478-956c-c18e5bcc1b44)
+![image](https://github.com/user-attachments/assets/6dcff245-211f-4bf3-93b7-0df65d862173)
+![image](https://github.com/user-attachments/assets/cd3d02f5-2d00-413d-a71a-e92acb33c7d6)
+![image](https://github.com/user-attachments/assets/2291cf32-0340-4f2a-ba5e-ccde5ea61623)
+![image](https://github.com/user-attachments/assets/9a86de1c-8d2a-4032-bef6-f740de0a6e4d)
+![image](https://github.com/user-attachments/assets/ca4e8d8f-73ab-44a9-aca2-9e0e9fa4dccb)
+![image](https://github.com/user-attachments/assets/1c962d0b-722f-45c0-b5b5-f22d7ce4c82b)
+![image](https://github.com/user-attachments/assets/8ff9d43b-abae-4bf3-ad5b-67d27c500833)
+
+
+That is all for today!
+
+See you tomorrow :)

From a3803ede8c4baf8bed38988b64ab73886dc62565 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:24:50 +0900
Subject: [PATCH 189/638] Create 2024-3-30-day-89.md

---
 _posts/2024-3-30-day-89.md | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 _posts/2024-3-30-day-89.md

diff --git a/_posts/2024-3-30-day-89.md b/_posts/2024-3-30-day-89.md
new file mode 100644
index 000000000..3bc4233a5
--- /dev/null
+++ b/_posts/2024-3-30-day-89.md
@@ -0,0 +1,52 @@
+---
+layout: post
+title: (Day 89) More basics from ISLP
+categories: [theory,statistics]
+---
+
+## Hello :) Today is Day 89!
+A quick summary of today:
+* covered Classification, Resampling methods, Tree-based models and Some considerations in high dimensions from [An Introduction to Statistical Learning](https://www.statlearning.com/)
+
+Besides the below and yesterday's notes, for not I just read some of the other material of the book. For a while I had the Probabilistic ML: An introduction by Kevin Muprhy on my list (saw it recommended a lot), and it covers 
+
+![image](https://github.com/user-attachments/assets/d7a8c386-a37e-42ce-b03d-f15d970b18d7)
+
+and I am particularly interested in the Foundations part, seeing how the basics are 'born'. But I will start that tomorrow ^^
+
+So ~ here are my notes from the ISLP book for the 4 chapters covered today:
+
+**Chapter 4: Classification **
+
+![image](https://github.com/user-attachments/assets/b28b96ba-ab60-41f2-90a0-99227d86c834)
+![image](https://github.com/user-attachments/assets/b843cdc6-b323-4258-9358-1e9b539484a2)
+![image](https://github.com/user-attachments/assets/3c8b7bb6-348b-4042-b252-4f798e6a6c6b)
+![image](https://github.com/user-attachments/assets/06dd0556-2446-4f92-b0ce-fb704b45079c)
+
+**Chapter 5: Resampling methods**
+
+![image](https://github.com/user-attachments/assets/69ba4973-3cd9-41ee-85b4-511a72f315c3)
+![image](https://github.com/user-attachments/assets/af2bbc4c-2cf9-49c6-b496-87cdbe92be4e)
+![image](https://github.com/user-attachments/assets/56144fbc-8ab6-4bd2-9163-433fe6a55535)
+
+**Chapter 6: Considerations in high dimensions**
+
+![image](https://github.com/user-attachments/assets/a0d81029-e100-454b-9d43-df48072ccb96)
+![image](https://github.com/user-attachments/assets/15b39cfd-372e-480b-bba8-e418b189cb9d)
+
+**Chapter 8: Tree-based methods **
+
+![image](https://github.com/user-attachments/assets/eb9018fa-c407-4c9e-86bb-a0500fd22db3)
+![image](https://github.com/user-attachments/assets/31cdc4e7-2947-484c-97c3-70407ba39731)
+![image](https://github.com/user-attachments/assets/34e9b148-09bb-4c33-bce0-be7c388a3a9c)
+![image](https://github.com/user-attachments/assets/d1775a83-46b4-4bce-9333-cca6a213cfeb)
+![image](https://github.com/user-attachments/assets/a4fc0168-998b-4ee1-a003-0170f661b8ad)
+![image](https://github.com/user-attachments/assets/04b5ce1b-5691-4c88-b230-3202fe793492)
+
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 3433ae3765ff3078ebaa90c88c518c5ad650f7d5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:30:21 +0900
Subject: [PATCH 190/638] Create 2024-3-31-day-90.md

---
 _posts/2024-3-31-day-90.md | 45 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 _posts/2024-3-31-day-90.md

diff --git a/_posts/2024-3-31-day-90.md b/_posts/2024-3-31-day-90.md
new file mode 100644
index 000000000..cc2e0cc8e
--- /dev/null
+++ b/_posts/2024-3-31-day-90.md
@@ -0,0 +1,45 @@
+---
+layout: post
+title: (Day 90) Probability - Univariate Models and colab 0 from XCS224W - ML with Graphs
+categories: [statistics,theory,gnn,applying-knowledge]
+---
+
+## Hello :) Today is Day 90!
+A quick summary of today:
+* covered the first chapter of the Foundations part of [Probabilistic Machine Learning: An Introduction](https://probml.github.io/pml-book/book1.html)
+* set up my environment and covered the basics from the 0th colab for Stanford's XCS224W: ML with Graphs
+
+**Probabilistic Machine Learning: An Introduction**
+
+**Chapter 2: Probability: Univariate Models**
+
+![image](https://github.com/user-attachments/assets/b45697d2-7070-45cd-a53a-fe2819bcee4e)
+![image](https://github.com/user-attachments/assets/51635ca0-9221-4409-8334-6dd1c27234cb)
+![image](https://github.com/user-attachments/assets/0d8c7514-ad6d-43ed-a7a1-0e17731ced03)
+![image](https://github.com/user-attachments/assets/9d7d9799-5bcf-466f-b66d-355f4e26ad58)
+![image](https://github.com/user-attachments/assets/a1563441-bb90-403e-a29d-db4b7897341c)
+![image](https://github.com/user-attachments/assets/cd79d48d-6ecc-4f68-9b98-91156447d4fc)
+![image](https://github.com/user-attachments/assets/877bdb21-d1c0-46a7-96fc-25cc13ea5d7a)
+![image](https://github.com/user-attachments/assets/5a214ec4-17eb-41e2-bf7f-2abc70244cd3)
+![image](https://github.com/user-attachments/assets/e42f44a1-fc63-40f0-9166-b46ba2f32702)
+![image](https://github.com/user-attachments/assets/fd8b232d-0eab-4483-a447-a8b2999f2bdd)
+
+Today I got added to the slack channel for XCS224W: ML with Graphs by Stanford, and I went over setting up the environment and over the 0th colab - some basics about graphs. I am not allowed to share any of the code from the colabs, but I found this [similar tutorial](https://mlabonne.github.io/blog/posts/2022-02-20-Graph_Convolution_Network.html) that is public and I can talk over how I improved upon it. (I need to go over the animation part of this tutorial and showing the nodes in 3d space)
+
+To avoid any risk of a lawyer sending me an email, I will just say how I improved upon the above tutorial. 
+
+After training a GNN on KarateClub data from pytorch_geometric, the result looked like
+
+![image](https://github.com/user-attachments/assets/6e1b1a8f-1aa3-4200-a3d1-034359bee467)
+
+And there was a conclusion under it saying how even a simple model, can learn to separate and classify the nodes. Given that final visualization, I was not sure how one can come up with that conclusion, so I looked around to learn a bit more about the dataset, and how graphs work, and the final new visualization I got is:
+
+![image](https://github.com/user-attachments/assets/a3af7b66-8198-4aa1-9cc1-b4aade7931ed)
+
+
+And now we can easily make the conclusion that our simple GNN classifies almost all nodes correctly and separates them well enough.
+
+
+That is all for today!
+
+See you tomorrow :)

From 8a80ca762922d28d66d27755647a9502b6cc0043 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:33:41 +0900
Subject: [PATCH 191/638] Create 2024-4-1-day-91.md

---
 _posts/2024-4-1-day-91.md | 44 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 _posts/2024-4-1-day-91.md

diff --git a/_posts/2024-4-1-day-91.md b/_posts/2024-4-1-day-91.md
new file mode 100644
index 000000000..b4c5a6c1e
--- /dev/null
+++ b/_posts/2024-4-1-day-91.md
@@ -0,0 +1,44 @@
+---
+layout: post
+title: (Day 91) Probability: Multivariate models and Statistics chapters from Probabilistic Machine Learning
+categories: [theory,statistics]
+---
+
+## Hello :) Today is Day 91!
+A quick summary of today:
+* Covered Chapter 3: Probability: Multivariate models, and
+* Chapter 4: Statistics of [Probabilistic Machine Learning](https://probml.github.io/pml-book/book1.html) by Kevin Murphy
+
+It is April 1st, but the Stanford XCS224W: ML with Graphs starts on 1st of April 9am California time, which is 1am for me (in a few hours). So I guess I can start covering the lectures from tomorrow. (exciting)
+
+As for the Probabilistic ML book, I still have a few chapters to read/cover
+
+![image](https://github.com/user-attachments/assets/dfc7d742-6e29-467c-aecf-b2d0a9fbce52)
+
+Below are my notes from chapter 3 and 4
+
+**Chapter 3: Probability: Multivariate Models**
+
+![image](https://github.com/user-attachments/assets/6e9e8f84-2004-4539-92ee-46860e63cd70)
+![image](https://github.com/user-attachments/assets/f212f1a0-fb84-460a-abd0-736d50c4f572)
+![image](https://github.com/user-attachments/assets/673088e6-70a2-40e7-afa5-38c6cf5aec5f)
+![image](https://github.com/user-attachments/assets/0ca5a845-f1ea-47b9-88df-97ee52c9783d)
+![image](https://github.com/user-attachments/assets/09015084-5de1-4c07-ae02-5302784fb58a)
+
+**Chapter 4: Statistics**
+
+![image](https://github.com/user-attachments/assets/85fd7594-d16e-470f-b2ca-505dc4252a8a)
+![image](https://github.com/user-attachments/assets/69086f1e-e849-4db2-8e16-0b240ec39ea7)
+![image](https://github.com/user-attachments/assets/61d9854e-bd26-4a1f-9348-40ae5721bd75)
+![image](https://github.com/user-attachments/assets/6c7139b0-7049-4350-b41b-d99c4f190b78)
+![image](https://github.com/user-attachments/assets/970d4c4c-e0a6-42a6-8232-4c1182176807)
+![image](https://github.com/user-attachments/assets/92186306-abd6-40ee-ab70-5585cb1f3e34)
+
+
+That is all for today!
+
+See you tomorrow :)
+
+
+
+p.s. today in my uni statistics class we went over Bernoulli, Binomial, Geometric, Poisson distributions and it was nice talking with the professor and hearing an explanation in person haha (up unitl now every explanation has been through a video or google/chatgpt)

From d4a7f05fbaa5324812f9032e628be1a244f2c12b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:40:30 +0900
Subject: [PATCH 192/638] Create 2024-4-2-day-92.md

---
 _posts/2024-4-2-day-92.md | 57 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 _posts/2024-4-2-day-92.md

diff --git a/_posts/2024-4-2-day-92.md b/_posts/2024-4-2-day-92.md
new file mode 100644
index 000000000..240176a57
--- /dev/null
+++ b/_posts/2024-4-2-day-92.md
@@ -0,0 +1,57 @@
+---
+layout: post
+title: (Day 92) Starting the official Stanford XCS224W: ML with Graphs
+categories: [theory,gnn]
+---
+
+## Hello :) Today is Day 92!
+A quick summary of today:
+* got access to the lectures + assignments of XCS224W: ML with Graphs on Stanford's platform, and covered Lecture 0: Intro to ML with Graphs, and
+* Lecture 1.1: Traditional ML on Graphs
+
+Similar lectures may be found on [youtube](https://www.youtube.com/watch?v=JAB_plj2rbA&list=PLoROMvodv4rPLKxIpqhjhPgdQy7imNkDn). 
+
+Learned about graphs and their application, nodes, edges, adjacency matrices, different types of graphs (undirected, directed, weighted, unweighted), features that help predict nodes, links between nodes and entire graphs, graphlets, clustering coefficients, way more in the notes below ^^ 
+
+**Lecture 0: Intro to ML with Graphs**
+
+![image](https://github.com/user-attachments/assets/15e3b316-31f1-4b59-ab59-c359c69c8592)
+![image](https://github.com/user-attachments/assets/4e20ce91-55d5-4b60-bd94-bf4937a6e3c6)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiVT8r16a12gC_138kqlpuWQxnlQuxnWeDzWNB2cWV6Mn6L0ARzys1eT2jNKUTUwjcagBLpTT5DE6P0pUitSerEi2ssYwvpz4S974ppPXiV-LJNtVM3xQaqBDWleiCYMLNIIAYtr-7pOeHdV6QuE-Q56PPQ3uf9OgDHGgQZUDgeI0ZApT7xhmklcVUCIyV2)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhS1y9zEqY_DGB65ri2znU6FpD_nL4kwHA9FPhHmBnpe4e2ySUsT0PFnEQEtOP4ONNZR8hD2hEYCuN6Oj6ftsfbkdX7uaC6RgkAyNNnfink_gn48jFXu21InN044m_5Amun0uHdpbdFk5kkjG6ij2OyXsOLmQVzkWOH3srEBqjbcuEMr-h4aN6_E5Fu7huX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjIHEYNKLEv1cqZTCm_LvFj2eJdLxktmJjZhG_-kiJDV0A4qCO3O6PgBD3Onj0pCFOu7Kow_si8ei5FdMFlKBgebYpTUVHs-TJJf40HjTBMvwvBXUpyQBPXdtX0sNB7pm9fX4RsC8uJR2ntHnxN5nUHV-LTsBb0g3FI9kOvXV5LhjtIoCbWrcBCcfRj9Oru)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgIRaEz667Y8Tw-ytMzITYRxLWqDWXhlqBpXMlYFjHKhW0PfdUMmmOLuNwbakEfuJdXaRt72dBgMvwkJBr7yymPB0m-0fOV6spki2nZ5EUVuHqdQC4YZG6iSqI9B504yGmqsZX2Tyhwo7kkzcfTxkciiRCiMnTvAr1B_e3BVB8l_74GFWQ8OUJaXw0zJCnv)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjgI0m7qF53MWByyzSwLGMDvvPf7XeB7XJM2PvPSG7imrVhgmUfLGkWGeFBwHLONhNXBjMMJSvkjUdqk14py6ou2dy80IWlMOvZdthyIgShrH6mPklzBQjfVYsu5KV6pIv8U7ORxMCZ_1eAF8RM5CQypxCArCgpPShJ-YeGQDiiRtPM07tTidr4zXPAGjnW)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgZNmMYkeV0KYEEo89NPIPjJwB2DMEnTQDdR2N7_ZzN6JGfGbjyvPo5zdaymEyRAN8Ztn50-Bk_TOJ7Lskjb7waJhBuchB4cjOjVLXSoqfodt32LCffTHRtJra2FbwjX-KdM022vHJS-6RgqArsitFTbkFkIic2BfYsJLhfBP-RQ__snnjcilF92zpjMRUN)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgqa5qndSh5oRuOr6y4zQlnFY32GnZxpowaZasq4Ji_nTgNsvDUkKkEKgnLEtLEj_9AELVUSaUbSLiWutqt1BnT-iOe-Ese1eVdcdcCJGt441aeVp27LBoXgO1zxiPG8qqoh8q-cNNgUo6P2dSeKSsTRrIkVoI7HHo6PJ4YkbOd6IHbWIdqabCUu40YsRDi)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjQF80XMJe83Jae6Pl-LbSPnvS8CXtUyZcP4Hpxeizt1JtWLBseTFQu9C1DUm25OybqG_mBBmttuNoVFUuGpVreQXIxIqWKkBP232VLB9TrYFg-ZhcPsb7ozVYZ8qWIvQ6TmNvYwByF0Cm8Zo_em0w6ko3KS_KKm61Mg3OTEGQr5GLQqeMTT8Ygx9G4lRCL)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg4F6BIMB-GY8FZxbNAPV_mXov1_gfsoJcyT_Cp1AeVGEhk6a6k5bp5n4ADqbBs6uvDEE9rZiWmhbjB-GQO_kXdMszLLRf1ap4rQnrYG2rokst00RPu4j2uoPuIHgHSxUJypdJz501EZc1got1tKvdgD1IwkkKxR5sMlFqVdGOCnabq2ailGLYuD43sV-Mw)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgjW4VGL_yeVBLINTMayzdosoK72914obAIKAwxCeFXMOd7qwnLVocJtt2GU0wFvTX_VDDuY95CIJgwsfLIaqtZeDuUVX3ODqqOh7R0i188BL9P6lHx0z7st3ICJ1h0d-ofL7Tpf8AQv8Fs-2A9ow60ZQh2IjU5Lablj9Qf1XOZ_pB4Es2uEhwMW3m-33HN)
+
+**Lecture 1.1: Traditional methods for ML on Graphs**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhKJE78cvlqBkkHV0cfkdtRth02IjvBp6KRxAdBKuXB6LDWplYwqmXb7rgx_xQJjJAv7lh0cDlKlKpqwfQ-Qb56-ckSqubrgPNgaB-LkyNcXB23VyoUxPvLW66VJk9Vad9ktWjD-oGYet3hvTBv4p7_hgDPCixLXr202NAUwI9zK5UpYAiT0ablfN4s39_k)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgcyJuxCUA1bIW1Ht6E6pTkaRwSpRt6EcUQvvA9o024FHDIKM6EWJXvQ58rX0BVmdB4cLFffNTcJwCGpF48hMzgq10QOWeKint-3xkATadhYvl3MU-KH4Ji4NRIK9a9mQfyz1ikwXdhLbMj3PnL9gude_FSdS4Jq_l61di2ljIYwpnIpDxIvTZmbMEtFoID)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi5usLH6qu4BwQ5iOhwnAhURzJB7vqT8nFc2sWfeLdSUCfrxIxMrhRn_Hb1Q0XN_wAE8WFFKiXM0uJNSUJRgDkt93S5fjGVZ-w6DNpozMNvlGe2tECTEXHAnYR8eCUuh791yhvJnW7J4T5Pb04r_yZFoRzVg79M4GouWholbkmQhEgmu67gC41Om1lVUHrI)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhk_eX3g6ihxfqZBi72UcSF3D5f3twMShed14XpLqNijfuHoQPANTX-VlzPXf5ZkQ2TaM-KKRpj4R-OnIszpSF_O5RCjb8FI8NPr1pvwgjWRIjpYjvLP6-7FRQAJpIH9nBKI4tHtJpqsOBvmVttj2z5KpSSRlayrRaXaFJ9_AQkMyqkTI8PvmVq9QxAcL4o)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEggL1fIRn-dTOGbblpdzi983Iyk3jY5MdYKyIMhoZOY31KzDiDkXXHkTERtnIStpJ1oyB4mtcUI9llSTJ5i02w_as-8C-k2Jt5gQZjyHrKeOHWtds5Rq3wUHuFNGhP9vXuWnr_gRxj9JWzzJdZPkz1Hh8pXhpOr7ViTSaewLrF5TgSu8Q0lDQAEVXIhVvEz)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjH2TUrQNIhSqpXftfP_t2BApwWUrjzyL_9HwDOCaqKBjvHYrCOXqn1PcH8w04fsKSDzPBXhSayRSHtxZAis4BME0-GIcxX-AtGvfG2GQ276egfQb_ql_QCPwgMr4LvaV-lJfaCTKeKWl22w5VQ5NYspgE6vyt0meFpuG3mvxXORvcdy97ejR9SMtwksVso)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhWioQylzd-hX4aOjJW8utjsP1XdRpQrG3_3CpOWhhI3mMgha29-u2zCqbek2lOllParCC_eou1h6w2ixlqZiRSl-cr9vaaFH0oEJafqoyFRT0bpnpn2uGnHCeNz4Nxj__GueyBGwcTtUaMwdpk84az3UmYNHJ7iPZCeB4T0wq1x4hJcptu7RNAFjcE_pq6)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi5DQX4SAQQInKjDywuZ6D7uSs_JYAGzzD-rq3nOEQSvzlOiMXv0wa7qm7YM7Y8dC_eogyX5BxEwcotSlofTbRcZ_0hoZTh40wcFItq6hxF0q84znfXx9UMJwi0NcuDMYzpyX96PaJfRDAWu9LxRYHYwbSotbiTUkNSP6MVMKyhQIuaHH84tXUJSv03zTmA)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhjOuFB8zUZC-RbVZVX6K9NGZohXQEO5dMCq0plqY_OlPrgy8mtR5D9kHkJdYPtn3Eu5N4KyUVtAiA02A00pDFm0WZlgRKA9q3pboJ-fS3zqOX5SsBFhajFJ-ELAmgbRU0zfjMz3xiryld6GPJqsKj3DwjWNocp7ZYWVxETdiap1b2Z8seeZypkzUfI_4aw)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgZPNDuxQ4GnA_tNQtLKdxic0gZB-VMmJ3dIwshEjPiFqZ9rd3RKHQcTiTFHptP0jKZ7JeEs6X73Wda-Lh2P0rGrEq3P7A14Hk8pEvVZZuKC6ztwCHNY_Qq_LY_eCvvxMbJzJlM2R_HV50tBLnphIx43CZ1rDOL7tPWs59YDCuBVZgWDo-9WUm78poyybUw)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi2FpcMwGPwBg_FBbw66_PCMmHrvP8kU63639DxG8UG5_tS6bMwlOpVkyjMqKTUxRFtOSaFQBVjFL7SiR5rZSO3m_CJGy5f_HM-h_MsczqhWC8q5u4A9HIVnNSeOLm4bxunKXreRw70zJghNzzROFHcmyyAnFuqk48m46U340UvWbLitPmpNh--miQ-rdPm)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhkplv3A7Zl1GzfHFvda8YnjGOAPUD7P6otPwYuu241WGN1SXr1EktE4f6OedkJ1MyaxLibK-CmcrXJTNuBXXTqn8Tga-2j-nfW7h1tb-IBKbHy3l7l0dW69vBiWtK2dyvTI1G53AtykvlSChOYVIPJq5bSpiS1BqDqhB1sZEySEYeEkRY989CaDAJG-81g)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEguzAtNM1_-CplT_7yLgduKDebTEdlcLZBIzp_VFSihgpjvs4GdSZjTUfgRoG65X8xqOB8v5VYQjB-TI5yA-YFp9Ot2MK2d3ybJw0pLVrt5VIHX7Nurfb5VeqLxuf4C6_ns1RoofrNrkq129FW2z0H9gjFCyugleniNTCWlXqHrdzQrRd1X0IbDJpR2kJho)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhzDmhN5g7-Ne-Ucv1RP2FZtBop9muXyPZ67Bqnr5YWgkfkTOkEKfAY4oRxlatNTbW9AkzpMWcD4cHOq1YbAtJieOn2gwN-YSf0pv_ejFpztf2Q00CC36nZhsUvOOmGrffk_TNOtLP63iZ_hKw2xT2rJfCLP1-SsFRB_n5wsXX1X1N5rfOrHCdaPdHkIzE3)
+
+The rest of lecture 1 covers:
+1.2  Node Embeddings and 1.3  Link Analysis: PageRank, Random Walks, and Embeddings
+
+After which I am supposed to do Assignment 1. Exciting stuff in the next days/weeks!
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 2e3ecd7b7bf3e91a6bf5101ab1be64f55f3d3419 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:50:45 +0900
Subject: [PATCH 193/638] Create 2024-4-3-day-93.md

---
 _posts/2024-4-3-day-93.md | 67 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 _posts/2024-4-3-day-93.md

diff --git a/_posts/2024-4-3-day-93.md b/_posts/2024-4-3-day-93.md
new file mode 100644
index 000000000..7ea7da48b
--- /dev/null
+++ b/_posts/2024-4-3-day-93.md
@@ -0,0 +1,67 @@
+---
+layout: post
+title: (Day 93) Node embeddings in graphs + some foundational statistics/math
+categories: [theory,math,gnn]
+---
+
+## Hello :) Today is Day 93!
+A quick summary of today:
+* covered lecture 1.2 Node embeddings on [XCS224W: ML with Graphs](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs)
+* from [Probabilistic Machine Learning: An Introduction](https://probml.github.io/pml-book/book1.html), covered:
+  * chapter 5: Decision theory
+  * chapter 6: information theory
+  * chapter 7: linear algebra
+
+First, my notes from lecture 1.2 Node embeddings for graphs
+
+Covered topics: node embeddings: encoder and decoder, random walk, unsupervised feature learning, random walk optimization, negative sampling, node2vec, anonymous walks, learning walk embeddings
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh4M-TcBoKeGMH1v8hC2q86EEsFMyCCqDWLuNINTpjBlktznUAmrDsoAdhSbkWEvhG6VKCr1DHTmB9M_kLsiYwLaN2qD1WBdRtlT4z2ogw4PypgbLIPGw4M4S1bOmj5rmlmTlNGEJK7BaQ3tLt_uiRQqSBFgRXxRt93wKTLbOS9ZoDVaGCwlUlNRhIOplyH)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgY3oNlhyTowzJYvAYaWIy4qFbf8RKttSOLM6NejpsXsLzAXBpL9A9El6ERV4I9le6XpuxswdhqQI4R_JgYDbmbq3DaWQUhebRsVHWMpkMEK-m2Hd9J8kgxfwCSEfasgG2WpE7xGXyUUL5vXM5hGw3VP97NJchIR5Shtf7D7ZCsRq94-1HU3Q5HW-Kk5OPQ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgQW76S2K1I46iiA5bh8xszkBJQBZOsyvctApe3nRch_kS5oqFV-1B6zZkcc-rkC8293pMIiOoPwMNDadU4LyYceWe2OXF4GJ91I1NzwGuO8kWZsnb1kJ4mSFB8l7f26KnJxqKQ5QyqKY08Xr3khAeeBA0x30FOZnG9zHAxWV2-MDqyW-MLSZkwXbMhdqh0)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi2xW2MHOXOiasFdNBvcYkXYPp8M7dodR134k_MVgSvJezOmm18rpyzlFZb9mRNFISJp85Jwi2zDG2Wo5IbCr3dTuLY2FTaGmIJiEQ8pLMfioy9GLQpKdwN-9GOVxGukDc6Kw-qETmXRXj9hQhiEqCX4OUwENOBgwBZ7dtnv0vDyZDa6i1a9pnLu1g9ZEn5)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEibG3TO5bfyJPLl80-UgWzoCs6sZMyJuvkAJDoH8gt9PEn3mU5N9uqjtVUAn4S1rJkmf3QJnhM0uc8XMAopDuK_cjDWArOUBLiJDEwXHOEaOQHZ8Ly95wQqJqx1Red7Lfy64JtWvsYnlC7cEz9yGEpwt-pqZyD5Cdzy5c4EN7G6zYfNZVLEbqwj8o5L5u6M)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgLYb7BcCxhwuqTEFIXptzfPYHiQj1Z3s4Tklm7dS4KtqRZg5lVHPxqEF0idUqY_O2cXqmLv7h6u120uS0ZS5hrlOr9ipU1-d71ohBisfZTYZlFV16_4XgnETz0VTro8WU-TowO20J2rL1vZh4HQgh-nuzwQ_GaR6_rUw3st3iAdK7BUZC9OCBuoe6-umZW)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjjADVpU7HHDQjOka4iNbpL0f9KVezyksmAxsvTwKygJCVlvoT16XrIKTpko3uXqMeoj_d_hLRW7iISZpPHJnmDrlCPdjUsH-izHyljK3AWWILHMRtq4owLuWjyTYZVwvPHPp_QsGYov-LL6VqC7aFCKpfK_YA5mjew-5CONcOqe-aKTJn5kl6LuL8l9L-C)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgsZpZ0vqcgG1KQj1x8Du3rGsSDZQlo8o7ygEgEWPVtSQErjDKuWX1vC1cnWOcPGKHMkdAee_HpXkH5eKA9MXRRqa00Ob4BC8OEyEJ8_chmpsozRBT6Gw3K5cwDFmUbJ7DKdqWpnyWzB-TYs2FbotKkOr6SheFnhZD8KGF2YT1rlN_YgbZ7PqboVVYOB5Lc)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjFJA_A96P2wOl5QGCp04ahwUX4tL4g20LRL_iw6OURw075V_ix7VFY2AJMExuXWXoj_ydJhMg1npdl2NI7o_AMlZprBUL0Vj3wApK8oaFmL35_8-z0prgjX0FLkrsNsIUzZm1AbF5l3JIuNKSXVsLBeXPqCtLW5eUDjX5n3mqRSM3gG5dZToHYJe0Wvtkw)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhHGviDTmZpDcg3AIcLD9xRk0jp5ZRvqQDB5Yzk5f2ykfXM-zvK8RB-6UQknyddijTmnznPWf0ijiUtlNestfg1AFRjA9RKWIBEiKZvhHaMH9xuE6PdyepmpsiXJ_fODQ9wbCfTYqvlrQXfi19Va2Dnigy6DG4DGhY4rDp7f-HIsOTiU4ooCLFkolaYSRs1)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjTiF5tyV0am8drN1epoNe79nUQu9GgJBs5TPg-yycjgvNHCgqYKLbkp3QR17Lwlzzr9rlNzDePP86H24hbDKqghqEq-X8u_18TVriJhrhgoYr4VwhnPLXSycfqFXssba80jmcL2vjTaynkPVdECmQTp5l-KfeeHBlP26lajYlnbtXt94KojORt9CU96rcM)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgjSB3j_emPsercui8EXo6ec42FbxzGGoomQekK1_jy3b8_B09WpgGWZud9aPxn9aj39KbpDgDdnwT1xf9wOq-rCJtVE-UYNY4I7SZ1R2KMlJXdVAQlwqV_ASVfZ7MksDWO7aJKDDkpRnPr1EeONf7SCpAk_Jm2lzXuzQgf2SMuCyVgzG6vWT0TloryzY5t)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiWiSop26Aj2ctWZg0riUIaObsBUpJMOfJdWDp74bstYBqn806tTVdiiubPO1b8Tq0el9NQuHHDk6YO91e8QE0xTqWumOpQVZkl1PMhkXJz30szRLuejlGPHtwsUl7c0Mq6Ld27RfvTryuAYxq9-RZ9gxB7tvuaJpaRYztbCMEL57dVxfN7FT4DHONIZIWn)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjuhSuaTFi1oRtZoM9MgeoIzVdkrXYFHU5jJEtxJUf1s3wEsubMDbyXHXUOxJnGKxUVivSMvYcIprLCxx_xocadZC6A-KdtBzp0_59ex6oOUkgXRssUJOtje9BuaRWz7haN2HUom0hj7i9f2_K_BaQHvZ0qNzGJ7CUFFZPIO2-YPxYiIJwwyCUN4obloEwZ)
+
+Next, from Probabilistic ML: An introduction by Kevin Murphy
+
+**Chapter 5: Decision theory**
+
+Covered topics: classification problems, ROC curve, Precision-Recall curves, F-scores, Regression problems
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg1ctlbY6LpTUc7VOTBqVXchmWlWR97YUeU5Qvp_vBAbwBJo8IPUqpf-CJV_7JXz2xAOy9_Kx1GLJ8Sk8ezzJor5Pt9mfjFMOChnX5KHcdVzIBwDBVrWHrPUZomsvbl6RR6cScbNA5W3IpqXgu2Yg-a45n5vN1CjBCoJ8GjZVmKbKrFEdxv6PTelx2afxBl)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg0IwtHDQvMdR4rr2hJ6i_kFvL1uTWVkCiZE9ASHHaE3cmfZhbEsKeNhZfl5zE8vP7nwP5Xcx0FfXysbAsLYbZSSaE_DEqG19oufeQQoyP3D-ReEbJvJuU83Qgu8mJqvZ-CD7s8nDDxQ7lQrpix-wrPmTLR5joGsZjj28Yrzp0jjS2f56tLjswn8iUGJTi4)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj63OF9wz--AH7JdeUIlRMaio5OGwCCLWSVXMn8y-WsOnzFNvh_tBV6DaH_ycaiM6feN4YRJlIqAgLx0qDMf-PIhMAqnjZAeg4x9CiS21dgpzr6xoUBByCWacz_QPIa-VfHT4oP-ko9cUSkEniq6ESapo6wB0YAdDdxcIYaFILiIt_8LCKrID9IWVlEqYyu)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgHI_stjjk49aLPoYQGZnADo-hMOIFHyXX6Fta2aqc3Cx8lUIhrWe_wcUghDh37am4P9oYE-7mH8PTQ5tptns4CPluSbPj2XMVkYRuzXDO5Ig-qgbo97sBtoqrqYgdwm8vGKIEu4WiXb8PdUiCaCFOXCyfDATRmCtKqn-slQ-m23uubTCwzGkxo6ZgvVq0U)
+
+**Chapter 6: Information theory**
+
+Covered topics: entropy, entropy of discrete random variables, cross entropy, conditional entropy, perplexity
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhgbqRK-o4QGQ0Cr8ex4ItTPCumSr4AEjhQT8hwSj3rfytEzIivGx9CJggDhPQb91_UwedQ5nKk-UR1RL66YrU0xkH2b22iX5szQS-NkWQ98245_zq9IJXOhlTG6oDdgWiOymQfqpEkce4v9MvE4lxDqN98UWq6TeruR4oGE3lql7vtPgUdwhs0NR0rAOog)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhap-Nz4xr2FuszsEVc9IdfpwKO2_BLX7P-alRlNiBkmExj1VpzhvLRM8qi4llQFlv8By5pYdkwz02P7rxEvGwfWlwqD_OMJi2jvVOiIJzVD5IYEZkHI4uDQxCq4C_y4u57isvS3BGdk0zKy5m6V98pRvLJXopQQm-lLO2Qdj0WHP7O7i9FZSLYMrdyGAre)
+
+**Chapter 7: Linear algebra**
+
+Covered topics: notations(vectors, matrices, tensors, vector spaces, linear map, properties), matrix multiplication, inversion, EVD, SVD
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi7pNdAUDwbsAtHkxtJBncPUBUaoGBAcFyDOKR_bn8tUCXNvaqY3SFO8UbfPmRqGQ8NtObpjf5Ap_hCOSjSuW55RGxhRqvWw_SlnKbCURAg0QM0djjSXozBzTbWuq9BWTnDehB1kjik95D74_W2gDdoKZUCm2iEWbk0Tysi4v3wDPV4oMFfKX0iKopbTmQP)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiCFHClBsh_sVEGotf294K2gGy7XgIU9fP9_xHsslR73ENQEl0sZ8noAVb1ZjMKNC3fD_bbwC1pXdEDYYEWcLm2j8WEhd9Yib8D0a3YwNdpX9XPYXMNPtbrxtLg8dNTBesXA8z1mnFq7k8ccGVnSNKYt2d7n4vHAXzL9E4n-9pFu4GO4IITjZpYS8I8fvCX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi1oElkTyq2-lqjIEbqy2SJE2Umra4sBBwZZzuFOjeaIaQEdJ-8Uv2-ZRYcNAolkNmGn3pxAC_WlD2bXlOqZQjoTSLl4xX6ZGmev4MWH9QZt5wA1ZvVej6qpdwvwz6rbrUQGmZFfBsNfbFqn6BKtZUhlAYfOvkArdhR_8pyFnAdxCwMj60fO4fhmjrhPYIs)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjgwjX0oB1Zxt7JSNNmiorMgGfWQLiLX7OqnM-__EUzcVo_kEo18OQSBfZ616q-DbHvr8rCvR1QSLJaoedRLsoF_OieEtB43CV0elS_IaONH2Tctc3TC8ejrZKtHZUm9YY_0YAZet5bscfcF8tj9eApzV106aU7E1ekKyDhJAX5xQCzpWkbQ53a5gupVoww)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg9DKDLMUD5LmB-tXhWAXIfXXZ3UqivtyEND3po2U0jljyrvxmUdvVtG8_kJHmPNxhlppyQ58gFCIibogiYBdOmIulR_usHff17AUc5SRxRTeCUUZfbLUjAVp32Zy9qgwxMkCNU0VftDsZAmFEhwHwosu6V7Qc2zcOBALLK-uc6Yp7C1TU3v-JEpemKbKIn)
+
+Tomorrow, I believe I will continue with the 1.3 lecture from XCS224W: PageRank, Personalised PageRank and Matrix Factorization.
+
+
+That is all for today!
+
+See you tomorrow :)

From 2ae5e093e0163c799029ee6541e9de71e139de00 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:52:58 +0900
Subject: [PATCH 194/638] Update 2024-4-1-day-91.md

---
 _posts/2024-4-1-day-91.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-4-1-day-91.md b/_posts/2024-4-1-day-91.md
index b4c5a6c1e..bb663adb2 100644
--- a/_posts/2024-4-1-day-91.md
+++ b/_posts/2024-4-1-day-91.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 91) Probability: Multivariate models and Statistics chapters from Probabilistic Machine Learning
+title: (Day 91) Probability - Multivariate models and Statistics chapters from Probabilistic Machine Learning
 categories: [theory,statistics]
 ---
 

From 65bb1ee715d3feafc39dc1afe07ea5d4e88ce463 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:53:17 +0900
Subject: [PATCH 195/638] Update 2024-4-2-day-92.md

---
 _posts/2024-4-2-day-92.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-4-2-day-92.md b/_posts/2024-4-2-day-92.md
index 240176a57..ea21ddd68 100644
--- a/_posts/2024-4-2-day-92.md
+++ b/_posts/2024-4-2-day-92.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 92) Starting the official Stanford XCS224W: ML with Graphs
+title: (Day 92) Starting the official Stanford XCS224W - ML with Graphs
 categories: [theory,gnn]
 ---
 

From 926de3d798d9b593414712b018debbe6c5da4d6b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 02:59:08 +0900
Subject: [PATCH 196/638] Create 2024-4-4-day-94.md

---
 _posts/2024-4-4-day-94.md | 55 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 _posts/2024-4-4-day-94.md

diff --git a/_posts/2024-4-4-day-94.md b/_posts/2024-4-4-day-94.md
new file mode 100644
index 000000000..7cf8a5a70
--- /dev/null
+++ b/_posts/2024-4-4-day-94.md
@@ -0,0 +1,55 @@
+---
+layout: post
+title: (Day 94) Link analysis page rank random walks + First assignment + Short intro to GNNs
+categories: [applying-knowledge,theory,gnn]
+---
+
+## Hello :) Today is Day 94!
+A quick summary of today:
+* Finished the last part of Traditional ML methods for Graphs: Link analysis page rank random walks and embeddings
+* Did assignment CoLab 1: Learning Node Embeddings
+* Covered first part of Module 2: Intro to GNNs
+
+Today I continued with the coverage of [XCS224W](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs): ML with Graphs
+
+My notes for Link analysis page rank random walks and embeddings
+
+Covered topics: PageRank, Matrix Formulation, Power iteration method, Solutions to dead-ends and spider traps, Personalised PageRank, Random walk with restarts, Using Matrix factorization to express node embeddings based on random walks
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEirvNezQ04bqaR4S4W4-aZKwRp-Mdv1i-e8aL_V8mHGOyZASBxfaq7SMSN5lv936O5rdqIbo6_vXzkjUTyFmn_fFj6V8s4YGVclQKSugxufiCyZZnYUuvjvny60ONPTcKsBL3AjpZt3x618dDPc8gAb1Eu8tQF3bbc2CMJT6ehwLiQqBAYd21zlGplOfCeQ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgkManzJVQXSEEeLHBSHnQo78pa6SpqQ6Vud-L-m-fTBrDcOrK0e4qnMaozoCkzW7u4UUWP1BhIReIcZmEJo2M-0ftvXWvuslk7Oy0GPRcHeLyUQCcWQ7Kd1qWwynN0pz_QFlws3HemTXwQ9pdMTsmq6Q5Ix3Sb_M-dM5uEvfNiX5GQH26mZF84WIGhwTC7)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgbqWKV8tyeJPg6TXDaisNOV8uD24-CvASckGxasdlqtEgIinMi98wQ8h11aTlU37tS-CJoc9N2fT4NseZKiyA6aGEx6hZ_A89M27MOKjfYxT3zk0IShZCsrFPvcO-Ei1nzm7uj3sHCZ6oIXJu6FgpBySbsf-N5jyBoo-Yk8ER5E3-CyRT-YwFtw0mL53vq)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhCbqLR0iYyrKLBiTz2brI1xjDD08nI2ISjIOGk3qe5FfPF165MSL9dhCx0Ye4KC2gj_HTGRgLSdk6o3yUhrNFfPG5WZl8hxAFLZyN97xZyacNrmlEa3EFdVeaeA9ZO0gBDxUYciZ7AD8zQ-T2QH64-829KvgLdBOZ1KmeH4CA8MBjgtDuclbBmTs0tD5O9)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhhT1zGtpI6tmOUdZDidiAewms13cOmK7XwA2n4G4jspqinCcK4zUabgi3OaCU7KGSkkj_WUq259gqwwFNcLW3d6sSrXjbReXw6xTqeZUZofETt__cFRNTuPOxtO7VI0pqmUv0UFNQs3Gn3uqn8DoITsd_MibWQqRc0aHVpVpsn35PmawwKYTiFWtX5RtxS)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj8iqHfM_-f68iDtXwqZ0RZKY6WpFIkA8gtk1A1KY0Ds7twcdqX6n38hp7gfNatxscM-l0qLWmQWG02v2FBqdlITVMs9e9DBt2nHSDz_-QGXZazyqoEuqQINPWsvCesU9SHS18P_xXJfMLYQDkyekwemz2bX-zveHPOAd70SaO-FflhXfmkAe_KMziNXb6v)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjOeOfu4CaOeWCM9E1qd86t1sWmEtVXJYZAk3NTuR90mSpxapiWZdjZZup0tvxsx-vD-v6N1FqM6s2YScoH7RGiQCj2MFy3ZHL87ncbFOSPXhL5z-9pHVZbMIgJjmoXfKsgRxvdzCg4Ao9CqmaJEULZlXr1rW_p1TdN9cxSUdnODESEM4C7i3m9rgiKBvfR)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjjLHp4QE4h8t8tZ43B_YLHhB96E6ohVESRwMON1U4Tei2YZILOaQBepcx1Qf1v9wEi8I1Q5zzkjYjRQV4YwEjzfb4b4s6jmvcpP-rB1hV41ZDNbiB38_bE8vfXIJoR-pvGOID8KsasmgvAIvIz6x4IZ_WTW0py_roqXLaN-4sLmDZ8i4ZtQzK2yostPORi)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgt4D6qj7DIlaXQcf48UOQVXxWbFMYNuXzAG54c6Zaah5X2ivsIhS0aEdSULHHWNrH_IH53RDQTF3RNSusLEXAM00QthW3G3lD_1J2DvXav4aLDvHfPaDG8drm3BWs8NR8TdKlXN6QYAkfLvNFK1Rsb-oyGHJ5OuBHGHzMCVW-xzlu4SglbsZLpMEeG67hY)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj1SEezKBPsJ60w_2fste3UJMrZcAxAVGoL6mqNjxyJ63KuJm3NqC-ssIunFESa4AtkVoZCEUGfAcZq2LlNLeiNeT3JDaQul8mXfjIgK6LaH0NPTitz0GrtHsmkPVJNlPicOap9DlwctvchfozHkxLY6uPJXCjvztCA6Nvbmncxi14k7zeGEX7vsWYjXKKW)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhYaKLoHh6JgBnRUO3vrN2S8U4ZisNGM-N_LEn7cQW2bv2m-3DJ87NDK1vFlyf3Cjp1dHondSpZ3uH0gRAfxWGtkhHfu3-DJ-mhq13y4t597pAXZ35_b1iPBaH9SGiEY4u2VrLbvN8ndplTeiyJSgWGb0XB_6Xi9na7sBtV_5uJtSvi0ty1Q6dKD2ok9agY)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjGcU2R8PHyL_KLTlsQ_hM7oShTSGNBbw8t_bCyWAVaafg6-nq2WACb7eUPWveJVnN8cYy8nQct9HubBNmogWFk1pawRz-J7au4GyElCuaLrpSC2kHRWnz666it1JVAQWaTfwbe-z9KMFuDd9crIiXiNXc7e5uZWGfuh_j8n1W5pVpPld7hWddI382ydMDj)
+
+**Assignment 1: Learning Node Embeddings**
+
+ We are not allowed to share any of the code, and I really do not want to risk anything, so I will just say, similar colabs can be found on the [course's main webpage](https://web.stanford.edu/class/cs224w/). But I spend a lot of time to understand each line of code that I wrote, and how theory from Module 1 on node embeddings is applied to practice. Also, I got 30/30 ^^
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhsHnCOLgECEvLkuisQdYoXaQTPYLQR0l3vWVTKFqJsApCV47-M0ZUwzSCML3HqzX_aREF6p0hpXqzf4vVYq9tpkmCUlSQw7Az4Qkjy-z1p8LJeDfJyK6oizdjHBYK4oLVR7K8HcHWB7LcDbkdvbmn4TQq2zL8W8OnLiIgKTLMrKINB6XoeAwKsdryFYq0W)
+
+**Intro to Graph Neural Networks**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiOHTQOjEGmdYrH8M2LMa2dIvrO8KGupIThATTOaMoCUPeybKj62s965y8BRrLL7vbOYQbTESluJeoZHrKvVnDSal3b1fgK5EfsUXz8CmQNh4RXHRHX9MAn-LzcfR3Q9_oDJVdxn7ysiG4ExCp0JAT8SO__eObuvW1jhpaj7PqjZ6vjhAvoo_eKKh8NEodQ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEicF9ZAq5qpx0oWxK44hUGBHERW6OS1PlPbvOxpZ_u4XzVfO52J8j3-dH0gVgaMoQzp9yqCokzLiJKL5yvctdoRGZtUEOpMCVkBFPKr2vV4JOZKIyZlaw1_iNSQxywzqYC6laSDqMNE2TpNpMawcb0tE61r4bIbGNP9wYDb2QCYYesTIoOkl7Pb5PeV8mMr)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiHBGV1FxucdSZfj4nedQXJZojz9TxT3pt8mdiUHOborZ2Fd40P-UndBNiO1FotnHAo16SMKu7QOS0V2sAlnLbqrVlxJ2BEhPQMIRvRuRdyulGwTsB-Lf89u5qdYEk7SWvQeWID7jwfPC0wBPOrRw39sDF0J1q-rx1PzIj6iXQddrPBuTtyG4utV3WkgK2M)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjNjogfPXLTIqN-bvkH7_k73mnkIfixrYoVEnRnIvV1kBBoQCFAQWc4Z0u0dWiLUhzBsJJ2x2UoSBvhmH5bZNrkCJdccP8fIxreCeWmxSO-bpCMIuJyjf36qgGXhMjlVpru4Up0_Uq_uIWUk5A4NctM0CLkr3Zupy8dYwgkrYsNxmjEz5_RAc1DM7wJCT1t)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiHgvChmbZ-8JwO0lqUCb7VHvyxoIW_0hEGwhnrSU89vBb0gDEvYGtvCLJny3TULrU9lsaPsTkTuvuZmohTMrdg8mJlrQzGIEz5e9rnYmYs5THNGEty1AOZ3AN3uu3RQpl6z5j9e4lcMMzB4MRnw0OA3d_wOebF9lKVnK_cNgZ3JHs9om-T4ZgAMPn3ZSXu)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhMfb_Sgf3pDrN-W5lkJ-Huv3V_ltXAAAYVRHX0jP7-kto-XoOFhA74OKt8qDjICQpDMiCymu6iESepFjkzbvtmQMYY_pLmJnillheUZ2aIHG19043IqVxuUNgkr30GqXlPHLx8OnWTC0ooTSHqQqikURvsh-hNXVB6Wk9RqXX_He5DEd1_JuxbEJEOOEcE)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh9-ieOb5znqHaCHZ5nrAD15CZz3SvMbYqKHALLtzswYNQGKYB1xr-Lnbe-94ZmWzSdUHsAkB0UwJrqUYZWpvKDRo-LCVPY29qGZXK3P0hzXLFRXLVMHqmT_FOZIME4WVX89vx0kfEEq8dlmw2Nr0llXG1tSCJCA2ahc1byOhuZVDbuNbmHxtzgSrhOXGc3)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjkHatOGVcOvyxSS0Gcb7hJqz0wArB7L2Z6t0W4iSE5xYQjVjntI_oTJBbvNkRXHTc_yiNCJlBPECL8KHoqfVaCL1bQUHmf1_ol7b8AmcAqfQax20vQuKjsv3gMvb87_wa_hlJLCDt2KpBw5-FelM1tvtzecdZLJ0YVplpgAGf9U05WV9NIR5Jb6N_3gaQr)
+
+
+Tomorrow I continue learning about GNNs
+
+
+That is all for today!
+
+See you tomorrow :)

From 9f02dc916bd5dfd780e11b0886f04ff7734ce42f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:02:42 +0900
Subject: [PATCH 197/638] Create 2024-4-5-day-95.md

---
 _posts/2024-4-5-day-95.md | 40 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 _posts/2024-4-5-day-95.md

diff --git a/_posts/2024-4-5-day-95.md b/_posts/2024-4-5-day-95.md
new file mode 100644
index 000000000..a86c4a100
--- /dev/null
+++ b/_posts/2024-4-5-day-95.md
@@ -0,0 +1,40 @@
+---
+layout: post
+title: (Day 95) Designing a GNN layer + becoming a fellow of the Royal Statistical Society
+categories: [statistics,theory,gnn]
+---
+
+## Hello :) Today is Day 95!
+A quick summary of today:
+* learned how a GNN layer is constructed and operates ([XCS224W: ML with Graphs](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs))
+* became a fellow of the [Royal Statistical Society](https://rss.org.uk/) (I thought I already was)
+
+**Lecture 2.2 GNN Design Space**
+
+Covered topics: Designing a single layer of a GNN, message computation, aggregation, GCN, GraphSAGE, GAT, attention and multi-head attention in graphs, stacking GNN layers, the problem of over-smoothing, shallow GNNs, using skip connections
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi1zcuhtFuFdxL6vbsA0MjdATlBNvYDFGfErdSEu9uR7RB0WjApMNlgc-Kp4KBw27IL_963HU3VYYhg_-W0JtkPuwsASi2UTqwGcX4bUv-Vy5v0oxBllbKpPRUtb9tJ4JSWB6_GrAZmAh28Ur954-l9whJQMkNYxHDfJEcbDBOADVGIAVcO1cA_x94qiA2S)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEibSJTyb6uVYU44Klr0GRxkzl7cBN3vth8n0CpKzceEtqzTaM_0ujKZgCKo23l4S4naM4XFJMG9tshk0zEl60pT3apQhju1gumsWKs068bwOki1oQIXnmrwLylZ71KqCrmC_N4WPkLDrwH2MRWr_ro5QI1-1x3DRb01lUHYaXrmD9ZqiJ1xNcJlE3FhrSPU)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiEm1DGx2Ff9rdr0hVX_fPRCvLpDzUosb0IUcCpo73ZHkrMyNdPFOtWDc_nRh6jfVWAPSGg3bptzTvlf7yfAbLK26wjJJZse9rcBHauuGQydGH-dEmlhDRyJAoPZTerzSQ2BiCK4rcpOTHpN_2i1RusuaABL3JXDFCjgj-ltBbsLPZYvY70RWRpbeDW8g8O)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhYiHABOt8_Ne1S2P3RyxH7kByHRm1JPGfiMLiDWbNhszHorBkZGYn_xCdEXZndNcpy7hXqZc_XnBSlDllyvUs6jM-UioGWd10V4qfxB8eAko2KuxxqgQf1bJH7fqdxYr5F4aVudVu_pG7i31iftLYhJ192JbJFGLOz6AYf4SPgurpuie76dGxC1ou1twW1)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEixSLNTVqVJWBA3gXbnw653AqwhBFxXLcexmr19U6Uk1Y4F1d-x3Fg9d1IEcEVrmmvYKN9Me1LMraI8jF7K_lMP6Zg1phgQrouE_TLMorbheoX_RrkTt06aoP9ZhJkWFLpaxd-c27VRtGe7JniKHekQKqvCVwegXpzebEE9C3WwqGOKUrNQjkOvFW_gGTsI)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjrK8yZtjGXdfqXrp4x3nnE_wnjudfglyxBLWoGcq0cmif1uRoFDQowEa2o5zW3tT5Dt_jJk4IB2be0bf_D7UZdxJPLLOP9vDnTDMt0XRGqZkl4NhJSqh0uq_xNViKcvYhUf3dsaLvT4bpouRVhepOJbr6U5XO0tLWUu1XuT3LQJhl6e3coSZogGnRb1aid)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiB5izLrBLku9fSqrFlemPbzcxYS1HRXoty__PJGu7MGdPme3E-6pRhyop82ivZ291blKmEsVQisJmsj6u_WtbhovTEpZ8VbbLSoIbAgGLi3t1o2xLaNwTZcc32PmZS5-15_8dMPl8wWVlrf6hpf0Tmzfcx2lDxJdupHmtxzWSRWgPsO6JIjV5i1hs8P23f)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgFCnCHL7ZIyB0RZN9pFsSqDcIccWj4kNNuk-tPrXwf8SGFkKF06tJ6NgbnKv0qWlkk_ldILfcuheOUdREkhAKIEB5iBXnH5bh59BI8yvdmvfw5Nz10lBg6TsTSjWdT75XySvkFauIaC_N0fvKJKkpE96rdn1oJA6CbmcrogWTCIxN3VT7zJQ4mz-vvgakI)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgXJTvVIWsPli_tvDqaXRbbDVZbKbp76Ta-QKVoBFR8aQ7ENa3vNI7k6Jeg7VKMQFwqFR-djyNACpiuhiZ9h3lfO83mSxXqMw2SO6-yD2RMhOxNGZ8xe-nk3IYfT1-qPY56IfhoQnD1yLnwgxM4M8QuHzBu45Grbf80nEJYJTm0h-6TYJC5DYTGMfOLVwEv)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjYb4VT1izY4GqAG1xGhHxrGy11nB0-qWAuwnDEH9n3MBtYwh4byXVmLJmbeQU8shL4cyvd6KKhKlOTPiPOn0pD-Cm1qMTROwLmNBdhV4YilHmMoGiqwWClHaKdpR6GRIHzZ7UZ2gNT66muChXLYMMjwrHEMlFPSRleiaz-8D6RlvIoWjI4LWhYjVq3QEH2)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh6EDoDpeZADwQpK3JVps3gSiNLVvfgZvSxjZ1M4YNIR05A_K6yI3Jl9dStEF-ilztR1dkYQ-NgG-aZxj9Kc3ph4-5Gnkaczb5HC5i-Rh8NwdIeCo4b7q3Fd_1OXg9iu_cHOpUNfSEgDF_kt1LEYbS015nd9Ei0iJHaXirFXEMv6YHnIXJQRnhn0jCNkwor)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgjECRPHdaXSpCgaLPvgNd5SStR8huKq0cja5BR2EG5ElITxsMh4EKQkTrsKwpFCRjiFrQ15rQ5eTKnPUPfDShuftvooqLJyP9xYHeZVI8BfFixgWzDSeHQ_vQFM3nxRCh5EcS_qy2tykljKdIw8n53LZh1Vj9wmkirNEmhwIvJIc3abOYj83gOLo89mjtQ)
+
+**Becoming a fellow of the Royal Statistical Society**
+
+I was registered as a student in the RSS and I thought I was like a regular member, but apparently I needed to register as a proper 'Fellow'. Going forward I see there are events that they organise, and have some courses on statistics through which I can climb the pyramid
+
+Also, for a bit I tried making a discord server for study groups and discussions between students of the XCS224W course, so far noone wanted to join. I am not sure how they usually do study groups, but we will see.
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From cda7485e97af00186e223aebfb3da29e44145997 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:06:13 +0900
Subject: [PATCH 198/638] Create 2024-4-6-day-96.md

---
 _posts/2024-4-6-day-96.md | 41 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 _posts/2024-4-6-day-96.md

diff --git a/_posts/2024-4-6-day-96.md b/_posts/2024-4-6-day-96.md
new file mode 100644
index 000000000..52176454b
--- /dev/null
+++ b/_posts/2024-4-6-day-96.md
@@ -0,0 +1,41 @@
+---
+layout: post
+title: (Day 96) GNN Training Pipeline + looking for opportunities this summer
+categories: [theory,gnn]
+---
+
+## Hello :) Today is Day 96!
+A quick summary of today:
+* covered the GNN training pipeline from Stanford's [XCS224W: ML with Graphs](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs)
+* applied for a few internships this summer + found theforage
+
+
+My notes from the training pipeline lectures
+
+Covered topics: graph augmentation, feature augmentation, training GNNs on a node-level, edge-level and graph-level, pooling for graph-level tasks, DiffPool, supervised, unsupervised and self-supervised learning, loss functions, evaluation metrics, splitting data
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiB_fcWXf1jmvIJOBeNM5pjm7s8O-J-sMwsWzwr6yyuCCDdwV9S6ybaXBfAGtRM19ugQ4g0HpGsDRE77D8h0mfpXkG6ku4Eg7I8bl32Srs-X4iUd-8UqXaBUpT9hgb_cPEsbe4mxKuoDJfdS1YzMcec8gd_gx4Nx-F7p3839z40flCFVN7opj25zx-HIopT)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj0eAYDnfffnnc7nahE1CRGN57n_DHPjIDQwgqrNNSCURL1xBDtXRJxeM3deDNvKCSg6TzEBUjTk_s4-um5Dpy_w-mcO9HRAGeapX6xCC7F_6CHnDJw9lFXoNlZccgRC6B3WQMwdomQN1p_aRILfvowYJ3KM5dLvGGcYZw0hFaqLih3S03op1KXwhGy7NIX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi69T2E2GwwTGgomLjnGAxYffXIidRoJQTO7L3Yme6tR3zNt_L7TPn-upaXzsAJ_GJi4rYD4WSNHJE4DtNpznqsA0SVYwjfwHuJ5WxaJg8_v1kgggnhIR-_FwlDk2ErSM87QQIklGCvBy1MD-h-405aEpszn5Qon51DYYWTYZWG-GPvlnDQDa7wdnBvN8oJ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh8T-Go46CoTXjO-Ug9fPS96jdCudPZayqIES8DNM4s26y0qDrIj3BWMtPBPTjKSivFZ8Ga9R7i0P8d8N4IvqpUpzJIz2IBzsQqcLbNgyWG4EYwTZZUDABobHnPczi6aVH7Kqt68BP2327yLeWFzklnAVfxYP908JmqC6z6ylL63WL_PpaRHbo1qA0Mcr-B)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhOIthiIoJOZpYL4z1hJdW4iZza1Gh_96GevN_GsCMAlKWUxnBf8MLPFVDPkHbX01bB3nR848oXK7Tgy6lOOgzD_fKbM4RABrk560M_7Nc1geKblddux2TAKz93rPc8gqkTJs-NGMS5CSQV9Phqx6LB4OJiP4e1gc-ojvv2pRNyo_BgEbXWX05k-VcLYecO)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiU0vF7AsdYO7Lhra6clSrxKpaNam0dBcwxoCwR-e7uHz37g4dQTIcLFckJRgzDJagzM0S3g4PYVdx1Le13DX6m7hMy80BNf3bn84OW_bY4Mb8Ph9XU5ocV987MxPR4MBC83u6pULF0inOoWXFdwc1hdjpIIH0_u2i8GVUeLNellwxKqMgWZ6VGmK-s_UOc)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEja7lLHvfxbCt5vRhRpFMWUc9Qk3000ddu1TRJEgWSEsZFwvLSXbMYMbHUStQ90vRkpYsoCMMtpojPlmeH-_KWw0_HNZBGmyEUKgc0BERf4ioemRnW9jphBfRJ4tEsFdUHr82SUuPBa9D3UmEJA1FFU7unOw4gZeXvKabcE1pk0dskuSvQdAOYL8PfbE1kf)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgClzl7ZN7QzHIPY9ldqqQNpw-uM73S73r79w1NaJC3w6oOIuXsvSg4YHtthJG-3ToafpLE_uPBhNcSSPqerhNqcjnMHvoB_dn509UhjrgRKSXEnoW3N0-0-KgoM9DJ24S3OccoJ7-hKXoobf1U8q_3woACSgKllgCqYOruyCAWnyLHG00DtUu4MeTC0pj-)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg8rPl54lj-VK9Vyi1Qlw3zbZsu-B6aZCoCSbdVEYVtQxmFQS4IF6LkYbd9QiQTMbsB4L2YY0L9ny-fl5wseQR7526A7UwniC4ppmUnprhk6S_Pa1lYnD1k__3MoT4HwDhqjtqWpk5KlmTYo9P2qCORo3bdeQjRGIpgtahTfb1-hY4xKDo84ZEzL9JimghP)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjBGVAAvbsUcMVmoIyh1zvEGRs5pTj6Stu00chuaycqgw8oYen2joS5_KjIcQd-DXkVhPKjTn8RhbNCnBX_R3zmkLzKSPBPZ5LJGCnxCTY1Xj2UYciWegpy_UAm_UkEE4G9PstVVLoIIkBT-HCCJWq-gexDcbPowInADzLbk9t7kk6EtYNuIXhneM7utVVF)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiOtLP9Y7OqiowsIQd_KzoeDob3lWFq9Ewm2wmOPxoM3jrPR-_56pUxtoL_yXJLXJCe4HErEd_k8uXgCdpy02Xl6hxJrL939QpOkG-7AYRXK7s_MZIRdcbJ0DvEfQISojmaj2j6fXrFFFGhCgyRTxwGdvNEiDATKuFcqJaUwcw1BJsM7RdqHOSqS5nEmP7G)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhKkG9uu33Mvn4UDDXg6O6OMvE8Ge7uFAXhyzpcMCow-Wv4X2SFIm5DBuEzr2yBAfmUFOnBTgFnRFCSfK6-l5kybCbNs67Bv7Owf5LlOhDFweidOJ5DgEMa84hkrUd8IQBIUZdVlRdD4V-L5AP9LTy3xlvJgkr0Tbx6sr6N0zRnhIVoV7aimOHZdMX1jrvE)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiubN1n7DOsMFou9psRNi_Qzts6SdbkTDEHvd2y150FV-lqCV-zGaVKM22-L9Tq0O5NqP1oJuxfLxTVod6rOk4kx2uQghEYmzRKBjtUyX8NVFQwHj5rJmTZVaEW69kHMH6ubxbn4wclQdMXR71i3Jar0uXh1_7lvdlYZt6e4EuD_MUmy9ySO4UL-V9-m-Kq)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj72otEQnBplG_gSlEmd9Bh2LwmQow-kxyj-jTLKZfcr2IX5JZrOiVZ1og75GJNQA9Sh51vHwxX1O60YJuVCAmL4RXQzqTFZ94P58penBDgD9scpU4TLouH1U4sraylGXnha4iZWzRcAKyhsaCT6hBq_MM9Y5yC-Rse5CmbH1Emg6IYH_OCuoWYYLmtH1gp)
+
+
+**Looking for opportunities this summer**
+
+I am in South Korea, but I still have the right to work in the UK, so I decided to go on Bright Network to look for data analytics/science internships and saw some good opportunities. I hope that if a company shows interest they would do an online interview. In addition, I found [TheForage](https://www.theforage.com/) which provides free virtual interships. Though short, they could give some interesting perspectives into the work life of a software engineer or investment banker (these are the 2 fields for which they have experiences at the moment) in some of the companies on their website. A few days ago, I also found a big Bulgarian company that does internships so I submitted my application, but we will see the result in the coming week ~
+
+
+
+That is all for today!
+
+See you tomorrow :)

From ac823ea808f51c05156fb006d95ea30f5a92eb83 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:10:16 +0900
Subject: [PATCH 199/638] Create 2024-4-7-day-97.md

---
 _posts/2024-4-7-day-97.md | 45 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 _posts/2024-4-7-day-97.md

diff --git a/_posts/2024-4-7-day-97.md b/_posts/2024-4-7-day-97.md
new file mode 100644
index 000000000..1fb0002c8
--- /dev/null
+++ b/_posts/2024-4-7-day-97.md
@@ -0,0 +1,45 @@
+---
+layout: post
+title: (Day 97) Review of the GNN structure and training (last 2 days) + starting Colab 2 of XCS224W - ML with Graphs
+categories: [applying-knowledge,theory,gnn]
+---
+
+## Hello :) Today is Day 97!
+A quick summary of today:
+* read through and reviewed my notes on intro to GNNs (Day 94)
+* read through and reviewed my notes on Designing a GNN layer (Day 95)
+* read through and reviewed my notes on GNN training (Day 96)
+* started 2nd homework of the course covering the above two
+
+First I had a quick look through the 2nd homework that covers material covered in Day 94 and Day 95, and I decided that it would be best to do a proper review of the important concepts from those days
+
+Topics I went over:
+
+Designing a single layer of a GNN, 
+message computation, 
+aggregation, 
+GCN, 
+GraphSAGE, 
+GAT, 
+attention and multi-head attention in graphs, 
+stacking GNN layers, 
+the problem of over-smoothing, 
+shallow GNNs, 
+using skip connections
+graph augmentation, 
+feature augmentation, 
+training GNNs on a node-level, edge-level and graph-level, 
+pooling for graph-level tasks, 
+DiffPool, 
+supervised, unsupervised and self-supervised learning, 
+loss functions, 
+evaluation metrics, 
+splitting data
+
+As for assignment two, I am not allowed to share any code from it, but today I did not finish it. I decided to go slowly with this one and completed just 4 of the questions. Tomorrow I think I will be able to do it all and submit to get a final grade ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From c605bb72dae5ee7708ac9837bc045e868a55b51e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:14:25 +0900
Subject: [PATCH 200/638] Create 2024-4-8-day-98.md

---
 _posts/2024-4-8-day-98.md | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 _posts/2024-4-8-day-98.md

diff --git a/_posts/2024-4-8-day-98.md b/_posts/2024-4-8-day-98.md
new file mode 100644
index 000000000..d90c854c8
--- /dev/null
+++ b/_posts/2024-4-8-day-98.md
@@ -0,0 +1,34 @@
+---
+layout: post
+title: (Day 98) Finishing XCS224W - ML with Graphs' 2nd homework on GNNs Using PyTorch Geometric
+categories: [applying-knowledge,theory,gnn,nlp]
+---
+
+## Hello :) Today is Day 98!
+A quick summary of today:
+* completed the whole 2nd homework of XCS224W: ML with Graphs
+* started reading Vicki Boykis' 'What are embeddings?' [book](https://vickiboykis.com/what_are_embeddings/)
+
+**About the homework**
+
+The topic was: GNNs Using PyTorch Geometric, but I am afraid that is as far as I can go. We are not allowed to share info about the homeworks, but it was based on my notes from Days 94, 95 and 96. 
+
+My score:
+
+![image](https://github.com/user-attachments/assets/a38bb464-4b6d-4d53-9e16-84f64204ce1d)
+
+I had a quick look at assignment 3 and apparently its based on Days 94, 95 and 96 as well, so for tomorrow I will either take a break and not do 2 assignments in a row by continuing to watch the next lectures, or do it as well. (the 1st option is more likely)
+
+As for the book 'What are embeddings?' by Vicki Boykis 
+
+I had it on my list for a bit and I just wanted to read it and make take some notes (which are below). But most (if not all) the info was already covered when I covered [CS224N](https://www.youtube.com/watch?v=rmVRLeJRkl4&list=PLoROMvodv4rMFqRtEuo6SGjY4XbRIVRd4) NLP with DL: from Day 57 ~ Day 65. Still it was nice to confirm my knowledge. I did not cover all (the notes below are up until the half-way point I believe)
+
+![image](https://github.com/user-attachments/assets/d6196191-159c-4d5c-8f79-b5b958cc6590)
+![image](https://github.com/user-attachments/assets/1e8b1e19-83aa-4521-8428-6efb543cbf2f)
+![image](https://github.com/user-attachments/assets/3603de6e-bd2f-4776-b81c-1d792f91c7a5)
+![image](https://github.com/user-attachments/assets/5bc7eb44-75e2-41d8-9ac0-9921de21f8f2)
+
+
+This is all for today!
+
+See you tomorrow :)

From b27a37138d1b0c8c7d9bb8db5b40ac2614ee1507 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:16:04 +0900
Subject: [PATCH 201/638] Create 2024-4-9-day-99.md

---
 _posts/2024-4-9-day-99.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 _posts/2024-4-9-day-99.md

diff --git a/_posts/2024-4-9-day-99.md b/_posts/2024-4-9-day-99.md
new file mode 100644
index 000000000..51563c14c
--- /dev/null
+++ b/_posts/2024-4-9-day-99.md
@@ -0,0 +1,26 @@
+---
+layout: post
+title: (Day 99) XCS224W - ML with Graphs: Theory of GNNs
+categories: [theory,gnn]
+---
+
+## Hello :) Today is Day 99!
+A quick summary of today:
+* covered the next part of XCS224W: ML with Graphs
+
+Instead of doing the 3rd homework which is related to createing and training a GNN, I decided to continue with the next theory part, which funny enough is called 'theory of GNNs'. 
+
+Covered topics:
+Expressiveness of GNNs, computational graphs, problems with GCN and GraphSAGE's mean and max-pooling, designing the most expressive GNN, the Universal Approximation Theorem, Graph Isomorphism Network (GIN)
+
+![image](https://github.com/user-attachments/assets/1a80395c-250f-4544-8c2e-4dd9d0358e98)
+![image](https://github.com/user-attachments/assets/5ac46c4b-b1e0-4e63-9a7a-d3f5cf41af5a)
+![image](https://github.com/user-attachments/assets/accd541a-5675-4e80-9bc0-0d053d6af8cb)
+![image](https://github.com/user-attachments/assets/ccc543b9-87c5-46a0-8989-f63aa8350b09)
+![image](https://github.com/user-attachments/assets/49ae68b6-d672-43a9-a4c9-90354816f203)
+![image](https://github.com/user-attachments/assets/5ab3a864-6b49-451a-9859-f8833e7bb6b6)
+![image](https://github.com/user-attachments/assets/7519d2f6-407d-4b7b-bc1a-32fa3e432c3b)
+
+That is all for today!
+
+See you tomorrow :)

From cde91a400a5211e322e4f6b4715033c3eaac1adf Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:19:25 +0900
Subject: [PATCH 202/638] Create 2024-4-10-day-100.md

---
 _posts/2024-4-10-day-100.md | 53 +++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 _posts/2024-4-10-day-100.md

diff --git a/_posts/2024-4-10-day-100.md b/_posts/2024-4-10-day-100.md
new file mode 100644
index 000000000..cf4399b56
--- /dev/null
+++ b/_posts/2024-4-10-day-100.md
@@ -0,0 +1,53 @@
+---
+layout: post
+title: (Day 100) Embeddings in practice + reading a couple of research papers + trying to deploy an LLM in production
+categories: [applying-knowledge,nlp,research]
+---
+
+## Hello :) Today is Day 100!
+A quick summary of today:
+* finished Vicki Boykis' ['What are embeddings?'](https://vickiboykis.com/what_are_embeddings/)
+* read 2 papers:
+  * Graph Convolutional Neural Networks for Web-Scale Recommender Systems (Ying et al., 2018)
+  * TwHIN: Embedding the Twitter Heterogeneous Information Network for Personalized Recommendation (El-Kishky et al., 2022)
+* tried to deploy a PDF chat app using streamlit (spoiler: failed)
+
+**My full notes from Vicki Boykis' What are embeddings? book**
+
+![image](https://github.com/user-attachments/assets/f08a4c50-16fe-44a6-817b-400a286af59e)
+![image](https://github.com/user-attachments/assets/32e2f55d-4635-4691-b3ab-f2c5e5c33d59)
+![image](https://github.com/user-attachments/assets/007d3734-96b6-47c8-bc05-1a38335a7dbc)
+![image](https://github.com/user-attachments/assets/8a43b0f9-29dd-463b-b2de-e19f2fae0f9c)
+![image](https://github.com/user-attachments/assets/58f605ba-ccc9-44cb-8269-f075f4e448c4)
+![image](https://github.com/user-attachments/assets/dbcb187a-9c8d-4209-986e-25e10f83d96d)
+
+The research papers below, I saw them in Vicki Boykis' vook and read them because they referenced GNNs.
+
+Graph Convolutional Neural Networks for Web-Scale Recommender Systems (Ying et al., 2018)
+The paper talks about a a large-scale deep recommendation engine developed and deployed at Pinterest called PinSage which tackles the challenge of scaling deep neural networks for graph-structured data to web-scale recommendation tasks with huge amounts of users and items (pictures in pinterest's case). PinSage utilizes an efficient Graph Convolutional Network (GCN) algorithm which combines efficient random walks and graph convolutions to generate embeddings of nodes (items) that incorporate both graph structure and node feature information.
+
+Some of the key points from the paper:
+
+* scalability: PinSage is trained on 7.5 billion examples on a graph with 3 billion nodes representing pins and boards and 18 billion edges making it one of the largest applications of deep graph embeddings
+* efficiency: PinSage leverages on-the-fly convolutions, producer-consumer minibatch construction, and efficient MapReduce inference to improve scalability
+* innovations: the model introduces new techniques like constructing convolutions via random walks, importance pooling(mean vs max vs importance pooling), and curriculum training which enhance the quality of learned representations and lead to big performance gains in downstream recommender system tasks
+* deployment and evaluation: PinSage is deployed for recommendation tasks at Pinterest showing state-of-the-art performance compared to other scalable deep content-based recommendation algorithms in offline metrics, controlled user studies, and A/B tests
+
+**TwHIN: Embedding the Twitter Heterogeneous Information Network for Personalized Recommendation (El-Kishky et al., 2022)**
+
+The paper paper talks about the development of TwHIN which tackles traditional tweet/user follow recommendation systems and networks, by using GNNs to understand complex relationships between users and tweets. The main part of this approach is in the usage of knowledge-graph embeddings - a technique that maps entities and relationships in the TwHIN onto a low-dimensional space. Using this new technique, Twitter can effectively capture the structural nuances of the network and learn embeddings that maximize the likelihood of observed interactions while minimizing those of unobserved ones. Also, traditional systems focus on recommending users to follow TwHIN extends to broader personalization tasks like recommending ads, filtering offensive content, and even improving search ranking.
+
+![image](https://github.com/user-attachments/assets/3aaab95d-b52b-487e-8133-c8385f455bd4)
+
+Knowledge graph are discussed later in the XCS224W: ML with Graphs course, so most likely after I study that content, I will be able to relate more to this particular paper. 
+
+
+Next, trying to deploy a streamlit pdf chat app
+
+It was not hard, I even uploaded it to [github](https://github.com/divakaivan/pdfs_chat_webapp) but I was developing it locally wondering if I can run a huggingface LLM in production. I ran into problems even during dev, because for some reason now when downloading and using gemma-2b-it my laptop was not able to handle it. As for the pdf, I used the What are embeddings? pdf (~60 pages), embedded them and put them on mongodb, but sadly I could not use it. I 'fought' this problem for a bit, and while I was eating I looked at some videos on how to deploy a huggingface model to production. I found [this](https://youtu.be/ZQPm2-uR9zA) lovely video about deploying on azure/aws/google cloud but sadly all require a credit card haha. So at least for now, I will put this on hold, but in the future, I will try to set up an account and get free resources to deploy a simple model for production inference. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 3356abd1a8135e3c28efab947fc5387aff1d8402 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Mon, 15 Jul 2024 03:19:58 +0900
Subject: [PATCH 203/638] Update 2024-4-9-day-99.md

---
 _posts/2024-4-9-day-99.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-4-9-day-99.md b/_posts/2024-4-9-day-99.md
index 51563c14c..ed469f610 100644
--- a/_posts/2024-4-9-day-99.md
+++ b/_posts/2024-4-9-day-99.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 99) XCS224W - ML with Graphs: Theory of GNNs
+title: (Day 99) XCS224W - ML with Graphs - Theory of GNNs
 categories: [theory,gnn]
 ---
 

From 31b3b77f7a90fc9bd331dc38b9299009c618ddc4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 16:23:23 +0900
Subject: [PATCH 204/638] Add files via upload

---
 _posts/2024-04-11-day-101.md | 5 +++++
 _posts/2024-04-12-day-102.md | 5 +++++
 _posts/2024-04-13-day-103.md | 5 +++++
 _posts/2024-04-14-day-104.md | 5 +++++
 _posts/2024-04-15-day-105.md | 5 +++++
 _posts/2024-04-16-day-106.md | 5 +++++
 _posts/2024-04-17-day-107.md | 5 +++++
 _posts/2024-04-18-day-108.md | 5 +++++
 _posts/2024-04-19-day-109.md | 5 +++++
 _posts/2024-04-20-day-110.md | 5 +++++
 _posts/2024-04-21-day-111.md | 5 +++++
 _posts/2024-04-22-day-112.md | 5 +++++
 _posts/2024-04-23-day-113.md | 5 +++++
 _posts/2024-04-24-day-114.md | 5 +++++
 _posts/2024-04-25-day-115.md | 5 +++++
 _posts/2024-04-26-day-116.md | 5 +++++
 _posts/2024-04-27-day-117.md | 5 +++++
 _posts/2024-04-28-day-118.md | 5 +++++
 _posts/2024-04-29-day-119.md | 5 +++++
 _posts/2024-04-30-day-120.md | 5 +++++
 _posts/2024-05-01-day-121.md | 5 +++++
 _posts/2024-05-02-day-122.md | 5 +++++
 _posts/2024-05-03-day-123.md | 5 +++++
 _posts/2024-05-04-day-124.md | 5 +++++
 _posts/2024-05-05-day-125.md | 5 +++++
 _posts/2024-05-06-day-126.md | 5 +++++
 _posts/2024-05-07-day-127.md | 5 +++++
 _posts/2024-05-08-day-128.md | 5 +++++
 _posts/2024-05-09-day-129.md | 5 +++++
 _posts/2024-05-10-day-130.md | 5 +++++
 _posts/2024-05-11-day-131.md | 5 +++++
 _posts/2024-05-12-day-132.md | 5 +++++
 _posts/2024-05-13-day-133.md | 5 +++++
 _posts/2024-05-14-day-134.md | 5 +++++
 _posts/2024-05-15-day-135.md | 5 +++++
 _posts/2024-05-16-day-136.md | 5 +++++
 _posts/2024-05-17-day-137.md | 5 +++++
 _posts/2024-05-18-day-138.md | 5 +++++
 _posts/2024-05-19-day-139.md | 5 +++++
 _posts/2024-05-20-day-140.md | 5 +++++
 _posts/2024-05-21-day-141.md | 5 +++++
 _posts/2024-05-22-day-142.md | 5 +++++
 _posts/2024-05-23-day-143.md | 5 +++++
 _posts/2024-05-24-day-144.md | 5 +++++
 _posts/2024-05-25-day-145.md | 5 +++++
 _posts/2024-05-26-day-146.md | 5 +++++
 _posts/2024-05-27-day-147.md | 5 +++++
 _posts/2024-05-28-day-148.md | 5 +++++
 _posts/2024-05-29-day-149.md | 5 +++++
 _posts/2024-05-30-day-150.md | 5 +++++
 _posts/2024-05-31-day-151.md | 5 +++++
 _posts/2024-06-01-day-152.md | 5 +++++
 _posts/2024-06-02-day-153.md | 5 +++++
 _posts/2024-06-03-day-154.md | 5 +++++
 _posts/2024-06-04-day-155.md | 5 +++++
 _posts/2024-06-05-day-156.md | 5 +++++
 _posts/2024-06-06-day-157.md | 5 +++++
 _posts/2024-06-07-day-158.md | 5 +++++
 _posts/2024-06-08-day-159.md | 5 +++++
 _posts/2024-06-09-day-160.md | 5 +++++
 _posts/2024-06-10-day-161.md | 5 +++++
 _posts/2024-06-11-day-162.md | 5 +++++
 _posts/2024-06-12-day-163.md | 5 +++++
 _posts/2024-06-13-day-164.md | 5 +++++
 _posts/2024-06-14-day-165.md | 5 +++++
 _posts/2024-06-15-day-166.md | 5 +++++
 _posts/2024-06-16-day-167.md | 5 +++++
 _posts/2024-06-17-day-168.md | 5 +++++
 _posts/2024-06-18-day-169.md | 5 +++++
 _posts/2024-06-19-day-170.md | 5 +++++
 _posts/2024-06-20-day-171.md | 5 +++++
 _posts/2024-06-21-day-172.md | 5 +++++
 _posts/2024-06-22-day-173.md | 5 +++++
 _posts/2024-06-23-day-174.md | 5 +++++
 _posts/2024-06-24-day-175.md | 5 +++++
 _posts/2024-06-25-day-176.md | 5 +++++
 _posts/2024-06-26-day-177.md | 5 +++++
 _posts/2024-06-27-day-178.md | 5 +++++
 _posts/2024-06-28-day-179.md | 5 +++++
 _posts/2024-06-29-day-180.md | 5 +++++
 _posts/2024-06-30-day-181.md | 5 +++++
 _posts/2024-07-01-day-182.md | 5 +++++
 _posts/2024-07-02-day-183.md | 5 +++++
 _posts/2024-07-03-day-184.md | 5 +++++
 _posts/2024-07-04-day-185.md | 5 +++++
 _posts/2024-07-05-day-186.md | 5 +++++
 _posts/2024-07-06-day-187.md | 5 +++++
 _posts/2024-07-07-day-188.md | 5 +++++
 _posts/2024-07-08-day-189.md | 5 +++++
 _posts/2024-07-09-day-190.md | 5 +++++
 _posts/2024-07-10-day-191.md | 5 +++++
 _posts/2024-07-11-day-192.md | 5 +++++
 _posts/2024-07-12-day-193.md | 5 +++++
 _posts/2024-07-13-day-194.md | 5 +++++
 _posts/2024-07-14-day-195.md | 5 +++++
 _posts/2024-07-15-day-196.md | 5 +++++
 _posts/2024-07-16-day-197.md | 5 +++++
 _posts/2024-07-17-day-198.md | 5 +++++
 _posts/2024-07-18-day-199.md | 5 +++++
 _posts/2024-07-19-day-200.md | 5 +++++
 100 files changed, 500 insertions(+)
 create mode 100644 _posts/2024-04-11-day-101.md
 create mode 100644 _posts/2024-04-12-day-102.md
 create mode 100644 _posts/2024-04-13-day-103.md
 create mode 100644 _posts/2024-04-14-day-104.md
 create mode 100644 _posts/2024-04-15-day-105.md
 create mode 100644 _posts/2024-04-16-day-106.md
 create mode 100644 _posts/2024-04-17-day-107.md
 create mode 100644 _posts/2024-04-18-day-108.md
 create mode 100644 _posts/2024-04-19-day-109.md
 create mode 100644 _posts/2024-04-20-day-110.md
 create mode 100644 _posts/2024-04-21-day-111.md
 create mode 100644 _posts/2024-04-22-day-112.md
 create mode 100644 _posts/2024-04-23-day-113.md
 create mode 100644 _posts/2024-04-24-day-114.md
 create mode 100644 _posts/2024-04-25-day-115.md
 create mode 100644 _posts/2024-04-26-day-116.md
 create mode 100644 _posts/2024-04-27-day-117.md
 create mode 100644 _posts/2024-04-28-day-118.md
 create mode 100644 _posts/2024-04-29-day-119.md
 create mode 100644 _posts/2024-04-30-day-120.md
 create mode 100644 _posts/2024-05-01-day-121.md
 create mode 100644 _posts/2024-05-02-day-122.md
 create mode 100644 _posts/2024-05-03-day-123.md
 create mode 100644 _posts/2024-05-04-day-124.md
 create mode 100644 _posts/2024-05-05-day-125.md
 create mode 100644 _posts/2024-05-06-day-126.md
 create mode 100644 _posts/2024-05-07-day-127.md
 create mode 100644 _posts/2024-05-08-day-128.md
 create mode 100644 _posts/2024-05-09-day-129.md
 create mode 100644 _posts/2024-05-10-day-130.md
 create mode 100644 _posts/2024-05-11-day-131.md
 create mode 100644 _posts/2024-05-12-day-132.md
 create mode 100644 _posts/2024-05-13-day-133.md
 create mode 100644 _posts/2024-05-14-day-134.md
 create mode 100644 _posts/2024-05-15-day-135.md
 create mode 100644 _posts/2024-05-16-day-136.md
 create mode 100644 _posts/2024-05-17-day-137.md
 create mode 100644 _posts/2024-05-18-day-138.md
 create mode 100644 _posts/2024-05-19-day-139.md
 create mode 100644 _posts/2024-05-20-day-140.md
 create mode 100644 _posts/2024-05-21-day-141.md
 create mode 100644 _posts/2024-05-22-day-142.md
 create mode 100644 _posts/2024-05-23-day-143.md
 create mode 100644 _posts/2024-05-24-day-144.md
 create mode 100644 _posts/2024-05-25-day-145.md
 create mode 100644 _posts/2024-05-26-day-146.md
 create mode 100644 _posts/2024-05-27-day-147.md
 create mode 100644 _posts/2024-05-28-day-148.md
 create mode 100644 _posts/2024-05-29-day-149.md
 create mode 100644 _posts/2024-05-30-day-150.md
 create mode 100644 _posts/2024-05-31-day-151.md
 create mode 100644 _posts/2024-06-01-day-152.md
 create mode 100644 _posts/2024-06-02-day-153.md
 create mode 100644 _posts/2024-06-03-day-154.md
 create mode 100644 _posts/2024-06-04-day-155.md
 create mode 100644 _posts/2024-06-05-day-156.md
 create mode 100644 _posts/2024-06-06-day-157.md
 create mode 100644 _posts/2024-06-07-day-158.md
 create mode 100644 _posts/2024-06-08-day-159.md
 create mode 100644 _posts/2024-06-09-day-160.md
 create mode 100644 _posts/2024-06-10-day-161.md
 create mode 100644 _posts/2024-06-11-day-162.md
 create mode 100644 _posts/2024-06-12-day-163.md
 create mode 100644 _posts/2024-06-13-day-164.md
 create mode 100644 _posts/2024-06-14-day-165.md
 create mode 100644 _posts/2024-06-15-day-166.md
 create mode 100644 _posts/2024-06-16-day-167.md
 create mode 100644 _posts/2024-06-17-day-168.md
 create mode 100644 _posts/2024-06-18-day-169.md
 create mode 100644 _posts/2024-06-19-day-170.md
 create mode 100644 _posts/2024-06-20-day-171.md
 create mode 100644 _posts/2024-06-21-day-172.md
 create mode 100644 _posts/2024-06-22-day-173.md
 create mode 100644 _posts/2024-06-23-day-174.md
 create mode 100644 _posts/2024-06-24-day-175.md
 create mode 100644 _posts/2024-06-25-day-176.md
 create mode 100644 _posts/2024-06-26-day-177.md
 create mode 100644 _posts/2024-06-27-day-178.md
 create mode 100644 _posts/2024-06-28-day-179.md
 create mode 100644 _posts/2024-06-29-day-180.md
 create mode 100644 _posts/2024-06-30-day-181.md
 create mode 100644 _posts/2024-07-01-day-182.md
 create mode 100644 _posts/2024-07-02-day-183.md
 create mode 100644 _posts/2024-07-03-day-184.md
 create mode 100644 _posts/2024-07-04-day-185.md
 create mode 100644 _posts/2024-07-05-day-186.md
 create mode 100644 _posts/2024-07-06-day-187.md
 create mode 100644 _posts/2024-07-07-day-188.md
 create mode 100644 _posts/2024-07-08-day-189.md
 create mode 100644 _posts/2024-07-09-day-190.md
 create mode 100644 _posts/2024-07-10-day-191.md
 create mode 100644 _posts/2024-07-11-day-192.md
 create mode 100644 _posts/2024-07-12-day-193.md
 create mode 100644 _posts/2024-07-13-day-194.md
 create mode 100644 _posts/2024-07-14-day-195.md
 create mode 100644 _posts/2024-07-15-day-196.md
 create mode 100644 _posts/2024-07-16-day-197.md
 create mode 100644 _posts/2024-07-17-day-198.md
 create mode 100644 _posts/2024-07-18-day-199.md
 create mode 100644 _posts/2024-07-19-day-200.md

diff --git a/_posts/2024-04-11-day-101.md b/_posts/2024-04-11-day-101.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-11-day-101.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-12-day-102.md b/_posts/2024-04-12-day-102.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-12-day-102.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-13-day-103.md b/_posts/2024-04-13-day-103.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-13-day-103.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-14-day-104.md b/_posts/2024-04-14-day-104.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-14-day-104.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-15-day-105.md b/_posts/2024-04-15-day-105.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-15-day-105.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-16-day-106.md b/_posts/2024-04-16-day-106.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-16-day-106.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-17-day-107.md b/_posts/2024-04-17-day-107.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-17-day-107.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-18-day-108.md b/_posts/2024-04-18-day-108.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-18-day-108.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-19-day-109.md b/_posts/2024-04-19-day-109.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-19-day-109.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-20-day-110.md b/_posts/2024-04-20-day-110.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-20-day-110.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-21-day-111.md b/_posts/2024-04-21-day-111.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-21-day-111.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-22-day-112.md b/_posts/2024-04-22-day-112.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-22-day-112.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-23-day-113.md b/_posts/2024-04-23-day-113.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-23-day-113.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-24-day-114.md b/_posts/2024-04-24-day-114.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-24-day-114.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-25-day-115.md b/_posts/2024-04-25-day-115.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-25-day-115.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-26-day-116.md b/_posts/2024-04-26-day-116.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-26-day-116.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-27-day-117.md b/_posts/2024-04-27-day-117.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-27-day-117.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-28-day-118.md b/_posts/2024-04-28-day-118.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-28-day-118.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-29-day-119.md b/_posts/2024-04-29-day-119.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-29-day-119.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-04-30-day-120.md b/_posts/2024-04-30-day-120.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-04-30-day-120.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-01-day-121.md b/_posts/2024-05-01-day-121.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-01-day-121.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-02-day-122.md b/_posts/2024-05-02-day-122.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-02-day-122.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-03-day-123.md b/_posts/2024-05-03-day-123.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-03-day-123.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-04-day-124.md b/_posts/2024-05-04-day-124.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-04-day-124.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-05-day-125.md b/_posts/2024-05-05-day-125.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-05-day-125.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-06-day-126.md b/_posts/2024-05-06-day-126.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-06-day-126.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-07-day-127.md b/_posts/2024-05-07-day-127.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-07-day-127.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-08-day-128.md b/_posts/2024-05-08-day-128.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-08-day-128.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-09-day-129.md b/_posts/2024-05-09-day-129.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-09-day-129.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-10-day-130.md b/_posts/2024-05-10-day-130.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-10-day-130.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-11-day-131.md b/_posts/2024-05-11-day-131.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-11-day-131.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-12-day-132.md b/_posts/2024-05-12-day-132.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-12-day-132.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-13-day-133.md b/_posts/2024-05-13-day-133.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-13-day-133.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-14-day-134.md b/_posts/2024-05-14-day-134.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-14-day-134.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-15-day-135.md b/_posts/2024-05-15-day-135.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-15-day-135.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-16-day-136.md b/_posts/2024-05-16-day-136.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-16-day-136.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-17-day-137.md b/_posts/2024-05-17-day-137.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-17-day-137.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-18-day-138.md b/_posts/2024-05-18-day-138.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-18-day-138.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-19-day-139.md b/_posts/2024-05-19-day-139.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-19-day-139.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-20-day-140.md b/_posts/2024-05-20-day-140.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-20-day-140.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-21-day-141.md b/_posts/2024-05-21-day-141.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-21-day-141.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-22-day-142.md b/_posts/2024-05-22-day-142.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-22-day-142.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-23-day-143.md b/_posts/2024-05-23-day-143.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-23-day-143.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-24-day-144.md b/_posts/2024-05-24-day-144.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-24-day-144.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-25-day-145.md b/_posts/2024-05-25-day-145.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-25-day-145.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-26-day-146.md b/_posts/2024-05-26-day-146.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-26-day-146.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-27-day-147.md b/_posts/2024-05-27-day-147.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-27-day-147.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-28-day-148.md b/_posts/2024-05-28-day-148.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-28-day-148.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-29-day-149.md b/_posts/2024-05-29-day-149.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-29-day-149.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-30-day-150.md b/_posts/2024-05-30-day-150.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-30-day-150.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-05-31-day-151.md b/_posts/2024-05-31-day-151.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-05-31-day-151.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-01-day-152.md b/_posts/2024-06-01-day-152.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-01-day-152.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-02-day-153.md b/_posts/2024-06-02-day-153.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-02-day-153.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-03-day-154.md b/_posts/2024-06-03-day-154.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-03-day-154.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-04-day-155.md b/_posts/2024-06-04-day-155.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-04-day-155.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-05-day-156.md b/_posts/2024-06-05-day-156.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-05-day-156.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-06-day-157.md b/_posts/2024-06-06-day-157.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-06-day-157.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-07-day-158.md b/_posts/2024-06-07-day-158.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-07-day-158.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-08-day-159.md b/_posts/2024-06-08-day-159.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-08-day-159.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-09-day-160.md b/_posts/2024-06-09-day-160.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-09-day-160.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-10-day-161.md b/_posts/2024-06-10-day-161.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-10-day-161.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-11-day-162.md b/_posts/2024-06-11-day-162.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-11-day-162.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-12-day-163.md b/_posts/2024-06-12-day-163.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-12-day-163.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-13-day-164.md b/_posts/2024-06-13-day-164.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-13-day-164.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-14-day-165.md b/_posts/2024-06-14-day-165.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-14-day-165.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-15-day-166.md b/_posts/2024-06-15-day-166.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-15-day-166.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-16-day-167.md b/_posts/2024-06-16-day-167.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-16-day-167.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-17-day-168.md b/_posts/2024-06-17-day-168.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-17-day-168.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-18-day-169.md b/_posts/2024-06-18-day-169.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-18-day-169.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-19-day-170.md b/_posts/2024-06-19-day-170.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-19-day-170.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-20-day-171.md b/_posts/2024-06-20-day-171.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-20-day-171.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-21-day-172.md b/_posts/2024-06-21-day-172.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-21-day-172.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-22-day-173.md b/_posts/2024-06-22-day-173.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-22-day-173.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-23-day-174.md b/_posts/2024-06-23-day-174.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-23-day-174.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-24-day-175.md b/_posts/2024-06-24-day-175.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-24-day-175.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-25-day-176.md b/_posts/2024-06-25-day-176.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-25-day-176.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-26-day-177.md b/_posts/2024-06-26-day-177.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-26-day-177.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-27-day-178.md b/_posts/2024-06-27-day-178.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-27-day-178.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-28-day-179.md b/_posts/2024-06-28-day-179.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-28-day-179.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-29-day-180.md b/_posts/2024-06-29-day-180.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-29-day-180.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-06-30-day-181.md b/_posts/2024-06-30-day-181.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-06-30-day-181.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-01-day-182.md b/_posts/2024-07-01-day-182.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-01-day-182.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-02-day-183.md b/_posts/2024-07-02-day-183.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-02-day-183.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-03-day-184.md b/_posts/2024-07-03-day-184.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-03-day-184.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-04-day-185.md b/_posts/2024-07-04-day-185.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-04-day-185.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-05-day-186.md b/_posts/2024-07-05-day-186.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-05-day-186.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-06-day-187.md b/_posts/2024-07-06-day-187.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-06-day-187.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-07-day-188.md b/_posts/2024-07-07-day-188.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-07-day-188.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-08-day-189.md b/_posts/2024-07-08-day-189.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-08-day-189.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-09-day-190.md b/_posts/2024-07-09-day-190.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-09-day-190.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-10-day-191.md b/_posts/2024-07-10-day-191.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-10-day-191.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-11-day-192.md b/_posts/2024-07-11-day-192.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-11-day-192.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-12-day-193.md b/_posts/2024-07-12-day-193.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-12-day-193.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-13-day-194.md b/_posts/2024-07-13-day-194.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-13-day-194.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-14-day-195.md b/_posts/2024-07-14-day-195.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-14-day-195.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-15-day-196.md b/_posts/2024-07-15-day-196.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-15-day-196.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-16-day-197.md b/_posts/2024-07-16-day-197.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-16-day-197.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-17-day-198.md b/_posts/2024-07-17-day-198.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-17-day-198.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-18-day-199.md b/_posts/2024-07-18-day-199.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-18-day-199.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-19-day-200.md b/_posts/2024-07-19-day-200.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-19-day-200.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---

From 985c95642a72194ea0c7e05f919445dc71634a63 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 16:32:46 +0900
Subject: [PATCH 205/638] Add files via upload

---
 _posts/2024-07-20-day-201.md | 5 +++++
 _posts/2024-07-21-day-202.md | 5 +++++
 _posts/2024-07-22-day-203.md | 5 +++++
 _posts/2024-07-23-day-204.md | 5 +++++
 _posts/2024-07-24-day-205.md | 5 +++++
 _posts/2024-07-25-day-206.md | 5 +++++
 _posts/2024-07-26-day-207.md | 5 +++++
 _posts/2024-07-27-day-208.md | 5 +++++
 _posts/2024-07-28-day-209.md | 5 +++++
 _posts/2024-07-29-day-210.md | 5 +++++
 _posts/2024-07-30-day-211.md | 5 +++++
 _posts/2024-07-31-day-212.md | 5 +++++
 _posts/2024-08-01-day-213.md | 5 +++++
 _posts/2024-08-02-day-214.md | 5 +++++
 _posts/2024-08-03-day-215.md | 5 +++++
 _posts/2024-08-04-day-216.md | 5 +++++
 _posts/2024-08-05-day-217.md | 5 +++++
 _posts/2024-08-06-day-218.md | 5 +++++
 18 files changed, 90 insertions(+)
 create mode 100644 _posts/2024-07-20-day-201.md
 create mode 100644 _posts/2024-07-21-day-202.md
 create mode 100644 _posts/2024-07-22-day-203.md
 create mode 100644 _posts/2024-07-23-day-204.md
 create mode 100644 _posts/2024-07-24-day-205.md
 create mode 100644 _posts/2024-07-25-day-206.md
 create mode 100644 _posts/2024-07-26-day-207.md
 create mode 100644 _posts/2024-07-27-day-208.md
 create mode 100644 _posts/2024-07-28-day-209.md
 create mode 100644 _posts/2024-07-29-day-210.md
 create mode 100644 _posts/2024-07-30-day-211.md
 create mode 100644 _posts/2024-07-31-day-212.md
 create mode 100644 _posts/2024-08-01-day-213.md
 create mode 100644 _posts/2024-08-02-day-214.md
 create mode 100644 _posts/2024-08-03-day-215.md
 create mode 100644 _posts/2024-08-04-day-216.md
 create mode 100644 _posts/2024-08-05-day-217.md
 create mode 100644 _posts/2024-08-06-day-218.md

diff --git a/_posts/2024-07-20-day-201.md b/_posts/2024-07-20-day-201.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-20-day-201.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-21-day-202.md b/_posts/2024-07-21-day-202.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-21-day-202.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-22-day-203.md b/_posts/2024-07-22-day-203.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-22-day-203.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-23-day-204.md b/_posts/2024-07-23-day-204.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-23-day-204.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-24-day-205.md b/_posts/2024-07-24-day-205.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-24-day-205.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-25-day-206.md b/_posts/2024-07-25-day-206.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-25-day-206.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-26-day-207.md b/_posts/2024-07-26-day-207.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-26-day-207.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-27-day-208.md b/_posts/2024-07-27-day-208.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-27-day-208.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-28-day-209.md b/_posts/2024-07-28-day-209.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-28-day-209.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-29-day-210.md b/_posts/2024-07-29-day-210.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-29-day-210.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-30-day-211.md b/_posts/2024-07-30-day-211.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-30-day-211.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-07-31-day-212.md b/_posts/2024-07-31-day-212.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-07-31-day-212.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-08-01-day-213.md b/_posts/2024-08-01-day-213.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-08-01-day-213.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-08-02-day-214.md b/_posts/2024-08-02-day-214.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-08-02-day-214.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-08-03-day-215.md b/_posts/2024-08-03-day-215.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-08-03-day-215.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-08-04-day-216.md b/_posts/2024-08-04-day-216.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-08-04-day-216.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-08-05-day-217.md b/_posts/2024-08-05-day-217.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-08-05-day-217.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---
diff --git a/_posts/2024-08-06-day-218.md b/_posts/2024-08-06-day-218.md
new file mode 100644
index 000000000..cadf548b6
--- /dev/null
+++ b/_posts/2024-08-06-day-218.md
@@ -0,0 +1,5 @@
+---
+layout: post
+title: 
+categories:
+---

From 4dc16c624c52581077d1953e5d82639f9e403981 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 16:45:23 +0900
Subject: [PATCH 206/638] Update 2024-04-11-day-101.md

---
 _posts/2024-04-11-day-101.md | 168 ++++++++++++++++++++++++++++++++++-
 1 file changed, 166 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-11-day-101.md b/_posts/2024-04-11-day-101.md
index cadf548b6..af70c8271 100644
--- a/_posts/2024-04-11-day-101.md
+++ b/_posts/2024-04-11-day-101.md
@@ -1,5 +1,169 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 101) Doing the Google Cloud Digital Leader Learning Path
+categories: [cloud,model-deployment]
 ---
+
+## Hello :) Today is Day 101!
+A quick summary of today:
+* did [GCP's Digital Leader Learning Path](https://www.cloudskillsboost.google/paths/9)
+
+After yesterday's attempt and failure to deploy an LLM to production, I wanted to learn a bit more about the cloud platforms used. The top 3 are GCP, Azure and AWS. From past experiences (in my previous job at Lloyds Banking Group) I heard many people using GCP (some Azure), but also that AWS is very popular. I decided to check out what kind of learning paths GCP and AWS have, and I found the GCP's Digital leader learning path and [AWS' Cloud Practitioner: Foundamental](https://aws.amazon.com/certification/exams/?nc2=sb_ce_exm) and both have certification. 
+
+GCP's Digital Leader Learning Path covers the below topics:
+
+* Digital Transformation with Google Cloud
+* Exploring Data Transformation with Google Cloud
+* Innovating with Google Cloud Artificial Intelligence
+* Modernize Infrastructure and Applications with Google Cloud
+* Trust and Security with Google Cloud
+* Scaling with Google Cloud Operations
+
+and below are my notes.
+
+**The cloud**
+
+_The cloud refers to a network of data centers that store and process information accessible via the internet. It replaces the traditional setup of managing IT infrastructure by combining software, servers, networks, and security into one accessible entity. Cloud technology supports digital transformation by offering scalable, on-demand computing resources through various implementations, including on-premises, private cloud, public cloud, hybrid cloud, and multicloud. On-premises infrastructure involves hosting hardware and software on-site while private cloud dedicates infrastructure to a single organization. Public cloud services are managed by third-party providers and shared among multiple organizations. Hybrid cloud combines different environments, like public and private clouds, while multicloud involves using multiple public cloud providers simultaneously. Most organizations today adopt a multicloud strategy, combining public and private clouds for flexibility and efficiency._
+
+Its benefits
+
+* scalability
+* flexibility
+* agility
+* strategic value
+* security
+
+Challenges that lead to a digital transformation
+
+* data management
+* technology infrastructure
+* hybrid workplace
+* security
+* sustainability
+
+**Strategies for going on the cloud**
+
+* Private Cloud: Involves virtualized servers in an organization's data centers or those of a private cloud provider, offering benefits like self-service, scalability, and elasticity. It's preferred when significant infrastructure investments have been made or for regulatory compliance
+* Hybrid Cloud: Combines private and public cloud environments, allowing applications to run across different environments. It's a common setup today, enabling organizations to leverage both on-premises servers and public cloud services
+* Multicloud: Involves using multiple public cloud providers, leveraging the strengths of each. Organizations choose multicloud to access the latest technologies, modernize at their own pace, improve ROI, enhance flexibility, and ensure regulatory compliance
+
+**Abstractions**
+
+![image](https://github.com/user-attachments/assets/4fd714e8-0fde-435c-a192-1cbdee154630)
+
+
+* On-premises: It's like owning a car. You're responsible for its usage and maintenance, and upgrading means buying a new car, which can be time-consuming and costly
+IaaS (Infrastructure as a Service): It's like leasing a car. You choose a car and drive it wherever you want, but the car isn't yours. Upgrading is easier as you can simply lease a new car
+* PaaS (Platform as a Service): It's like taking a taxi. You provide specific directions, like the code, but the driver (platform provider) does the actual driving (manages the underlying infrastructure)
+* SaaS (Software as a Service): It's like going by bus. You still get access to transport (software), but it's less customizable. Buses have designated routes, and you share the space with other passengers (users)
+
+**Responsibilities depending on model**
+
+![image](https://github.com/user-attachments/assets/257eb793-ef14-4c94-bfac-47972938cf7c)
+
+![image](https://github.com/user-attachments/assets/a44a8295-6203-46ee-b512-e887a67cc375)
+
+**Data Value Chain**
+
+The data value chain is like a assembly line, where raw data undergoes various stages to eventually generate actionable insights.
+
+1. data genesis
+2. data collection
+3. data processing
+4. data storage
+5. data analysis
+6. data activation
+
+**Data governance**
+
+Data governance involves setting internal standards and policies for data collection, storage, processing, and disposal, while also adhering to external standards and regulations. It aims to make data available across the organization's lifecycle, ensuring accessibility, security, and compliance. Implementing data governance brings several benefits, including increased data value, improved decision-making, cost control, regulatory compliance, enhanced trust, risk management, and efficient data access. Organizations without effective data governance may face compliance violations, poor data quality, delayed analysis, missed opportunities, and reduced accuracy in AI models. As data becomes increasingly crucial, every organization needs robust data governance to navigate digital transformations successfully.
+
+**Data storage options**
+
+![image](https://github.com/user-attachments/assets/d578e69c-c64d-4169-b0c4-c42e7270a4bc)
+
+![image](https://github.com/user-attachments/assets/39e2a62d-ca1d-4240-93bc-f1e4d5394bd8)
+
+**Looker**
+
+It is a Google Cloud business intelligence platform, enables individuals and teams to analyze, visualize, and share data through interactive dashboards and reports. It supports integration with BigQuery and over 60 SQL databases, ensuring rich visualizations without compromising performance or security. Looker's web-based interface facilitates easy integration into existing workflows and collaboration across teams.
+
+**PUB/SUB and DataFlow**
+
+The data pipeline begins with Data Ingestion, where diverse data streams from sources like IoT devices are received, often through Pub/Sub. Dataflow, a serverless and fully managed service by Google, then processes this data for analysis. It handles both streaming and batch data, simplifying ETL processes with Apache Beam's unified programming model. Dataflow's auto-scaling and management features free users from infrastructure concerns, allowing more focus on analyzing insights from datasets.
+
+**ML options on GCP**
+
+![image](https://github.com/user-attachments/assets/1d4452b1-ed58-4828-b702-e697ec9a1b09)
+
+**Other full AI solutions**
+
+* Contact centre AI
+* Document AI
+* Discovery AI for retail
+* Cloud talent solution
+
+**Cloud migration terms**
+
+* Workload: A specific application, service, or capability that can be run in the cloud or on premises. Examples include containers, databases, and virtual machines
+* Retire: Removing a workload from a platform due to reasons like it being unnecessary, not cost-effective, secure, or compatible with a specific platform
+* Retain: Intentionally keeping a workload, often on premises or in a hybrid cloud environment, under the business's management without full cloud provider control
+* Rehost: Migrating a workload to the cloud without altering its code or architecture, also known as 'lift and shift'
+* Replatform: Migrating a workload to the cloud while making some changes to its code or architecture, often referred to as 'move and improve'
+* Refactor: Changing the code of a workload, possibly to utilize cloud-based microservices or serverless architecture, aiming for efficiency, scalability, or security improvements
+* Reimagine: Rethinking how an organization uses technology, particularly cloud computing, to achieve its business goals, potentially involving reconsideration of cloud strategy and adoption of technologies like AI and ML for efficiency, cost reduction, and agility improvements
+
+**Benefits of running compute workloads on the Cloud**
+
+* Total Cost of Ownership (TCO) Reduction: Cloud computing saves money by eliminating the need for purchasing and maintaining physical infrastructure. Pay-as-you-go models and long-term commitment discounts further reduce costs
+* Scalability: Cloud offers the ability to easily scale resources up or down to meet changing demand without significant upfront investments
+* Reliability: Cloud providers offer high reliability and uptime through redundant data centers and automated problem detection and resolution
+* Security: Cloud providers ensure data security through features like encryption, identity and access management, network security, and real-time threat detection and response
+* Flexibility: Organizations can choose and adapt cloud services according to their needs, such as increasing storage space or adding new services
+* Abstraction: Cloud providers handle infrastructure management, allowing customers to focus on their applications without worrying about underlying infrastructure details. This abstraction also enables access to a wide range of services and technologies
+
+**Google’s Compute Engine**
+
+Google Compute Engine is an Infrastructure as a Service (IaaS) offering from Google Cloud Platform (GCP). It allows users to create and manage virtual machines (VMs) on Google's infrastructure. Compute Engine provides scalable, high-performance virtual machines that can run various workloads, from simple web applications to complex data processing tasks
+
+**Essential Cloud Security Terms and Concepts**
+
+* Privileged Access Security Model: Grants specific users broader access to resources, but misuse can pose risks. Requires careful management and monitoring
+* Least Privileged Security Principle: Advocates granting users only the access needed for their job responsibilities, reducing the risk of unauthorized access to sensitive data
+* Zero-Trust Architecture: Assumes no user or device can be trusted by default. Requires authentication and authorization before accessing resources, ensuring robust security
+* Security by Default: Integrates security measures into systems and applications from the initial stages of development, establishing a strong security foundation in Cloud environments
+* Security Posture: Evaluates an organization's overall security status in its Cloud environment by assessing security controls, policies, and practices
+* Cyber Resilience: The ability to withstand and recover quickly from cyber attacks by identifying, assessing, and mitigating risks, responding to incidents effectively, and recovering from disruptions quickly
+
+**Data sovereignity and Data residency**
+
+* Data Sovereignty: Refers to the legal concept that data is subject to the laws and regulations of the country where it resides. For example, GDPR in the EU mandates compliance with data protection laws for EU citizens' personal data
+* Data Residency: Refers to the physical location where data is stored or processed. Some countries require data to be stored within their borders to ensure compliance with local laws
+
+**GCP access levels**
+
+![image](https://github.com/user-attachments/assets/6e3a70b7-64b4-4873-a318-48d8feb4271a)
+
+**Operational excellence and reliability at scale**
+
+![image](https://github.com/user-attachments/assets/f22454b0-cf77-495a-a4cd-4af7a88f6e21)
+
+**Service level indicators, objectives and agreements**
+
+![image](https://github.com/user-attachments/assets/569c6654-bb65-41cf-9536-58e2ea602df4)
+
+![image](https://github.com/user-attachments/assets/af9dcd42-ad2b-432b-8cf7-f6e7b3a7ba8d)
+
+![image](https://github.com/user-attachments/assets/ef368063-800f-49a2-95e1-4d2e41ce7a76)
+
+
+That is all ^^
+
+![image](https://github.com/user-attachments/assets/6d0fb9de-d9fc-413b-92e2-216128760c0c)
+
+
+I will also look at AWS' for-beginners course and certification and try to make an informed decision into which exam to invest.
+
+That is all for today!
+
+See you tomorrow :)

From 3a9ce255ee051b0d2b5f12891784440cefb3f96f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:56:06 +0900
Subject: [PATCH 207/638] Update 2024-04-12-day-102.md

---
 _posts/2024-04-12-day-102.md | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-12-day-102.md b/_posts/2024-04-12-day-102.md
index cadf548b6..472934309 100644
--- a/_posts/2024-04-12-day-102.md
+++ b/_posts/2024-04-12-day-102.md
@@ -1,5 +1,37 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 102) Label propagation in ML with Graphs
+categories: [theory,gnn]
 ---
+
+## Hello :) Today is Day 102!
+A quick summary of today:
+* continued with XCS224W: ML with Graphs and covered the next part - Label propagation
+
+Before I share my notes, I saw that all the 3 major cloud providers - AWS, GCP and Azure have beginner cloud certification. But I am yet to decide in which one to invest more time and eventually money for the exam. One the one hand, yes it is basic intro to the respective company's cloud platform, but it also can show some initiative and show that I know at least something about the cloud to potential employers. I made post on 2 subredits (learnmachinelearning and deeplearning) and asked on 2 discord servers, but so far not many people voted, but AWS seems to be winning. I will keep this certification in mind.
+
+
+
+Besides that, continued with the XCS224W: ML with graphs course and covered the next module which is about label propagation, and below are my notes.
+
+Covered topics: correlations in networks (homophily and influence), how do we leverage node correlations in networks, label propagation, problems with LP and GNNs, Correct & Smooth, Masked label prediciton
+
+![image](https://github.com/user-attachments/assets/62a81b48-9f2c-45e4-9e76-bbc2002ee3d1)
+
+![image](https://github.com/user-attachments/assets/c1bfe89f-65c0-4ff1-afa4-39c95d7f2b91)
+
+![image](https://github.com/user-attachments/assets/f2bf1feb-0d5b-49fb-8c2e-9bd2611c57cb)
+
+![image](https://github.com/user-attachments/assets/66c10413-a9d9-4ad6-93b8-88eab4a8c9dc)
+
+![image](https://github.com/user-attachments/assets/ab603770-10d5-44a6-b2df-8699eb26ff77)
+
+![image](https://github.com/user-attachments/assets/00b32514-9c00-43b9-ab20-13c4e572113a)
+
+![image](https://github.com/user-attachments/assets/17511f65-34cf-45eb-8c5b-943c1ee9c29c)
+
+![image](https://github.com/user-attachments/assets/497c3ccb-e765-4f26-8398-018e25410387)
+
+That is all for today!
+
+See you tomorrow :) 

From 46fc750f39ff0f871a0c9f6122452f2a937d347e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:01:29 +0900
Subject: [PATCH 208/638] Update 2024-04-13-day-103.md

---
 _posts/2024-04-13-day-103.md | 59 ++++++++++++++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-13-day-103.md b/_posts/2024-04-13-day-103.md
index cadf548b6..c4940985e 100644
--- a/_posts/2024-04-13-day-103.md
+++ b/_posts/2024-04-13-day-103.md
@@ -1,5 +1,60 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 103) Knowledge Graphs
+categories: [gnn,theory]
 ---
+
+## Hello :) Today is Day 103!
+A quick summary of today:
+* continued with XCS224W: ML with Graphs and covered 2/3rds of module 5: Knowledge Graphs
+
+1st part is about ML with Heterogeneous graphs, and 2nd part is about Knowledge Graph embeddings.
+
+**ML with Heterogeneous graphs**
+
+Covered topics: heterogeneous graph,s Relational GCNs, benchmarks for heterogeneous graphs, Heterogeneous Graph Transformers, designing, heterogeneous GNNs
+
+![image](https://github.com/user-attachments/assets/875f4dde-69b9-431b-bc86-59ee9b9de16c)
+
+![image](https://github.com/user-attachments/assets/6c6f370f-3a5b-4144-a9f8-f52403d47616)
+
+![image](https://github.com/user-attachments/assets/2fac6572-f9e5-4ee4-8ea6-2f73c201302e)
+
+![image](https://github.com/user-attachments/assets/01477394-404b-4022-897c-adf116939821)
+
+![image](https://github.com/user-attachments/assets/96ed3cbc-aeed-4ea0-ac10-dc704fc86a05)
+
+![image](https://github.com/user-attachments/assets/d5398589-692c-4a83-81c9-52336b7c7ac9)
+
+![image](https://github.com/user-attachments/assets/d4abb401-c205-4e1c-aba7-641aa9d3edde)
+
+![image](https://github.com/user-attachments/assets/4e618b2e-4553-4125-8ae0-ebfc953d17c4)
+
+![image](https://github.com/user-attachments/assets/a59a10cf-2132-440c-9691-f0c9e7be6ed8)
+
+![image](https://github.com/user-attachments/assets/8ade05a5-d5b5-4e04-9f36-744a0698baed)
+
+
+**Knowledge Graph embeddings**
+
+Covered topics: why do we need KGs, KG completion task, connectivity patterns in KGs, TransE, TransR, DistMult, ComplEx, KG embeddings in practice
+
+![image](https://github.com/user-attachments/assets/2a78df4d-f19e-4b21-882c-922e9faa5fa3)
+
+![image](https://github.com/user-attachments/assets/3f3d7a21-fd19-4142-859e-be4f64d88a28)
+
+![image](https://github.com/user-attachments/assets/14b4cedc-257f-4bfb-ba20-220a2253b1da)
+
+![image](https://github.com/user-attachments/assets/e27612ab-8e0e-4ef1-802b-b5c6e43fc76d)
+
+![image](https://github.com/user-attachments/assets/b05d92b9-7ca3-40cb-baa0-e520f3fc3b44)
+
+![image](https://github.com/user-attachments/assets/15d68c0a-cdee-4fdc-8e05-c81144aae168)
+
+![image](https://github.com/user-attachments/assets/97120332-6225-4e9c-8109-5f1908a6983e)
+
+![image](https://github.com/user-attachments/assets/94f4ca01-ed19-4afd-bf2f-7f76ca6771e8)
+
+That is all for today!
+
+See you tomorrow :)

From 6325211f41381bc28a6dec69c9edb7c12301d4c4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:08:38 +0900
Subject: [PATCH 209/638] Update 2024-04-14-day-104.md

---
 _posts/2024-04-14-day-104.md | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-14-day-104.md b/_posts/2024-04-14-day-104.md
index cadf548b6..b0d28a850 100644
--- a/_posts/2024-04-14-day-104.md
+++ b/_posts/2024-04-14-day-104.md
@@ -1,5 +1,28 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 104) Reasoning over Knowledge Graphs
+categories: [theory,gnn]
 ---
+
+## Hello :) Today is Day 104!
+A quick summary of today:
+* covered Reasoning over Knowledge Graphs (last part of the module about Knowledge Graphs from XCS224W: ML With Graphs)
+
+Covered topics: one-hop queries, path queries, predictive queries, conjunctive queries, Query2box, entity-to-box distance, embeddings AND-OR queries, Query2box training
+
+![image](https://github.com/user-attachments/assets/d0a32d86-8e14-4322-86eb-a168007243a4)
+![image](https://github.com/user-attachments/assets/642d4ef5-4874-4dc2-8676-4a254d642124)
+![image](https://github.com/user-attachments/assets/66f4cb85-ea38-4875-8f05-7524fe3f5886)
+![image](https://github.com/user-attachments/assets/444c313b-be2c-4b5d-ace1-1a80920303f2)
+![image](https://github.com/user-attachments/assets/708f1767-c297-4ea1-9ee0-b7f2219e3bee)
+![image](https://github.com/user-attachments/assets/8fedd97f-6a10-45d0-a224-0d16ea0d90ab)
+![image](https://github.com/user-attachments/assets/3b9c1f33-3739-4abb-9779-05c8fc8252df)
+![image](https://github.com/user-attachments/assets/471d0fad-a145-4cc5-bc50-7cba5457d271)
+![image](https://github.com/user-attachments/assets/6b7a13f9-a746-405e-915a-8c63c2d8e6cd)
+![image](https://github.com/user-attachments/assets/d0b567ff-1877-45b4-869d-89fc8d30acd2)
+
+That is all for today!
+
+See you tomorrow :)
+
+

From bef6fe4c18c2e7cca214ca1832545475e834c1bf Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:12:26 +0900
Subject: [PATCH 210/638] Update 2024-04-15-day-105.md

---
 _posts/2024-04-15-day-105.md | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-15-day-105.md b/_posts/2024-04-15-day-105.md
index cadf548b6..825d9b426 100644
--- a/_posts/2024-04-15-day-105.md
+++ b/_posts/2024-04-15-day-105.md
@@ -1,5 +1,33 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 105) Network subgraph counting and matching
+categories: [theory,gnn]
 ---
+
+## Hello :) Today is Day 105!
+A quick summary of today:
+* Frequent Subgraph Mining with GNNs (covered the next part of Stanford's XCS224W: ML with Graphs)
+
+Covered topics:
+
+Subgraphs, graph isomorphism, subgraph isomorphism, network motifs, subgraph frequency, motif significance, Z-score for statistical significance, significance profile, subgraph matching, isomorphism as an ML task, Order Embedding Space, training neural subgraph matching, frequent motif mining, SPMiner
+
+![image](https://github.com/user-attachments/assets/ea162e4c-7b8d-40a0-a1d4-69519f52450a)
+![image](https://github.com/user-attachments/assets/266ad719-1dbc-4188-ade9-625fcbfbf66b)
+![image](https://github.com/user-attachments/assets/8017885c-ba0e-4f29-9bea-5e96505cb705)
+![image](https://github.com/user-attachments/assets/ee0cc694-06dc-40e3-bd44-6d545d9380c8)
+![image](https://github.com/user-attachments/assets/f1a3e8a5-3c99-45f6-80f6-5daaaf42363c)
+![image](https://github.com/user-attachments/assets/214ba9da-66cd-40d0-9605-69d65ae74188)
+![image](https://github.com/user-attachments/assets/9a761f7e-ad23-410c-b319-0c8cb207f69f)
+![image](https://github.com/user-attachments/assets/94ff3acb-1064-40c0-85e4-1caba4858fbc)
+![image](https://github.com/user-attachments/assets/912557ba-673c-415f-88fb-d25d73c3daa2)
+![image](https://github.com/user-attachments/assets/9821e0df-757e-48c7-a44e-5207594d0784)
+![image](https://github.com/user-attachments/assets/8c66bd8f-f12f-428d-a5b6-808dc0549bd1)
+![image](https://github.com/user-attachments/assets/3ccbc2c8-a4b1-4e8d-9ecf-2890540e5b26)
+![image](https://github.com/user-attachments/assets/8c04b988-7f96-4ee4-bb3b-5a350e4bd721)
+![image](https://github.com/user-attachments/assets/cf591bfc-166e-4650-85e0-9e7a4731a30f)
+![image](https://github.com/user-attachments/assets/f0fe652a-50aa-4f65-b0b1-04ea76762f1d)
+
+That is all for today!
+
+See you tomorrow :)

From 0742b8b7360a26bf46419ccb4ca1f432987d7fa7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:14:58 +0900
Subject: [PATCH 211/638] Update 2024-04-16-day-106.md

---
 _posts/2024-04-16-day-106.md | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-16-day-106.md b/_posts/2024-04-16-day-106.md
index cadf548b6..b953a6733 100644
--- a/_posts/2024-04-16-day-106.md
+++ b/_posts/2024-04-16-day-106.md
@@ -1,5 +1,30 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 106) Community structure in networks
+categories: [theory,gnn]
 ---
+
+## Hello :) Today is Day 106!
+A quick summary of today:
+* covered the next part of XCS224W: ML with Graphs - Community structure in networks
+
+Covered topics:
+
+Community detection, Granovetter's social science study, Modularity, the Louvain algorithm, detecting overlapping communities, Community Affiliation Graph Model (AGM), graph likelihood, BigCLAM model
+
+![image](https://github.com/user-attachments/assets/a9ec6be4-58eb-4522-98f6-c11a54e0c274)
+![image](https://github.com/user-attachments/assets/0e64c3a6-db64-41a6-bb75-b75ed1680b47)
+![image](https://github.com/user-attachments/assets/3a7d4de1-0ad0-47ca-b342-98b9698c6613)
+![image](https://github.com/user-attachments/assets/c30b056c-b809-4dd6-9b6c-f4bee0f29057)
+![image](https://github.com/user-attachments/assets/2ddd817c-ef13-4ac0-9660-a9fd17d525f3)
+![image](https://github.com/user-attachments/assets/84972f3c-1fca-47ba-bd70-1ad253c8eb7d)
+![image](https://github.com/user-attachments/assets/b60700f3-0bb2-417a-8d09-6d85081ef5da)
+![image](https://github.com/user-attachments/assets/2c7b4b9a-37fd-4abf-9a04-3fbaa8c1a26d)
+![image](https://github.com/user-attachments/assets/b7e1e488-b624-44ab-8f88-0712ea9deabe)
+
+Today's post is earlier than usual because I am a bit sick and cannot breath well. Nevertheless, I am glad I gathered the power to go over this lecture. ^^ 
+
+
+That is all for today!
+
+See you tomorrow :) 

From e7b8a558d8dc34780178a91e5edb254e2c84cd05 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:20:26 +0900
Subject: [PATCH 212/638] Update 2024-04-17-day-107.md

---
 _posts/2024-04-17-day-107.md | 61 ++++++++++++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-17-day-107.md b/_posts/2024-04-17-day-107.md
index cadf548b6..bb355060f 100644
--- a/_posts/2024-04-17-day-107.md
+++ b/_posts/2024-04-17-day-107.md
@@ -1,5 +1,62 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 107) Transforming natural language to charts
+categories: [applying-knowledge,nlp,project]
 ---
+
+## Hello :) Today is Day 107!
+A quick summary of today:
+* created text2chart - transforming natural language to charts [[github](https://github.com/divakaivan/text2chart/tree/main)] [[webapp](https://text2chart.streamlit.app/)]
+
+I am super sick, but in the around 3 hours of my fever not being high, I managed to study a bit.
+
+I was again thinking about creating some nlp project but was stuck on the idea of how to host a model. After looking around I found this article where they create a similar app called [chat2vis](https://blog.streamlit.io/chat2vis-ai-driven-visualisations-with-streamlit-and-natural-language/) and they use codellama. I know codellama is an open source model that I can get from huggingface and I wondered, how did they host it. Turns out - they ask the user to input a huggingface api key, which then allows for querying and generating content. (AWESOME! this is applicable to some of the other PDF chat apps that I have, but I will look to make an app of them later). 
+
+My fever eased a bit, and I sat on my chair to do some work ^^
+
+Here is a breakdown of the app.
+
+I used streamlit. Also, 2 default datasets during dev: Financial Statements of Major Companies(2009-2023) from Kaggle and Boston house price data from Kaggle.
+
+Usage
+
+1. Input your HuggingFace API key
+
+![image](https://github.com/user-attachments/assets/3a08132f-b7c1-4634-9697-332f521e1a69)
+
+2. Load your CSV data (optional)
+
+![image](https://github.com/user-attachments/assets/2beb41d1-ef4f-4fd8-8ba3-1184347785a9)
+
+3. Select data to visualize (default is the financial statements data)
+
+![image](https://github.com/user-attachments/assets/0a9c4855-ebaf-4ef8-a0b9-ac3766f1ed92)
+
+4. Enter a query, run and get your chart
+
+![image](https://github.com/user-attachments/assets/4252ed81-3411-4340-beb2-9aabc03885ab)
+
+I did some more experimenting as well (below are query+resulting chart):
+
+Display the revenue, net income, and EPS of each top company for the latest fiscal year in separate vertical bar charts for easy comparison.
+
+![image](https://github.com/user-attachments/assets/71b0e272-8d1f-401a-bee0-5bad7dd3125c)
+
+Compare the revenue, net income, and EPS of the top companies for the latest fiscal year using horizontal bar charts.
+
+![image](https://github.com/user-attachments/assets/334bc7f8-0bb2-43fe-b411-4853fded231c)
+
+Group by companies in the IT category and create a pie chart showing a breakdown of each company's market share.
+
+![image](https://github.com/user-attachments/assets/ff1e183a-eab2-466c-b351-c06a3af522ce)
+
+More examples in the README of the github repo.
+
+However there are still some limitations, that I can look into when I am better:
+* right now it can read columns as either categorial (if dtype is not int/float) or numerical (int/float), so it might have problems with dates
+* might not be able to make a more complicated chart (or may need a more specific query)
+
+
+That is all for today!
+
+See you tomorrow :)

From df727d700b6978c9bd5fc64d6cbf970336a7578d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:24:25 +0900
Subject: [PATCH 213/638] Update 2024-04-18-day-108.md

---
 _posts/2024-04-18-day-108.md | 51 ++++++++++++++++++++++++++++++++++--
 1 file changed, 49 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-18-day-108.md b/_posts/2024-04-18-day-108.md
index cadf548b6..4e95f775e 100644
--- a/_posts/2024-04-18-day-108.md
+++ b/_posts/2024-04-18-day-108.md
@@ -1,5 +1,52 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 108) Recommender systems + small adjustment to the text2chart webapp
+categories: [theory,applying-knowledge,gnn]
 ---
+
+## Hello :) Today is Day 108!
+A quick summary of today:
+* added a field to show generated code by codellama in the [text2chart webapp](http://text2chart.streamlit.app/)
+* covered recommender systems from XCS224W: ML with Graphs
+
+### Firstly, about the text2chart webapp
+
+When I create the prompt for codellama, I give it info about the type of columns in the dataframe, using the below logic
+
+![image](https://github.com/user-attachments/assets/1ba53f25-ddbe-4bb4-bcbc-6e4b67a4247d)
+
+It may not be the most robust, but it does fine for now. I thought a problem with the app is that the date column is read as numerical values, and tried to introduce the (now) commented if statement, tested it a bit and it did not seem to do anything.
+
+Then, I decided to show the user what kind of code is being used to generate the shown graph (probably a feature I should have had from the beginning). And this actually made me realise that sometimes the generated code is actually not that good, so a more detailed query might be needed. 
+
+Example:
+
+![image](https://github.com/user-attachments/assets/21de2b46-f645-44e4-959d-033cb87c956f)
+
+So if a resulting graph is not good, we can see what was wrong in the code, and give a more specific query. However the downside is, is that if this were a final product, it requires python knowledge from the user, which is not ideal.
+
+### Secondly, recommendation systems
+
+Covered topics:
+
+Modern recommender systems, top-k recommendations, recall@K, embedded-based models for recommender systems, binary loss, Bayesian Personalised Ranking loss, GNNs for RecSys, Neural Graph Collaboration Filtering, LightGCN, PinSAGE (the paper of which I read on Day 100)
+
+![image](https://github.com/user-attachments/assets/8dee0ba4-a346-4499-9374-22d2d9d0f0d9)
+![image](https://github.com/user-attachments/assets/1e2e64b5-dc0f-488f-a7bb-7ab49f4c5f24)
+![image](https://github.com/user-attachments/assets/e8a13acf-b62d-401d-b8e2-e55ced1a4304)
+![image](https://github.com/user-attachments/assets/8c31fa35-db6f-42b7-b5fc-7413895ff0c6)
+![image](https://github.com/user-attachments/assets/87309951-6e4e-42b0-b000-0029cc739123)
+![image](https://github.com/user-attachments/assets/bd80c612-e81a-4f69-a977-6d1248a8ccae)
+![image](https://github.com/user-attachments/assets/b19085c8-6143-47a7-b76e-9dab0607ab91)
+![image](https://github.com/user-attachments/assets/b92cc8a1-3233-414c-87e1-65c29b4dd640)
+![image](https://github.com/user-attachments/assets/189fbc4e-5774-4c76-8286-cca920101eda)
+![image](https://github.com/user-attachments/assets/89f1b448-a867-4612-9691-7f26dd5ccb2d)
+![image](https://github.com/user-attachments/assets/f341899f-3436-4a63-83a6-d1b794ab8b5d)
+![image](https://github.com/user-attachments/assets/e16e9662-9513-42b0-a8e4-c6740dd9755c)
+![image](https://github.com/user-attachments/assets/afae77e0-d8cd-4cc0-817f-f6ce93461d00)
+
+That is all for today!
+
+See you tomorrow :) 
+
+p.s. feeling a bit better today

From aa9973cd34e442aaa2fd72e7ffc99a3a4158280d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:30:17 +0900
Subject: [PATCH 214/638] Update 2024-04-19-day-109.md

---
 _posts/2024-04-19-day-109.md | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-19-day-109.md b/_posts/2024-04-19-day-109.md
index cadf548b6..6364f1a9b 100644
--- a/_posts/2024-04-19-day-109.md
+++ b/_posts/2024-04-19-day-109.md
@@ -1,5 +1,29 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 109) Graph Generative Models
+categories: [gnn,theory]
 ---
+
+## Hello :) Today is Day 109!
+A quick summary of today:
+* covered Graph Generative Models from XCS224W:ML with Graphs
+
+Covered topics:
+
+The graph generation task, generative models basics, GraphRNN, scaling GGMs (the tractability issue), evaluating generated graphs, applications of deep GMMs
+
+![image](https://github.com/user-attachments/assets/12970728-7b00-4c17-b45a-f690fbd2d386)
+![image](https://github.com/user-attachments/assets/90905686-ab39-4300-9b6b-b494faf8eacd)
+![image](https://github.com/user-attachments/assets/c104d039-26fd-491c-a10c-90e5a27a37b4)
+![image](https://github.com/user-attachments/assets/f7f3705f-40eb-4f1f-889c-19481f0c988e)
+![image](https://github.com/user-attachments/assets/a85438b3-4276-46b2-8d54-819a434e49f0)
+![image](https://github.com/user-attachments/assets/699acb31-6b54-4382-8a81-62b08ea131ba)
+![image](https://github.com/user-attachments/assets/2633532a-96dc-4c89-9b37-9ff10b815e6e)
+![image](https://github.com/user-attachments/assets/4ac7e779-5e02-4b1d-b5b3-194a696ae64a)
+![image](https://github.com/user-attachments/assets/92e8afde-7fad-41be-abc2-3578bef50017)
+![image](https://github.com/user-attachments/assets/ab28b491-e482-4057-b15a-664c1129c0eb)
+![image](https://github.com/user-attachments/assets/9a440320-5814-42d8-9e19-50e4e9c38e61)
+
+That is all for today!
+
+See you tomorrow :)

From 80d732927bc878534c10bdfcb7be0893a8e64744 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:35:01 +0900
Subject: [PATCH 215/638] Update 2024-04-20-day-110.md

---
 _posts/2024-04-20-day-110.md | 39 ++++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-20-day-110.md b/_posts/2024-04-20-day-110.md
index cadf548b6..7a8b87ef0 100644
--- a/_posts/2024-04-20-day-110.md
+++ b/_posts/2024-04-20-day-110.md
@@ -1,5 +1,40 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 110) Learning about Graph Transformers
+categories: [gnn,theory]
 ---
+
+## Hello :) Today is Day 110!
+A quick summary of today:
+* watched a hackaton held by the best CS high school in Bulgaria - TUES
+* learned about Graph Transformers ([slides](https://web.stanford.edu/class/cs224w/slides/14-graph-transformer.pdf), [blog](https://thegradient.pub/transformers-are-graph-neural-networks/))
+
+chnical School Electronic Systems (TUES) unfortunately does not have an english website. [This](https://www.youtube.com/@TUES/videos) is a link to their youtube channel where student video projects are uploaded.
+
+Every year, they hold a hackaton for ~3 days, and I saw a recording of this year's from last month. Understandably many projects involve AI, and one of the projects from a team of school graduates (my best friend participates in it. I saw him randomly, I did not know he participated :D) is below
+
+![image](https://github.com/user-attachments/assets/c262c74b-576b-4150-96e6-38d0288907a0)
+
+Essentially, it is a knowledge graph that takes laws, bills and policies and a user can browse through nodes (different laws), find connections and references to other nodes, and also on the right ask questions, and when a result is given, the user is also pointed to the nodes that refer to the GPT response. 
+
+There were many projects in the finals (it was a ~10hr stream) but I think the topic was to make projects related to the law (like my friend) or to help the environment because there was one that helps to detect when a body of water is clean, or with petrol/other chemicals.
+
+### As for Graph Transformers
+
+I checked my XCS224W: ML with Graphs course and the topic of Graph Transformers was not included, so I went to check the most recent official CS224W: ML with Graphs course and there were slides to it, so I decided to go over them and learn about GTs.
+
+Covered topics:
+
+Transformers vs GNNs, self-attention vs message passing, designing graph transformers (nodes as tokens, positional encoding, edge features in self-attention), Laplacian positional encodings, eigenvector sign ambiguity, SignNet
+
+![image](https://github.com/user-attachments/assets/3c592f25-8412-420e-9ebc-a46f54ec6969)
+![image](https://github.com/user-attachments/assets/0360edfb-85be-4d80-926c-ecb8c78526e9)
+![image](https://github.com/user-attachments/assets/a5cea408-a307-4e4c-bf00-ca810ab0c1b4)
+![image](https://github.com/user-attachments/assets/b6370f6f-0819-4f69-be53-a63dcb35607a)
+![image](https://github.com/user-attachments/assets/f7f9aeca-8d4b-4806-a840-f29ec9c4f42f)
+![image](https://blogger.googleusercontent.com/img/a/AVvXsEjZdomXhigdDmCDoIYivQA2XsoxOj8jMDkNA-j1MZtZcF-sf-XiappEjAMsY_uF7kau9t_VFSrsXWJTyF3Now80G7vfKVBgDTDxsv5P9rkrd4PKmi-sbvHBSybLrORmvTtVFSd7LRHUYbWJ7rcegMXnwbDVHOU9Fz0ayUV1rshke6Z8ERDOY59DmnOz5FWZ)
+![image](https://blogger.googleusercontent.com/img/a/AVvXsEiy1QyXJO026_3ZC1zmuGo5b4oZkmja5WUeMh0JMzY-0sCjvC47RZYXgLr-VPrKwDYcz7eUFz49g9EwjTb2nAnNYrRhe7rZ2dkseM8sWkJ4tMPIgyY9_qpjgAincqWwtl6CGk7m9W4LyroTka96BlQX8NZ16hM0fgdh7BRcljejYGXOoln7hGOONKsr4lgM)
+
+That is all for today!
+
+See you tomorrow :)

From 8d55881c8e57190f636d7a7324496f692bb02b0b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:43:40 +0900
Subject: [PATCH 216/638] Update 2024-04-21-day-111.md

---
 _posts/2024-04-21-day-111.md | 45 ++++++++++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-21-day-111.md b/_posts/2024-04-21-day-111.md
index cadf548b6..9036d95c5 100644
--- a/_posts/2024-04-21-day-111.md
+++ b/_posts/2024-04-21-day-111.md
@@ -1,5 +1,46 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 111) Advanced Topics in GNNs
+categories: [gnn,theory]
 ---
+
+## Hello :) Today is Day 111!
+A quick summary of today:
+* Limitations of GNNs
+* Scaling GNNs to Large Graphs
+
+Continued with XCS224W: ML with Graphs and covered the penultimate module.
+
+### Limitations of GNNs
+
+Covered topics:
+
+Position-aware GNNs, Identity-aware GNNs, robustness of GNNs
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjvBRwlQ4Ju1mQgAjibS2GL8I7a_vgVXhS58Cb0L7bXSUPeIMvxrZUPh2Ww-jt4y1JwZx1Q11cqMfnDnjIA1jLCY0dlbGcEiOhCq4YYlXdxbQW4GQ4FEDirO_CgTVlI9k_0elMiGpsD2_LQecNQBGfaHqz1JveXrSbmBqJuJ1KJknnrh9SE--3amrwIUsYk)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh1cMF-2egnKnt20xQKevkB-NZdhCNjE4wXlBQYAYyB4K00PoPh6qJUvcB5YsJlnAFpKgRqqjc6tYxGyjni7v-5HSVvo8MNDMBPRoLQLNV15scSDpiP9fAmg763oydfIjRIJDiS91dhd3EY72O2Pdmjasz4P5r63UlpWKsarsAvL5yAJHCedKspzshM8xoX)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhFadOTS3zHXlVz_ahhf5TxShx1eM13ufSrr7klcaKaeGyb-H30TY9kpnGSBkobX-mfvi_bDqoG4jyBVhjwLjaLHpijDt2urB6_ntbhliInVzBJzFqhq0y1LTuP363mvaAohZRkeHQqQDHiblIWK55A77il8JPG1zd39E1gYdBb8qoJ6kMB92aghd0VLi6H)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgHGYpDf3pI2nk-blgDcMMrKonnKWV-0GZMa_tXu1bSUF9P3ypRzN6XHpkbILoJgLUo-SjOTiSQOIJ2CY3k_Ko0sQuchqgXI1qh43kmjwlRVGKYArc1YjRQcPm72rWU18fc3-wdC2NXtAY47oiIoXu0WJ3msPZOWJX__ZkyIlIhySgUxcfNvAWtWY6-HNqS)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhBwNGKzWt65cqyWNIbugYXwcL-fyE9OSIoQjDNSUd-mEFrv5RV1lOU7GQULPZuj7cVstIasDvORoxL2Ty8JlLBiYEUSSgAms9yu_CP5Xh7yTIS93yklU8hOyk_T7K5OLybEmwBJ0HBDdRGiRxV4d_8adVjyZhLtpAMF3Zo8fMXVd0s1Aj49b3QRgICcoUr)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjDT63kcvZNDN6FRsx1YWBf8sOkvqBBecmsdR-ChRpg_8HeON7VDev5veUdb1I4yOJ-t-QgxtTT6OdicekUgQRAmRu0ij7wKb9ApcwkbB50m7jCHdaBW24cGcd0BrfYPl4ldjOj6wXrm395oxbThxvxwNcGw8W_ft-2yONtUy8VCHXuiWMX7E0J1HcrB2of)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjEoDsNccneNiBlpsw8x0z-vq2maowyQnkxB6DWc_JiSQeNnMc31jN__ieuqe4TiYwjeVFJ6FF7jxKtUBg_8dua-5-J2r8moaS2cIuxRwG4uqbetqTzNLP8MrmGMZmJIGVWT1TYwgQ1gq1d6ySx0NB8-chvHV49WukCbSGpenygHX-pZhkzmb_RST7LPeck)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhyT0HBeDPz_N5Y1vEw6hJN7oXADOGGTg5s6NzxtUUKdoIZuTenAHWmtLT4xjN2AyLv5mpjjuWkSZ7oP7ED3SpkBe3oVdAoGCFp6WlayzL0-NG_Z-Mthoc0-OdSqM8RTfT3xcQeYXUE7ttqthrGIFByC1091r0c5RG9ITUuMwdCAeWGhRnka3jw4eG2ir3T)
+
+
+### Scaling GNNs to Large Graphs
+
+Covered topics: 
+
+Why scaling is non-trivial, GraphSAGE and neighbour sampling, issues with neighbour sampling, subgraph sampling, ClusterGCN
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg7LrLe9JebhFRxf0WdqW38UMrxxvFkhyif6CXXk3VO2UldZ3oszA_lgeetHS7oJfEecgqiPUCqnY2niiYgr-Hx6iLD-bejtPL6x3w04k_IeB_9graAAneI68LhV2IGaUSiAbV_yw_heuVdK-c3MnT7ZICpOJx6q0EqshJB5RV-zDzjVG9XlxWaLj9bIIkq)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhRUx6omIbhwpmG8wUtL8zHqtocyHufXdJkiAEv58owPsQ6YXF_LtGcCcuFT6nEo6xX765c2bLo-eGmkMc5T1ekOityIZu78j085oWuWTUaPd5hd0u96chKChsAOJNRyoyDeR87frfm-aZjr_bxO8ZXAx3K2ewA8z-BrSdlVzFE1VXPdF7s5Y6ZzEynS9Xa)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgEsDeh_0g0siuRArJSVtwzUfEL8mYYoPvD0sgqnCqx0tRxHLuiZX4R3z4U4PhiAKSgwoErdMjU2BAzjyPlrg0rX0cadcUCc7xcyUmZzwRX68JwAWJuq5pCgcO5dsqVW7ufWweTDt901lcmgiTc6Xe74qcmbVfbkWFpgPHi-MaveF7Osb2_xtDkclCpXlP5)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjSqgOWQj_ZCewl_O2xIQsE9lUqxhtYGtAVszabxoYrmNBeIDI9rGhhUUDvEU3FMrKALGZWlOU8un9ny_gF-93r4ler-LhchgjvBu3tN368oK7X3VKoxnXT7AE2tWNrmsqpMf4h0skPrL70_dwuESTwSbRqLOYKZVW5ghVRI5eOtqCFhsmUdgIsOG-n9tXQ)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhB3fn7imkrAlAKyhqhKg6e2gEUdOWP2l9P_KSPRK51B--KVxjd60aWmJm8CAgH0xDL_hkXYQf66gWs2Atlk7UiKXxega2Il5-EMPyhk7tOBq98nFUC00hkmIzWKvEuCXELHpZfnl6f4xWxQQ17J4GNYWFEXUyrqnCgacR0rf4u-kyZxY2yrPIm84ujlkhf)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgOle4fA8TScZbudosxygx0wQ3sTd2D3p-kfAklx6rpmU1_zk5s1OIs8hgPemxLC9Iyr58-XveZDZZuhkHOJL367L1Rsqzax1ARzfol5oBWbU209OI6EznSV4oNjYyXQUFelks_GV5LcOIy3h_vqNDrqLV4GpEQejotDoH3JY7bo6FNKjXJhBcRVFfbLWcK)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEheDTl_NSnKYE3NNdhvMKIyZrMy41SS4P1GmNzxGOmL1MGMWRAayCsPupNToswa6zSzjCWh5wyC2mBrMibIFIpmBuBitvL2YUb_ZrK7PuF563wE8C6Fo87jVEiz7QkxsReRZO3hp4Om9VbJMlxQGVFg3lqoyxBIB-7S53oIw3FroKA64MrlpGb3W4Wwd9qe)
+
+That is all for today!
+
+See you tomorrow :)

From efb8a4a26f71a4be40d511b3e02958d0362713a3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:48:36 +0900
Subject: [PATCH 217/638] Update 2024-04-22-day-112.md

---
 _posts/2024-04-22-day-112.md | 59 ++++++++++++++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-22-day-112.md b/_posts/2024-04-22-day-112.md
index cadf548b6..9615649dc 100644
--- a/_posts/2024-04-22-day-112.md
+++ b/_posts/2024-04-22-day-112.md
@@ -1,5 +1,60 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 112) db2chat - Talk with your (sqlite3) database
+categories: [nlp,applying-knowledge,project]
 ---
+
+## Hello :) Today is Day 112!
+A quick summary of today:
+* made a webapp to talk with a database ([webapp](https://db2chat.streamlit.app/)) ([github](https://github.com/divakaivan/db2chat))
+
+
+This morning I saw [this](https://python.langchain.com/docs/use_cases/sql/quickstart/) langchain post on creating a chat agent with a database, and it hit me - I need to make one myself!
+
+The short but concise tutorial uses GPT, but I am just a poor student that prefers to use open source, and from a random X post I learned about Groq. It is crazy that we can use Groq models off-the-shelf (like mistral-8x7b, gemma-7b-it, and even llama3-8b).
+
+### A quick summary of the code
+
+1st: create a query chain for the sql code
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjY4J7_nXcN_kEQ9QoHtx_rIQr967NxwuFGOHXexoYJYMLxPp13V3kvFaTvFhwcGRAcXuGza75vKcle2huv4yLUeDJcm2gqnFRUWbXrvrgEaKInw176mKNuOk3-EZKDrS1du_k1JCXGSVXFe39UZdmGv83xWEecP4CL9RN_qNVrgPV0wDfJTUluat-DNg8t)
+
+2nd: query the model for a natural response
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhPw8yLoKY_8vroP8M75OTGvyvHSLsc3xUmCyUtxqC1INqzijZTAt9hdqtXsWTQmL-qsEbMVVuydodo_U491Blp1P2Ge4Gu_8MkEkS1o52YQG4V3pg32vFSLUrR7-3CdIVDqjhBzHeknoeBwuiFLlcvmE9OGfGf33dl30jcr_S5bHow2P3H8rXeNM0LRExC)
+
+3rd: I used streamlit because it is so easy to develop, and deploy a webapp with it
+
+### Webapp summary
+
+1. Input your Groq/OpenAI API key and select a model (mistral-8x7b, gpt-4, llama3-8b, gemma-7b-it)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhvq7LPMVhlS0hUR1vfTrWpSd61aVwQVciqaCvGYWN0ICXrg_kc1UlcctAIKlN7riPfWonrlfG60VIHqSChAN_n2s76gpW7v86SE0ZkD4cfzhJ4KjjbxWdfqHQ3H17caro3POeb6dmQ9_LarNvDGFcQzKdZRb-5sF5wB0jcEKQvr1KeoVcZDj15xNe-uvqa)
+
+2. Load your sqlite3 db file (optional)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhvsfd0aFw3ARbgz6_QmBzsTzmerQ8EbV3En6cALwPvmo_NJgS-DxDapgQBimi_6qRj-Ni5GYWEqrOwB7TRavrW5tp--aGTRa36cYx2HwXv0B6KzzZYIa0uDXqY-v_WDTqEbMy4qn1E0PGyomJKjTAHw_Xj6OmV4-VFTSP-6NNunJ1obsh8l9TTHLARQ3uV)
+
+3. Select a database to chat with (default is chinook) + click 'Connect'
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEirAiLV74pz96plgO9pcpyqLjk-dPpgb1vLjCVT14da4iR6l-Ex1Xt5NAVXYEYByIltyXsqSvuHwt0-MiqzC8YXePXPO-Xh-PU868SifwuYh1bMkVzJqW4rRxVqdl3_JwkUj_c_IxLZFODfrjiKIztn26pH13nUudxqG_-UPnvq4DkYJ2aVlwdpNXpS4VLW)
+
+You can view your db's schema:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgtLCzBl17t08Cp8J85zAckmQsHZT4B0eNVH_ipkPtvaDjdaKj3zYvGjIr-aCQYgM7fZW8a8SGoRAKOeXDOqTpwe6M5PQSNI2fhil47FLu3kVKGHXtklrFCLASgTvCTfaJcRWzFjO7153f41P3qFDeh-Za00n8JnGazXI1w-pA1jhncwC-ZsH_EnccgDPmI)
+
+4. Enter a query, run and get your chart
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiuDq4Ul5EsfPYsMurB2B75CzPeWV-eT5XDori6bMP_cmK5wplYSH9bwl-LuATrHz9djMLGJku00lPRC0W9E7SjV0kxJcLahVoY5U5uSz63HXFj_neRc-Od_S9xwl6e7nnNdDhNT6piB-E7tMsl3xPDdG2mr-W_1NiW_EZWacvSYXxfYKPC28auKlx0j4sF)
+
+You can see the executed SQL query by hovering the (?)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg2GOr_HcxKJry-nCJFmM0mNn2pdXY02fTCQ9jv62Ncx34XkYCTl7wcwCtbxxgxrW7Ng1RBJiNeQAtJqK09E4oeManZA1EVoQbDxBzL8KTW8OhbY-1S75sOOpCV1zdnzD2d2XpQG24DuexVIPsPyV2kYwhSZ_a_e-ZnP-SMkTjBtdnP-bUq4g-Ghh88RBvA)
+
+A potential upgrade would be to give a natural language input that asks for specific data (i.e. albums sold in 2015) and the model returns that and shows it to the user + allows to download it
+
+
+
+That is all for today!
+
+See you tomorrow :)

From d01061d4884bddc4882e233fed2266bd0b25635b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:52:46 +0900
Subject: [PATCH 218/638] Update 2024-04-23-day-113.md

---
 _posts/2024-04-23-day-113.md | 46 ++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-23-day-113.md b/_posts/2024-04-23-day-113.md
index cadf548b6..c09ac4a2d 100644
--- a/_posts/2024-04-23-day-113.md
+++ b/_posts/2024-04-23-day-113.md
@@ -1,5 +1,47 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 113) Making a better blog + Geometric Graph Learning for Biology
+categories: [gnn,theory]
 ---
+
+## Hello :) Today is Day 113!
+A quick summary of today:
+* began making a blog webpage using [github pages](https://ivanstudyblog.github.io/)
+* learning about Geometric Graph Learning from a special lecture in XCS224W: ML with Graphs
+
+New blog link (not finished as of posting): https://ivanstudyblog.github.io/
+
+When I start looking for jobs, I need a better personal website, where I can have more control over the design. Some time ago, I found a template from jekyll that works, but I did not do much work on it, it was kind of standing there in my github profile. Today I played around, and found out how to customize it for my needs. 
+
+This is the home page (with blog posts)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhUHBwViudOyiXbv3mNsVXYaxVcZNGwVle9SGulUY34CztOUptr56bPsON-19OgjypFVu6WtBMcwmiT3sdSUYiHglsaIhbQagapBd48ZUo7hiLpHuhdWrCA-JqsNP33QIX1lxmU6VIQng2dKRmz5rPStajSFJAsKoTzgt7RlDg7DUIHAui90u3yx7xIN47p)
+
+A 'Projects' page where I can list and better show my projects
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhVrFmhmimTcUktr3k1b1c005plBiScXaPveyYr3xBLzl6PJIfMa-aVLJ1EMuR17z55o5FBGUUo9mKOeTmm0iCPbkBcO6U_tqoD4kQDYw41IGIN_HGQ4T8-CrxmqH2033fMw01iBdVOsKL3QwSXPQE2MR8PLG1dg5_r_jIAzxsF1crFqWxV70Ka2r41x84K)
+
+There is an off-the-shelf search functionality
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjDzaNVfOfPYMEdycLEgbxc6GvBYh1NSgg4U-9ZkknaR8EKS7eY95KcStiPWcAFPrjirtmQswq9vCtL2GzZ9itN5s9rNmI-nrSyv2FkXDBYoCp4zh70xbDcoIn0jtjiCnvdNld4Azg4Prs6cLEd-c_QcQw7ALaS6iXXUyBEsWYNYsu3vIcjkNy5EJonz6co)
+
+In the about section, I plan to give a rough overview of what I have studied so far
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjX6fQQh_xPfWOeQTy5lv0f862hUcy98ogCSjiTWUi1LePFa_ZtAuq6YGj1nKp-aDALQ2yL2b_jzMEuFJHVqRUo5aUipb7BLRpkLTS_cMmTdApPQJVDA13xS3BrK8nEwFqgnxkNDJQET7RgyVtUBS1Xt3LrggBzkl31e-Le2HiJfrPZRxA3BxgzgBdWGpUM)
+
+I am pretty excited. Although, I might need to manually create a markdown (blog post) file for each blog post, and copy-paste content. 
+
+### As for the Geometric Graph Learning for Biology
+
+The lecture was pretty focused on molecular, protein and molecular-protein link generation and issues related with that, which is vastly different from anything I have done (or plan on doing), but it was a good thing to listen to.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh3IvDaP_n_DroAYeImvSgZvZSJmLDayrd-QgGspq6MoFvFV0KjFHUZ3SSnvgyBiks29tw266QkmNmTAq4QPaR5XLqSuhPFmLSTa3ydu4gVNaNKfjQzEYHT-rigbt8N_uap9myvifTd5GWY9Y2HXcU0F0CCMjrJ2QAB_iTY7EsqsC1kT0NQvjpTWwLMbqej)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhhibTrafbcMSylnKZ2e32Tcx9TmOk8rS_MHdF7rUYYMjXxKJTKgrT7RQXBYzM4xujh-UeWMAS40h2D5T_ma5BY_3P_qDmnY3qEsje4paYbPi5IiBsCDc1q3FqWCeLhpWYkK7frLSAWW7Afp98G2uq1PlltWlbz8R4mIXgbXpOU05HV72Bai2Z9GT1cHgqD)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhxi0LTd8QLTf4WS6y_fTfdywiTRZJd7gVm-EOvwP6LyS5uJ4QneK6JtR2k1zrt0juAP0jNnYGj_yOq_9jVZvLYLt8IEOnQfEF5CIsMJsJv0sVgqLfW2QyCUNienNJfDrMGO8ernHymg6SQbxJNj6IhAd60m0FiamnF4v2Hep_96v4ovZUUm7zKeQeVgb3d)
+
+Before I wrap up, I tried to do some testing on the db2chat webapp that I developed yesterday, but my groq api key has reached some kind of limit, and I will just let it rest for 24+ hours. 
+
+
+That is all for today!
+
+See you tomorrow :) 

From 3943dfcc130a6b0ad67419dd1e353d4a8e08df91 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:56:12 +0900
Subject: [PATCH 219/638] Update 2024-04-24-day-114.md

---
 _posts/2024-04-24-day-114.md | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-24-day-114.md b/_posts/2024-04-24-day-114.md
index cadf548b6..1bc4c9e27 100644
--- a/_posts/2024-04-24-day-114.md
+++ b/_posts/2024-04-24-day-114.md
@@ -1,5 +1,27 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 114) Trustworthy Graph AI
+categories: [gnn,theory]
 ---
+
+## Hello :) Today is Day 114!
+A quick summary of today:
+* learned about Graph AI explainability from one of the lectures in XCS224W: ML with Graphs
+
+Covered topics:
+
+Explainability, goal of GNN explainability, DL explainability methods, why do we need explainability, GNNExplainer, explainability evaluation
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiWZu74bfWxy6Z-nih2_BBh0gG5xpYwyULvxZHeIQzfxIQ7orZ-h3gyj0B8s-_bq5PqoOaLXoBOM4RqppuccbJCjAAx7-IsqJe7HLzAB4Mm2C5fD3rsd5VCTzUPF3mImoAZXw9fp7YjPRw_BfxltREJsAFF0rHP0tZTWUd8A6-JDs3ZrSsYu3HM8oU5ONvp)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjKfT0dE42Ov6lb7wpssPwuXFHvZtnkiUWproNZydZqisffzcFX5rC4A9OHxsK_fNp2XtpmR1KdgUrEzo7-jUQd0hQ8yu4cQx2VDoCjDgqOoEKy9LG2oSi5Dbslu6yPhPUgIHUre51OIRhMYBK0ctAQaLsGGmEEX8wfWS9QKjYxX6LW7ljnZqJ8BvrX3ZFP)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjZ3SeO27hxR25qshsXuVTlRGGGyBdwTWqsj3wlAyIRvZc47s1875WvJkecRXGwCsMsrWR5lG_rgUaH0Axu2bzxRv6lSeNMKbSK-vbm4bxYuvd2MIZ32qAObAyKRklhsta-_ShkpPzmCJ1IgeR8w3Vtule-r7VYD8Pmp5KslnabvsvgJgeruTjWtcrDTojj)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjVHl4aJ_Z86cZiCqD1cbDc6mJ-wyshPg-4WRdYfd-1edKh6qXBS2BjS9XC_1cBJN4UQGWnNdAq8g-fpyyoMBnsBGu9ufNcfD7N-h0n2Tlk-n73PX3JcKbgm48BLULYcXxGqerdfgYZ_n7sVVrzege32yXXeL0bw9YK8w3w_LdJ9hCsEM3nWhEXrzQi7rUE)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjuX5wFXytaDED6THmfsm4nz6wGLOfzcGPb8kgENfn4AUNoGeAZ6YmD2x3waFsHlco8PzDOGFygRNacTPu2PYGqkMGevRQ5d7kQ_Oo2zWD5lvsSltQNzycHzjdITfatKqklkOpfquGp7D-iKIwks_6SnjGcbsob3XIpN5MM6GouQl1N2pMWi65CFBDXOVJV)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg-NvyWhV-kQx2xY9lQ3U4nsnJXxdMgyQ9OzBSpPek9ULWqxKHMoDuVisPb5CxB0Luwdv1qoyWBH_bE51n_76lXrpob2oSZTXGD-sb71mXyBGW4MSYU8gs-wO_wtm1a4Tit9NpZm956NiVmBRGS-zxAA0-A_3GWrXti_OXWzHIOzS8AN0EKWt7G6BuI-jLC)
+
+Today is the last day of my uni midterm assignments, so from tomorrow I can dedicate more time again ^^
+
+
+That is all for today!
+
+See you tomorrow :) 

From 2dc1da92619b2d6b1983c53a924e07992b058e4a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:03:43 +0900
Subject: [PATCH 220/638] Update 2024-04-25-day-115.md

---
 _posts/2024-04-25-day-115.md | 123 ++++++++++++++++++++++++++++++++++-
 1 file changed, 121 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-25-day-115.md b/_posts/2024-04-25-day-115.md
index cadf548b6..a3d1216d8 100644
--- a/_posts/2024-04-25-day-115.md
+++ b/_posts/2024-04-25-day-115.md
@@ -1,5 +1,124 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 115) Exploring HuggingFace's capabilities and submitting 3rd homework from the ML with Graphs course
+categories: [gnn,applying-knowledge,nlp]
 ---
+
+## Hello :) Today is Day 115!
+A quick summary of today:
+* submitted 3rd homework of XCS224W:ML with Graphs
+* explored different huggingface capabilities with [DeepLearning.AI](https://www.deeplearning.ai/short-courses/open-source-models-hugging-face/)
+
+
+As for the homework, we are not allowed to share anything from it. But I can happily share
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhE9M4fc_R3825bvKN5XTtSIYvAaC6fm4D0RaMPueC18hTpIbVXeTOvkpaWd9LbFdoZ_8gUjxRkkhtdMO6DXVDwy8jJO735lYvrjNBl8i8Fw4FN0KB_3MzuzsuPCoqRtUmvxHbOrDMlaeTvTCfbRO9R0xDvpOSxz5FYDKe8ZI3ynGrnCa4bYx4fvUZdr3LA)
+
+I got full marks ^^
+
+As for the huggingface tutorial
+
+It showcased the different type of models that were available. Below is a summary. 
+
+**Building a chat pipeline**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh97_ytHjDqMXBl9fb6MlSNVoGREpsJ28IVS81_mJAkLdc2iPt-nhKtX_Tn9ZKWUPq0-I1Maz1IHVfo_mulLU9JwDQvhKGiLbw-U6cr5-MdaAmAm-ZvFzs5Z7TMSx9G5sBILjkAms1axNqTPmVMmni3j5VhuHpk-8KpAtLuBFiN3JXkNiIWLYd_GVQB8Fln)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjg5mVtJmv-icGcjQv71Mh5_ZpT76NYrq71BbOvbFTytw-9OJDO1Kwddnk5r-N9p6mkED7Ukp-OAOegnSG32ekEna4CiWnd_Zh2TYurFMl2472ZC_hisLgE1UTMZGkYKBhgb-busN0pw0YSdhg8-MPcWGklygmasUd6SJNp2-1GpY1T8rZHQBS9DSsYtW-g)
+
+**Text translation**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg3On7KwLvgyHO4LKfD8rdi8pPALlHh2s9x-5rf_J_KFffgFdTx0c_tYRdwalqdK2buS-DUPY9GrbOV-FbprvzEQNe4RkZ66kgz6Q3Vq7ECfmUMw96QSG_GfgArVPaLroMOHt1u7t8ud-7mn-DzMOLlUSrIpMURLIQRZDpfJWT3ie0OnJtVYJXKln6Vci1E)
+
+**Text summarisation**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgXkMKCk-fb70BQB1xejafX9V_jv661LKa44irFs84BTo4tQgZPARM28dgj3lQLzE5NcRI8N34AHJFWuyU10xfNuw7EmId0sOj1JhRi7yTryhk1nMeXwnpCfqG2Ye2hLbvWZreY32oBJGgtLDRaWioJbGEbWcZxd0CRdtD03v_0Mquc8-lVYSxw-UezusdX)
+
+**Zero-shot audio classifier**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhjrhgwuMHi25WBia7E0RYKKqcHblA-w6zjRuB7-fWfVTrIqLJtgK_3Y1_WJdVNjF8xMANHJhMcP5eFfTElCKge2CYZHm6-t9y3TMBkuFztzuvfmDFK-YCwM7zd-fNHuWyYQg_2Fh4Zn2P2ZFDd-ILWcE8Vj0k5nr0pGTjDKdBmFR0kmwz9qA_D3yERgh0X)
+
+Apparently, the model seens the audio differently -  1 second of high resolution audio appears to the model as if it is 12 seconds of audio.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgT40BNI74q6Yl8FLQx61TyQlscdUWV0A4BhuwItuE8mPezW0oxZ6r_9GUls3cfBwRDfM0hrC-2cfx7dXylcN623dx45dKyw4MHuRr8vxnVovZ7V8glKdluz4be6SPwFm5JjYrUiF8mD7rwo4ucs5yUCwk2maqLVNYTYaTyzkngb5JEpkbkGOKVFwtuCFmI)
+
+**Text to speech**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiaf41GH8XAs3FSmi3YSArbeiXruOd4n9k_O1dSiTLRz9wPyMGFvvHZqtgp6V0Mkar7wq_2B7aQ8HVVYPgUdPBn5_A9achbF3n4cYJNP6UiTNromurX4JB3ZUzPZ12ZVd081hvqSPLRYXwmFWLfbLMD9MvSWTGc1sqCTRkMR5t33LzT4oIekj6q2iH5KGgC)
+
+**Object detection**
+
+(code before the pic: od_pipe = pipeline("object-detection", "./models/facebook/detr-resnet-50"))
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhFWcSfU2JwgXuUKWfM1G_vB_gMm01_aPfMFDMtlaLeFRMS5gev0j7KFVhOI2IwoA8C-eh8kKko8GPCmuCM__B2fWbmukjRF2IyPhDJaz9_UZHN5Ev21EBfaBHhY_Tuml9jC3rZW_Ebu8pfe4fN_9QXv6qiF89pRVLZdzC8FDix6AdVmyqmdt_6DwdCQsSf)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg_YmMNUwcNnWt5KG0LbSu6tvaAT2EEEg_rYxTKjBaWn7B_akQDRnCkXJBeli1nufuTjllorS0i5ATw_FI8BRjS_uBjoSNYlSyP6tnnXevk8hAN2QQU1JpTgwAceOuINHsGg48M-GrJTM6_4F4LQ-UIh7JUSQK9hEuCdSnLDewr5Hllakorit8JPqDfbSTd)
+
+We can use gradio as a sample interface
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhiyTQxIoZjDse5q5QR1cb27W16n6oltgUOIKLsQRvMFN7xxr5zFxfYfz-gMsiNeCKWRHbvi1hPOlB9AvBDyy2z0FOK4kZ7HaAtMtcGVidPE1Fkk4bVWJ9Yt1osCNmB32-9iUfJfdlj9qS_BS0pw0_BbGbdTqv9fEJTglaM7dHAyvxHso1cl-LOo53I5ImC)
+
+I passed a picture of mine to check haha
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEihVyHEfUh4hH8woAdg_m6m44el9oXjBNjdLm8g_ASSKajcpUeaQNsIKCaT3p0BurOsJ_ZykbL7tfD7-2XUJv5vJfxVHO2bPs7bDUn9syV2XEdYqcAhKmo2jQ_uV5kOK31ld1RFQH712hRkS-kR22aJYOv7asgdTIMy8l-LaZxQ6qwiVGIGkEEhdE3m9GIp)
+
+We can also get natural language descriptions
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiWO2QCocaHIY1pvywHPB-1cN99kZnAksDGyYDqFSGnr5-IwKYQaSPeiHs7iXvDR1YEmpwhqsppm55sUvMNvCLxrQTytJfU74nBDwP4AQCA5mIrsD8ceCVQpXLmb_Xwxp7p3V_kTYsXyZOQ-2GibBmX1MZ_dE9uYDNzw_0zeictZNJOZYb0XU_kjcadCLNx)
+
+**Image captioning**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhh9wt5QNr3426KXRCVQmr-BrUqhN8xkcsbLSjCQVn7i0jlzDqaT40S6zDM0LlwgagbWV-cC6rZfcHzDylp-RQZ5z_tXfYdzmi8hF4bV-DksvV4VLefCMjcBXicaUUgFdPYB4jMJAF400OnciITfKEPd73sJ8-W1CjE9XlwaU3iyFRbNvGsTXfH9cUd3_HX)
+
+Example image
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiAj8b-nAFN-wddmNBrMXJZm3kF2HDvEO5ACtbdx0295mS5bWflhiEIQflYk-NV-eaJ3_knXtTUPxP2sLAovx3NquugP14PFpEFp9W-2mzUQckmzw8wHuwfqAcMhHAHjg2uP9iPQD5ZQh9UdnALSUG82KfI0WrDy6se4yqSCChGTm6F7dUwun8nv64ZGTJH)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhY3c2nuHP4P9fH1xEMtuoHAS-BkcvtYNcOZ1ljgMcNPOjz1nWX_w8wbOPMIZ4v4F_YTKOZUE0ontTHdR2puldOhmgpycCwLkaI-bQfCGX2LSwakr2ZgHzXcET-mvqRDwKjEPLItv13Nh0xmaqRBG2AFhSZ2gfQ9ywarg-cn3Zo6-nq791E0MRvYF5uhLNM)
+
+**Using the dog and woman pic again for multimodal QA**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjFuVOIUDgOyOQihNA9A3PUmtRnkQfS8eIZXfT6EFctJo_Co8endo-Hp2Djlo_6vXo5cH2KlpqPCA1ROmbSv9RyOLcv1uW2weyovyYmyn9Im3Wa3eG1XUgtpoyWpU8ZFxShIIIajv8PpmCd1oUO_fCV83448I7SQ31dnVcvXGZ6sGUoB3WPXkFGQVh9JncN)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgdUO4Kw-au4-if9oQg2aDdib9wRIcD4u4HINyg1ToGAVhcLAAaQLAypG2kkvsp9ptTFXNAExgV8N6L7Ido5nc8d10bpLev0rhv0CopfrghbtQpX8i8uRWUcokjW6jNUsdXMhlR5FtQTSSZTDUHT9S9VCFRRh6akiOlbGGzYPOZQZN0_aYYg9zgkg3Dv-Lr)
+
+**Zero-shot image classification**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgPYeMDEmeKVPiDr0BIOCfDOvPcyoYsV8i4EAjHqzTAjTXPie733C1Oj4ynRy9GNUH4oPJs2otf9IbAAO6VKTmPrII25f8FU6PyjSUdWp5nc4j1Jwch6u7n5sTeoBOWd-_FN5OBXyvPZOPevcoD0YFyLp30KHw7nC5yYz08q7TVzwBWX_0Vr7hg-SQx0COt)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjbLVmOA_bp2LYt1UBRfGvA94zhwMa91y9R0KA-Gy5xWWKoD-uXBiidg4G6zXG4XP7LBGa4F_sf-E02k-_QiZYVeTJFtDX7AYQStKcH5iSOgCwB81__GRS7VOfS3Jj_ZfBORkWiIRxOt4VGL531C-auxP6kOVbu9-VWIbjx7bGVUdwYSeyU6EypjrdaZxom)
+
+Giving labels: a photo of a cat, and a photo of a dog, we can get the models' probs for each label.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhiWtEW5Q57Avylo0IwJ2ivYFH9CmiJBT32N6YKN7IOTZEpQh8zWKGHL6zDHwXBhCmJUDwN_h_gTeMcEbSt7ocfJGbkKkGGKHa3SMiuPg17V8vY-gPpr2f90eoioxGAAIN-1y3jXqa9Im377RzjMcCW0HYvLI27nN9McI-oycW72TTpl3u00kA0MKa776IL)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjokzDB5h2wz7Yx0eV33ZcsZGj09usdaCYBjPleu-ookdmaQlD20qfHLYzxnjM0FXfCCbs12Sfa-SSqLtw9ugTtCXqGbG5zDqfvTvJy1AvsKmx9qELqfJgUhpHEq1OG9BA6keRIUj7yq8V5nwF78jEZLX4Pk68CnpDA0jC5kFCAlJWVC8lvoUht_h64HCxq)
+
+**Deploying a model**
+
+On hugging face to deploy a model, we need to:
+
+1. Create space
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjrXD7-RJuW5XCUN-0hdUSEDaG-wjggF-vKU5S9YhQnPKxu587Pu_8Rc4k5ONOq6wUuZqX8UB7i8XE8m2FkTKN7_Dar2V20iPBNwO1qvqTo4Z0cKluv5Ez9SRhquFU5swI4LCIWZb9t_-HA8pMQ-RK2wzbOumcuomsAFzYxi4hPfUQ_eP8S62RXu6xJLOLm)
+
+2. Create requirements.txt and app.py files
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh_tm4i9pvsJN5Oj0xEZLV-MDefZ7ChqMReVosxKHAV1vuwP6bW6KdzEyM-_fq4pTfPMIUPHQL5cNQJEGvBPCRI10ZPItMXqLp2Ng_NIeyh9ucKBKSrJBADAFcpvjHWP7Bws3WLdIqpGaiD2gfwkV58ue2031_-iPlWGzohHZZGwpGMX22O8IglT7Aa5NHl)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEidJZGQSNZhwHqCbwvJ81ns5qSFdzFnDTlxRiBv1Cq7__2rl_7noqhKpUwWPGC-NE8bsIO1ULM7Are8uC1cwko7EbMUfaj6FKi19hzff9rph4gGSF54bD5nHVGd0UnGlG-4FwzgJha6X6Govh3N2K-YJStjNyMX2kfCasGR_8Jw8dEIkgSbKJRw7907o2Q-)
+
+After I created the two files, after a few minutes, the model was deployed and ready to be used.
+
+In huggingface, the interface shows
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhfxErNcC2FpJByH0m6QDBtMcFMebUVlM8V4tC5ax0URERo4VrNYGzi5NMJhy3hbvA8PeQ3UlGMSNLG4nka4dNMh0COqWCDMcwQG018fijQ3Ik5ogcGD5XCIl6p1qw3OQeNTleyjnpC4oFFVkUvWr1cS74j0059opOSWbZjDuJoaFYzhAJ4Qx6t0T1tPqDw)
+
+I tested it with 2 images
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEi-DnA8lpU0JNheS8J1O4-14DJB1phOGYepOjKF8JA_zIxBa_DIW97fKLBzZ08De4z3CqiSUMRTsipTJt786aTStqanBfdMPmjxq5TNv8SXFZd7HzpZ_1yzQeWH8Y-dvmyteNEoTQd-zpbz0pn_TvhLQVIy2VO20R2fiHYlYTFKnQ2dwspdXovWYcShTqxi)
+
+Output: 'a stone path'
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhk-dZhxXHVKEqeTRsryEhMt9hx8CPCMOcPdCe3MbxUZNPEzhxT65Ig74-iwo2dO_jEVtJBDZ6WPQDUvhSxxUkEXM4MIlq9HI6uJscqaT0YnnMo8XUYDm11NCojXi4I1RvXwzEfllucHLMQH2bxFTe0DfyKGper-Rw-5TSOT2YZONiFRGCigTD4RXV1h6AM)
+
+Output: ‘a group of dogs sitting in a row’
+
+It was a useful tutorial showcasing the code needed to run various open source models and perform various tasks. 
+
+That is all for today!
+
+See you tomorrow :) 

From cb37e2be8eb022f0c940c8606bc922de9434b75d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:08:53 +0900
Subject: [PATCH 221/638] Update 2024-04-26-day-116.md

---
 _posts/2024-04-26-day-116.md | 88 +++++++++++++++++++++++++++++++++++-
 1 file changed, 86 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-26-day-116.md b/_posts/2024-04-26-day-116.md
index cadf548b6..46197c6d8 100644
--- a/_posts/2024-04-26-day-116.md
+++ b/_posts/2024-04-26-day-116.md
@@ -1,5 +1,89 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 116) Reading some research + transferring posts to the new blog
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 116!
+A quick summary of today:
+* read 4 research papers about graphs and LLMs
+* started transferring blog posts to the [new blog UI](https://ivanstudyblog.github.io/)
+
+### Do Transformers Really Perform Bad for Graph Representation? ([link](https://arxiv.org/pdf/2106.05234))
+
+Transformers dominate the field, but so far they have not been able to break into graph representation learning. The paper presents the ‘Graphormer’ which introduces several novel ideas with its architecture.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh8bDY1Lgh7KZBWhZCxmfA2tOxODZ0kuC6D5_VeEMFtxn1FWpN1STv-SLV2xyC-1Cb8d8YTMYh_XjTKQbtX1xXJMzPhEezAj3TRksJ5t78y7yPqbYuRi8XmkDWgSqlmjkVRgYBowTkJMDDTbrXm82ehewpnrZZ_3I71lsmuLz3RZsvkPssuK2xMsmSrq1pA)
+
+To better encode graph structural information, below are the 3 main ideas
+1. Centrality encoding
+
+- In the original transformer, the attention distribution is calculated based on semantic correlation between nodes, which ignores node importance (like a highly followed celebrity node is very important and can easily affect a social network). The centrality encoding assigns each node 2 real-valued embedding vectors according to indegree and outdegree, and this way the model can capture both semantic correlation and node importance in its attention mechanism.
+
+2. Spatial encoding
+
+- Transformers are infamously powerful when it comes to sequential data, but in a graph - nodes are not arranged in a sequence. To give the model some info about the graph’s structure, the paper proposes spatial encoding. By providing spatial info about each node pair, the model can better learn the graph’s structure.
+
+3. Edge encoding in the attention
+
+- Along with nodes, edges can play a key part in understanding a graph’s udnerlying structure. They propose to add to the attention mechanism a term which incorporates edge features, enhancing the model's ability to capture the relationships between nodes more effectively. This method involves considering the edges connecting node pairs when estimating correlations, as opposed to solely relying on node features.
+
+Implementation details
+
+1. Graphormer layer - building upon the classic transformer encoder, they apply layer norm before the multi-head self-attention, and the FF blocks.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhERLgv57MNo3zgBz8_33wUgkmc5x7jtW_f2TTPps8kvmLHZeT7BEy1SDe5tcRcVsHB17nh5b1AC16Xhy3mUaLXPntGMZO3FK6xj4Lb-aHtbhI53IESWc9WPOv2GLXtAALouKHQcWqtT9bQ922zo3-7iTJcifFl39k9P9LDgdfIKa5C7l1WJ2aZZaeRx_iK)
+
+2. Special node - they introduce this special(VNode) which is connected to each individual node. Its representation is updated as normal, and in the last layer, the representation of the entire graph, would be the node feature of itself (VNode). By incorporating this special node into the Graphormer, the model gains access to global information about the graph, which can improve its performance in tasks such as graph classification, node classification, and graph generation.
+
+Results after experiments
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEieD4r-zZdqCSlMWDE1Tbug2o1WXTEC-_8KSjUzXoRts2zv4PHzexiWXtFA-VYup67DIGMw3ZRnayXjfEt4x0eAok9eTv4cLUzOX21rDS2KCbO_DLNHAPaNYMDdTSTK-Y7EF1Pt2kCAo7SLWI8ZIo1qfQzZeaP-Be5YuQIRXMnfVrrGB2ew_4O-OA5821rI)
+
+Results on Graphormer’s graph representation skills
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjyeM7TaihahWT7gZsRBymTAn6lT_d-q_I9XbLvy46fa10X8DzqickY-NY8Ry33RjyBTE-cKLwoJ0Rqirebl56MkSolZISEvCx5Gi7M_wqpAXlLpiCtZAEqnS8FUEPXdTv3cgdqQSaFmmlc_dgWMhfsxvejFZNWRcsqZCmZgz9MzYMWITy_qIcc2d7oQLg7)
+
+Ablation study to see effects of different parts of the Graphormer
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjzVMXtrUOj7ZBlyhT6VgYrDmR-pC3WrHdoefYRO7JbBHaxGz5uXfwWPoVeh5q24gxrMVXYkLhJCUxmW2n7a6HKQ-4PtvC-2sE7FVsoXoO8kk3iaJMKZJ0VWd5_Z7wJ2IWiz7NrU5VQNjKlLiidD7c6BhNxCzr12KH6T3lAMyHoFNENyL1OWjJojBfuf19r)
+
+### Position-aware GNNs ([link](https://arxiv.org/pdf/1906.04817.pdf))
+
+I covered this topic on Day 111 so there was not much new things.
+
+### Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention ([link](https://arxiv.org/pdf/2404.07143))
+
+Infini-attention is introduced in this paper - a novel attention mechanism designed to address the limitations of standard Transformers when handling extremely long input sequences.
+
+Traditional Transformer models struggle with long sequences due to the quadratic complexity of the attention mechanism in terms of memory and computation and this makes them inefficient for tasks requiring long-range context understanding. Infini-attention introduces some novel ideas that help battle this limitation: 
+
+1. Compressive memory - it stores and retrieves information efficiently, unlike the standard attention mechanism which discards previous states. This allows the model to maintain a history of context without exorbitant memory costs.
+
+2. Mixed kind-of attention - combines local attention for fine-grained context within a segment and long-term linear attention using the compressive memory for global context across segments.
+
+3. Learnable gating - which allows to learn an optimal balance between local and global context
+
+Benefits:
+1. Scalability - Infini-Transformers can handle infinitely long contexts with bounded memory and computation, enabling efficient processing of long sequences in a streaming fashion.
+
+2. Efficiency - Compared to other long-context models, Infini-Transformers achieve significant memory compression while maintaining or improving performance.
+
+3. Plug-and-play - the mechanism can be easily integrated into existing LLMs through continual pre-training and fine-tuning.
+
+### OpenELM: An Efficient Language Model Family with Open-source Training and Inference Framework ([link](https://arxiv.org/pdf/2404.14619.pdf))
+
+Breaking away from past approaches that not always  offer model weights and inference code, and rely on private datasets for pre-training, Apple’s release includes the entire framework for training and assessing the language model using publicly accessible datasets. This includes training logs, various checkpoints, and pre-training configurations. (wow)
+At the core of the released models is layer-wise scaling which allows for more fficient parameter usage.
+Considering a standard transformer layer, they introduce alpha and beta hyperparams that allow to scale the attention heads, and width of FFN layers, respectively.
+It is awesome that such a big player in the AI game, releases the full cycle of the developed models.
+
+Comparison of OpenELM vs public comparable-sized LLMs
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiFKKyAlCh7eKLEbC2DrFEmnSGKdsiDvKczKrSlMdF_4aiJmqr1dFpAZf8wA2rvdUHJhe4cbKeUOK3TqxdH_8bVQQBmQ-XO60uSr2a4plpywaskfQWmgam1G36-ii_ELwVxP0lflXaFnhxuaGLRa0XxJpGBBQlmkT-aItTyMA0qz8qUjP_qriK-vZwAsmse)
+
+Also, I started transferring my blog posts to the new blog. It is a bit tidious at the start because I have to translate them, and cannot just copy-paste. Tonight I only did the first 13 days (out of ~35 days written in Korean)
+
+That is all for today!
+
+See you tomorrow :)

From cd1a6479bbcb561b8dda5fcfe45cc7e2f34bca3d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:16:07 +0900
Subject: [PATCH 222/638] Update 2024-04-27-day-117.md

---
 _posts/2024-04-27-day-117.md | 95 +++++++++++++++++++++++++++++++++++-
 1 file changed, 93 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-27-day-117.md b/_posts/2024-04-27-day-117.md
index cadf548b6..57fd23933 100644
--- a/_posts/2024-04-27-day-117.md
+++ b/_posts/2024-04-27-day-117.md
@@ -1,5 +1,96 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 117) Some linear algebra + eigenvector/values and transferring more posts to the new blog
+categories: [math,theory]
 ---
+
+## Hello :) Today is Day 117!
+A quick summary of today:
+* watched a couple of [3brown1blue](https://www.youtube.com/@3blue1brown)'s vides
+* transferred more posts to the new blog
+
+### Firstly, I watched this one about [abstract vector spaces](https://youtu.be/TgKwz5Ikpc8)
+
+In abstract vector spaces we are dealing with a space that exists independently from the coordinates we are given. and the coordinates can be arbitrary depending what we choose as our basis vector.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgKkFWDHRpkZbZDKaO3uIo8k-JhZwaIZNApfF3TV3ajFYFfdHRBKWpB9_vwvnzSKFf0-178VBJSbOvqUjZan6BwdPnBMVXXM-ej9Ya3jwMFfTOEeFU1_xBDNWu7UKkR96OS89kNisBIr2eZXNgbKsg13DZ_Xi7j6r4NGU-w0gMxLrKx4sn0cFQov2qIEOIj)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhPSZSJFr1S352SWuMo0hdYvGUUGYEkhx1bKG2NhfilDBgrVRLj52imnXlJRQ70FOFJ_IVRuEKF66GuVxrSsfLLOXh-nWe4EMI82GyQscfhHo6a-xNhz4dt_IHHU7fNAWjisA1v90l8VElUCDzWXdNVTdJwLLJ72DKddQHiSFyra3GFbHzEj5kovSggw2dj)
+
+Determinants and eigenvectors dont care about the coordinate system - the determinant tells us how a transformation sclaes areas, and eigenvectors are the ones that stay on their own span during a transformation. Se can change the coordinate system, and it wont affect the values of the above 2.
+
+Functions also have vector-ish qualities. in the same we can add 2 vectors together, we can add 2 functions to get a third, resulting function.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjuIeiFMwe04l-IWdg5K_c_MQ_ZmclZw2eK94GrfUOB-ZwteGRgbEnDsWXAQp5u0JvrjSBtnhzThWwgJXuqkSVQXYvcu3WFwzK-9FUFjei5805NE_7b_NWAH8S2kZjAUwTn6m9eD57WbnabsIQbtaPMY4DrSaGA2QJi93oVOtMdUPOEAj5zx7eEGsE29cHC)
+
+It is similar to adding vectors coordinate by coordinate.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgHnfLbdkfZI-pRtF-GlMyhCHiYltOgYSw5y4lebYWo20wCkygesaCIEDRVY3Y0ST95zTm0gX41nLRsZSRYHwDB8uPPQxZ7WpG0SKCJYxClIV5Z7TXUdSHuorLdUMzCqyiRQma818VLvPmJd95bEFm6T2eH1im0azbZ14NdLfX4tKbMEFAi_vILYIAmWh_S)
+
+Similarly, we can scale a function.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg5COfyg8k4ZtyL7ndYiX0ZXx5vi7ukMMdjtmMcS3lVjvwTiePAsc2TadjpPb5wee5t2VDHWKl73CHUWXFHh5hfsFnyfi4eIOjT2MZ9SSL7Wrj7W0bhVBNIPQOqCWPf5-VsCoseXytLQfaTn4s7_oGt0bN6_Vs_t076vzPRVdpdG3uemivnmOjQiVp6IkAj)
+
+and it is the same as scaling a vector by a number
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjKAnae7CdsAR0wC_29UX-DFkEWCTgim7rgxZ3ZnDwZr6WeG2F74-OYCC18RgPVTeas0VeB1UP8dX21mn-3-golltrrWFuOsdibqEivndVbJJ3THkkNPTJQDuihIm61W6eAKNwNMqjIQ963zslMLRkNnVM0OyaMbzPk0gzxN_PK0fIlmEzUq08xTbH6kZTl)
+
+What does it mean for a transformation to be linear?
+it needs to satisfy 2 properties:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj3Qvs13KJKZG86HCHqvlsPXFUp26unXw21vHv-F_NrvfNAB7g6uhcrw8UJ23ytlfHsPiOOIj5-1qE1cSSVXETpPTjuiztY_8y8oFFkYEdG5UsMfC-USdFyamh8_iqGNFJ149bPpkwzuRKWl9rQWP5OTAlMau8VvOddACd9c3tR8fa4Ofmtx5VG0-h9SMQ_)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjGyrO02atB45cakj0TWDYzwzyJSDIR3SOHsS2x4EBmw059OI1XPs5lB9uV_88OMBE2hlYdpX2MTN22wtlGuYAM4ytEfHcVfa85FuKeZzdpdf7n7ViSlzT7pv_7Rbfh5TABCiReaWx05lLKcrZz0ioDqRJZ3l2HZh6QdVSJADdD8zWnf3CY7nFVD_LXA5nc)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhKsKAceYR9l9JUV-Wbiq06owE5uuPCHuld3yTK6rAA0V6mmhwR-mC-6dXaQdIRdQEBYRu6OwndwUEF3jrhduU5TQtsMsWVKYAeurIh7Y2BnsTT6vJcl-JFuuTVhcNHBTcxnWtC3wR89K-zVcY8qXs2KGwur1knXw9asM-aDcJPadJ-QddjggHdTOkqrFay)
+
+Every concept in linear algebra has its twin brother with a different name when it comes to functions
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgwl-chbsUOX21jQQcceFpE5Q6aO5fakV56fwSdJvwEXQ7E-48FOFABO6sDaqW4jAHCeBexN9l-qyAJC5kTUM6665HrdabR5waCWBht1WZ2mC838LY0OJQbrzBJ1XuGIMT0Waw8uEhhSQV-ikEb_Z--9Ki12FH4M4gKzEsE1pO9ihVj5a6qUPQGdfin5AoT)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEinLIoKpy_6-bhjIbTcdWsMY348i5AjCgTIxkGfNY4Sni8gIdd2_r6yHKztIR7dV_WSBQEDDOCgOwYY7gfMg7-qqvydYV6xm2d0uajwLw4ltoED6C3TSWa0x-c_veEybsXO6br9NTh0za6KX-ceGemKN6YsAMPFJppI1Qw5FF5mtTj-BY2CiWCtoZGxLsAD)
+
+### Secondly, [eigenvectors and eigenvalues](https://youtu.be/PFDu9oVAE-g)
+
+A while ago, during Day 47-52, when I covered KAIST's AI503 Math for AI, I got introduced to these two concepts, and I kept seeing them around, knowing how to calculate them, but never 100% sure what they meant. Today I learned ^^
+
+Imagine a linear transformation in 2 dimensions.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhi6-Ifj2aDq8gxwmoKRUd91yRPstCqzrrlKSHtSgK1RFDR4YzngLf-_tkcW2MLsfLmYcT4-ircsa4ZA0LB6GlJHrnrx_7btD-Tnb_PY8c433cJLhVXjYIBPX8KHs_lnfmBK8gzydm5D-5kiiO0R3rvOrjqQ4waz6R18ACKfxYtbVM0raa0XEA-Dr-ycQ3E)
+
+becomes (we apply a linear transformation)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhEJgjk_Fw5-tYLy-EQlVpFBDIbMiBnXglhSOlwVN1ovICuaeyyRDOzZmwTBm9KlQmZ028iuuWT_4ktH90qtprDDFnSUBOKhKPyOIfDHEpZg7eBKkhC3IGGTSVspNMqmiIhs20X89mjSH9JXZtn-cPO7glEt6wjNNggTTgk1TFiXvQyMdNRE08BRy2Ytwfp)
+
+Most vectors will get knocked off their original span during the transformation. However there are some that just get stretched/squashed, but still remain on their span.
+
+For example, below, the black and green vectors remain unchanged. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEijlgF-FvfeXheFvuecB6Q3wxnX3VxsXrh7AjvovvoDGw3WO6r-pVCYNZjkdYsNh4XvoFVz-XkvhtVXWYJ0AwAUAQ38LjcuTtj7zmwDLzMSfMATGbyhUYh6RYxBmyOTMNvXLejvJxPvtdbKJJ-jBET-zdMUIm3DEyhb5qYGNplVLOiZyAk9GYo3Ce2g_Si_)
+
+becomes
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh_9MRB2RM02Tvd9AN8N7hy2dLnChrT65agiGuernPkzJsYGUZIZkgUSN2sqVlhsyo6BCYRZh-KcNShZD_jCRakTittUTTw3J_Y5j5yGQqynKqA_MmUx-vTD3PLhMJ7NJ2K-rQoAmI3ELNATpuwGiIrkRgVQxccjTD-_OtGyGvl5NZXUMClHL-GA6SXUhDr)
+
+These vectors are called eigenvectors of the transformation.
+
+And the eigenvalue is the emount the vector is stretched/squashed during the transformation.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiByBzUepgsiOTV5fZMpSJy-96TJTssqYvX1UHdnyIxa0l6DLTwB8-tiEh1jDkI15xRxL5HUq_lZNpa6XCPc0jqIXdfNKtlVG9cSLRm140vL0IuaOb-ngx2zFDyJz_b8LAuNnzORQVnUF5UpaxCm_GbdYvC3ZPz5m1fYoJjwstyOmUvgiHI8jDmUu2QsYeN)
+
+More formally, the result from the multiplication of an eigenvector with the transformation matrix is the same as multiplying that eigenvector by the eigenvalue. 
+
+Also, I found a quick trick on how to compute eigenvalues (for 2d cases). 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhvXWQFXsKZXy6xKaPIzs9N519fJ-EODWjUx1rz-i8hMhO4YkL8t2nLfTosWQ4W8mtnQ8gH4Tt1mqbdCBxMSsdNexwcPq8DdDjiZ-F58GM68Yj228upg-la50dBxA4wv1eYnYVUFS2PXm5xqwxmpkFxBKU8et72hpUxuj9IclKlC3aRIJoTOknHMKK_NG2Q)
+
+using the formula on the left, we can find the eigenvalues are: 2 +- sqrt(2^2 - -1) => 4.23606798 and -0.236067977
+
+Finally, I managed to transfer more blogs to the new blog
+
+It is indeed a bit tidious, but I have only a few Korean ones left, and the rest should be just copy-pastes. 
+
+
+
+Tomorrow, I am going to Seoul for a career event, so on the bus ride there, I will probably read 1-2 research papers that I can talk about on tomorrow's post.
+
+
+That is all for today!
+
+See you tomorrow :)

From 92dcdaa93f2848b69b6e75e7263778112b589174 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:20:23 +0900
Subject: [PATCH 223/638] Update 2024-04-28-day-118.md

---
 _posts/2024-04-28-day-118.md | 83 +++++++++++++++++++++++++++++++++++-
 1 file changed, 81 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-28-day-118.md b/_posts/2024-04-28-day-118.md
index cadf548b6..23e1eb28a 100644
--- a/_posts/2024-04-28-day-118.md
+++ b/_posts/2024-04-28-day-118.md
@@ -1,5 +1,84 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 118) Looking for a new book to read + Oxford ML summer school + short career event
+categories: []
 ---
+
+## Hello :) Today is Day 118!
+A quick summary of today:
+- received info on OxML: Fundamentals
+- reviewed interestung books on manning.com
+- attended a career event in Yonsei university
+- invited to do a talk about text 2 chart in an online Stanford event
+
+Firstly, a few months ago I applied and got accepted to attend an online ML summer school organised between AI for global good and Oxford university.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj-Pnk2zwG7y36EAhxkpazeXWmOwc6A9kqpeF0rclkqPUyIsrgpz7b1I3XVqfBY_sb7YSpgJ8PtQY5iaSdEEffPpw71Hce5B9iuqus_EE7xUjN5UGOzGSJT2RFrR8M151mAXseoo8N-BSJ0-iPx7OAjreczsq9qaMGmlEJDhG3StlI9nRti29LOGaDberok)
+
+Above is the schedule for the 4-day classes. Plus there are 2 networking sessions. The sessions are live and in Korea the above times are between 9pm and 3am. Even though the sessions are going to be recorded, I will attend them live so that I can ask questions on the spot if needed. 
+
+Secondly, during my bus ride to Seoul I started looking for an interesting book to read on manning.com and I think I found some good candidates worth reading.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj0rPOb7MAv0JHTl9TAJJ7vjU6NKOAEqmGzxc6UGD6V40fTO5HIjoexDHpWyhXRYJKoL9-RYQ3D6ANQyozxcRabysXc9QUTuS2GJ8CHbo316rnyecALQyNvYu_QOY07Pa5ZiizsLTy1cS3ZU2S3iLSUymlCzgh1nWkvX3knOrpznk4P2ftkbEdvJHP8eyAP)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgIUimheEitYCqK3JwpBGjZVr4szy8xIs0KnylvkMdo5X70MttB3iLkBBalQeR_ujc_KBNwrBT0v3hQyrlvbb01IdsCHpV1Y1o-4JhRzCEjEqSeSgbVLtCKUxQZ_tjKSSAtBN7zOXk3LnncgdkGpw3-HzmjSuZIOv2X3QjfBV9Wbx53l71MTk5g3J9rROI-)
+
+ I had a look at a few books, but the above looked the most interesting. Also, among them are 2 project type items which I might not purchase explicitly but subscribe to manning. Below are the table of contents for each one.
+
+Advanced analytics for Business
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg83egRWfAjvxVHkk3KfWl34Otm0-7Q0FQv7YjOzQFg47pNiBP6_wi5HmPNlY5nPJq3TK0vyeTZCIHr7mcO2QHF5o9VMIytXdGz-C0zq_vym62IeU0CfGsouGfwXBV9eCyue4uYnIBC6-ze0NIEwruWrwAwpA6iryenAHZ5kInkL4XqiiRWvIKVwJ4HnpkI)
+
+ This one seems good mainly because it touches practical cases in the business world. However it does not seem likely that the missing chapters will be added soon.
+
+Next, Four Al algorithms with Pothon - this is the projects tope of deal.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhIqvXketHF5H66Ee7_TgHOODe86WPouJyCP4jplPWk4tL72KXkT0Gvk819Og-reLN9_09N6jSi3sKwlGNJxAIKhynvHftIw7Y-In2xlI2Rv54-N8Jb7WDYN_Y_PggXR2I_KSnti_vcy5NrkSUxOpULzf2JMaAvRVU7xkgByC2SzscXyNxx0NbLEe7idjeO)
+
+ It includes a few projects, but the above one seems the most interesting. One negative is that I am not completely sure how recent these projects are (i.e are thes using outdated or older versions of libraries ).
+
+Next, Build a Custom Chat bot using lang chain and Chat GPT.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgN_lwj7xljDwddOEwwC5Kexbio0x-oc-MskmNjQA5MQV7cfZnHqH_Z6c8UxVNhl0fU3j4hGuOzg77rG65B2SUyeNvOrhkgrZBvf8Nnsej2uqGgOVrmdH3dpX2StHiyKM9Yd9MRL16iJg2D-R9t8XAg2BoaAUPaMm3JjToNiWPNuVujGn8mCCenneiPnKVT)
+
+A potential downside is that it requires an open AI api key which I don't have and do not want to pay for (at the moment at least).
+
+Next, Math and architectures of Deep Learning. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiDMl0OyVryRIc1DpYc7uCU1AmOtGztyeCmIFu1GXDeba853smxjYXy-n8tvPJClG_paleXmKLb1HlzSq_jgpBdV4x9N2NcsCZUZobD0tB67VFtXYhoJVjEqoiibnA6yhaJZrrZHknA8Tj4kSN8CuYpMFehFE4znGCzOY0KusT6CdMaMU5i4WgfoToccL04)
+
+this is one of the top candidates to purchase at the moment. It is complete and it never hurts to re-read and go over important moth concepts.
+
+Next, Algorithms and data structures for massive datasets.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgASN1He3QxMlTImrTufIoBR4n2aNAg2nOjlcuZiLeIcNu1UE3alP0zwcdEAEkOa2KflCNU0meA8fCiO25FcKLm3cCBaWBc_eRICDnpqhS_wSGuL1WYdddZvE__WMk7_qQj7fM-V2upVYz_ivAluuP4sMpTSwqbiXR7CxShQOw0XLFlt9h8-VARTbYH-zUn)
+
+This is another top candidate. Mainly because it is complete and also covers a, what I deem to be, an important topic - dealing with massive datasets.
+
+Next, Grokking Machine Learning.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjoVL6vBQOnzOp3c4kXpV1oHfd0CUvHvdWst3HX0Xqj1BIp3LVDy2thkt4btpkeeBHHZS9mqR6Uy1W7_gwRooAbVAZOjEt7mD7QilAI_Qfp1NG-aJhMhIbsab0uTSQ9JnYX1PcyTzxw6u2VB1u6DYRCWVSXX15iSTX0RUenpZu88B2Wbd6cKN6yfpKVX62L)
+
+ This is one of the books that I have actually seen as recommended and worth reading in various posts and blogs. So, before anything else, reading this one might be the best. 
+
+Finally, Fight fraud with ML.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjmcd9-nCoenE_HB6yOv1Ozl7ojK-PSbBc_BZx9xHFFh-GsNY6lkFKVXKWCDR__Eyvmk2pwVtsAYIafeohBRnQsLWVo_GCnCnh2A2RzL3gSUNvMOx3OVe_qT8fWcda9iIqDZ8lEpSAPu_ScYcbW9lx7RmSF2-icB9Lc4jjkeOTzWFl3kldJvzJWvfsaWKY1)
+
+ This one is extremely promising. However, the missing chapters might not be added one time soon. And those missing chapters are the most interesting.
+
+All, good choices, but I think I will start with Grokking ML.
+
+**Next, about the career event in Yonsei.**
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhfuwjG_czehmMOfxWlbTXeINz3uolnaQP7NOieMyu90-q5FjxGI-rNS7if1FJTmu5u1FpKuOWiAbXTEsFv1-ayra_C_zW-MpxeNjBYLRzdwb20FqCa3FU6z9OylUYjwMOjSRuX_q-pLGFf2So7P_0J_bGlxeBa_B6dOmgX1A0-psKgnbRttyKb_lt0cfMJ)
+
+ It consisted of 3 panels, but the first one was the most interesting to me as there were panelists from recruitment companies in Korea and thes talked about what is popular now ( Al unsurprisingly). The event was mainly for Vietnamese people as the other 2 panels were about Korea and Vietnam relationship and a Vietnamese influencer.  Never theless, I am glad I a ttended as I found out about 'Wanted Lab' which is a job website where foreigners can find work.
+
+Lastly, for the online Stanford event.
+As a student in Stanford's Al professional program, I learned about a show n tell opportunity between students that is held evers month. I decided to submit me text 2 chart app and talk about it. And... I got an email inviting me to present it and demo it at the end-of-June event. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 5a9561db73a8fa32d3d14674e36eb33007dc73b9 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:25:47 +0900
Subject: [PATCH 224/638] Update 2024-04-29-day-119.md

---
 _posts/2024-04-29-day-119.md | 75 +++++++++++++++++++++++++++++++++++-
 1 file changed, 73 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-29-day-119.md b/_posts/2024-04-29-day-119.md
index cadf548b6..dd6d3cea4 100644
--- a/_posts/2024-04-29-day-119.md
+++ b/_posts/2024-04-29-day-119.md
@@ -1,5 +1,76 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 119) Graph Convolutional Transformer application on electronic health records
+categories: [gnn,reading-research]
 ---
+
+## Hello :) Today is Day 119!
+A quick summary of today:
+* read Learning the Graphical Structure of Electronic Health Records with Graph Convolutional Transformer by Choi et al. (2020)
+* saw: Four Ways of Thinking: Statistical, Interactive, Chaotic and Complex - David Sumpter ([youtube](https://youtu.be/PPCfDe8TfJQ))
+
+I learned about Graph Transformers on Day 110 and I had saved this paper from Professor Choi and wanted to read it for a while.
+
+Firstly, I wrote some summary notes of [it](https://arxiv.org/pdf/1906.04716.pdf).
+
+One, it involves the usage of graphs, and two - Professor Edward Choi wrote it (I studied some of his lectures on [intro to AI](https://www.youtube.com/@mp2893/playlists))
+
+Studying electronic healthcare records (EHR) using deep learning can help in various tasks, including predicting diagnosis, learning medical ceoncept representations, and making interpretable predictions. EHR data is often store as hierarchical graphs as in the picture.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiRIuDNEYrCfrqTajKyqURrlovgSrysYyBufdw-dwBcp5qeXgDOkPfTQje9Yl2JClDMxjzKj1uJe1FSyJefr_3ZhsGOXnY-XPjfKknrvsXNp_1nCHZYS_7Evx8RWqYt_IFfowNXLtull7-L8RWRlFED-bM3jp4Xd-Eqgi9lJqfeG5UlWCb2usgEPzC8Oe0h)
+
+The common way to to process this data is to consider each encounter as an unordered set of features which does not care about its graphical structure. In the real-world there is incomplete information (some of the red links are missing). In addition to viewing EHR data as a graph, the paper proposes the use of the infamous Transfomer to explicitly learn the EHR structure, and also perform supervised prediction tasks - called the Graph Convolutional Transformer.
+
+Assuming links(edges) are missing, we do not have an explicit adjecency matrix A. So what they do in that case is the tell self-attention that all the nodes are fully-connected, and it will learn which connections are stronger than others.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhZv7HtQbijV3AykI7FrDVVcztJTGgRvsBD6cnJQguwGHhmJzr4axb4eA7ZDh9aFNVb8louFX1Ue3hfYCaDoK44xaZCfOxSdAWWReSH9HubheougA_tYa242kxx2N8Mm8Ii5w9p9O4cuwue7w0gizbyEnUei052xZ3hU6JwcKxhh_ba3nDRq_avi2DwCnqe)
+
+To help the transformer not to go around looking at all potential edges, based on: (1) data stats (i.e. treatment nodes can only be connected to diagnosis nodes); and (2) conditional probs from the data (i.e. p(EKG|chest pain) is usually larger than p(EKG|fever)) - they restrict the space where the search can look at.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgKHBFludmwNHBlKWK28BEYn_RHUN3Vpb8AjKzawMl0AVTwkP18L5Dzj19wXgXn42ObGoP9Sp3ezfo_vMXlDRAsbPG00fxBqLWCGiiQQrDLxbeYNO6wFAfWJcgszcIqqhCv1zOaKHII-yM4O7BboYS6I_qOCsxDOs9P4xfLl1ql8SbZK6Pa8RpyJ5Qe8SsV)
+
+They use synthetically created data and real data (eICU Collaborative Research Dataset) for evaluation on 5 methods alongside the proposed GCT (Graph Conv. Transformer).
+
+Results are:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhCt6vWh38_DJuiumwDEDnnfJYkXY7WoYeXZdGtdJPh0QT_hRZnTVSwHGmnYsWSnMgz_Ivdn7LwNdNYWdhOF5SdDJX7jyX4izS-vWvmwzLGk2rTO3YURWjX3P87b0_lKMdfZjxQJXmHvLx72v1YnCSbVx8RPeoPdjoB-JSQOqKIT90h1mspw3T-KtsCR_Ho)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj3DQbrQDqWVDwz6WnVsfgRbFm-fd2nBNbbU0llE_CbBpyVB7vYp1HFiXg_a7JCoiSctbGOT2D5Qq3G0qknaabgAJ4NgizVLIt0k1dvp4K0e-AZBQZABA1VaaF2W6XjRno_85W6J1au8aHBE86vr1k_2Ozb72YA8U5pzPduGDST-N5sc3FQSNO9NpgT-uTN)
+
+Having studied graphs for the past month or so definitely helped reading this paper ^^
+
+
+**Secondly, I took some pics/notes of the lecture on 4 ways of thinking**
+
+1. Statistical thinking
+
+We can always try to quantify something - an attitude, a change in attitude, feeling, testing whether we can taste the difference between coke and pepsi. But we don't really need to quantify everything. In addition, sometimes the data can be biased, and for example, even though students from top universities may succeed in life, there are high school dropouts that do as well. Context is important, if we quantify a player's performance measure, that does not mean that the overall game for their team improves. And finally, we need to be careful with causation vs correlation when creating models.
+
+2. Interactive thinking 
+
+Interactive thinking is to engage with information/ideas through dialogue or exploration in order to gain deeper understanding and generate new insights.
+The presenter showed these equations:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjWg96MPpTpGGRb5SNzEK-H7dQsupxC5rXsOEnh1--5J_CyEdZKJfgV18I4lt5EGZTrC8PCRxML6nFdcr7TUfETkAXTzoEl-tTE1t_2lKkIbXEzIn-moPf1UHybjgMaQb-gMaeZ-AZR6mMkxFry4_1Qr4ipKSuS75cwQuE7Zmfq6CfCCc3L4Z2hYmMvggJn)
+
+How can they make sense? top left: Person + Outside couch -> Person + Outside couch (1 person cannot move 1 couch so it stays outside). Bottom left: 2 People + Outside couch -> 2 People + Inside couch. Top right: Not smiling person X + A smiling person Y = 2 smiling people Y, and bottom right similar. This is the idea that groups of people follow and interact with each other. When a person starts clapping after a presentation, the people around them start as well, and so on. Everyone is interacting with each other, and we can understand the world through such interactions. 
+
+3. Chaotic thinking
+
+Here, the presenter talked about how 2 points can start together and even though we introduce small movement to each, after a little while they will end up on totally different paths. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgP-ffuMM8iVrMGt2unnfmgqb34WfAw3E-z9kjsmPP5wlzZduFhI9hQ9BGUJGBl1HW3lHfyJozESVgA5T5CxGP0ekh5pJ5nuPlqxT-yzJMaWzLhOT_Ph-8xp-PR8pOQPajcl8iSmd5RMR4cudZ_a3X-BQyFDQggDEA7ApracRtq7YMvV7Iy6dkizjocrzSC)
+
+There is chaotic thinking which we can regard as leaving something to chaos, and there is order thinking which says that if we really care about something, we should try to control it as much as possible, so that there is no error. Because errors stack on each other, and we end up with chaos. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjaHFMntYh8gZ80Qk4NrQvIw8v3625iMLRcgi9y7RyxyWo5AvI2JUS1BTJ5DhHPt7ZzzFAFQ0FGLvmJqq9EMRZtMPmmHfzGn4SCv7tGejrPWfhIrmtcgReabE1_7by--4mLpVj0r6ygP7IaWCEnJ0kuCgHRP_RbQl-47UzQbMmxVlL_Z54uafk0pMz4-zAb)
+
+4. Complex thinking
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiWzsi_LMCvybpvzgj3H974JO5ZCEToIsjeS-BMDBp1_BWU3LHbP4yZflNzepl4M8yxuGPSxKO1W9x3qWV1oQdDrMDUmnWn380YkUUngD_4I4R-PhDUneupotQpIawNGP0awyiT1__XbqAy2xTZPl7gQh1AWbw3gQ0OMzlzZByyvUQ3H9Snvuox3CYS-MYl)
+
+The above is a very high-level overview of the talk, in the talk I was presented with specific examples and how we can better understand each type of thinking, so it is definitely worth the watch.
+
+
+That is all for today!
+
+See you tomorrow :)

From 6a4f0892442e49d9ee6599536bc3e16b959489f3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:32:56 +0900
Subject: [PATCH 225/638] Update 2024-04-30-day-120.md

---
 _posts/2024-04-30-day-120.md | 74 +++++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-04-30-day-120.md b/_posts/2024-04-30-day-120.md
index cadf548b6..789fd8f0d 100644
--- a/_posts/2024-04-30-day-120.md
+++ b/_posts/2024-04-30-day-120.md
@@ -1,5 +1,75 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 120) Starting Stanford's CS246 - Mining Massive Datasets + MIT's Intro DL
+categories: [theory,deep-learning]
 ---
+
+## Hello :) Today is Day 120!
+A quick summary of today:
+* started [MIT's Intro to deep learning](https://youtu.be/ErnWZxJovaM) 2024 course
+* started [Stanford's CS246: Mining massive datasets](https://web.stanford.edu/class/cs246/)
+
+Last night I saw a notification that MIT is going to livestream their first intro to DL lecture on youtube (from their 2024) course, and this morning I decided to check what it's about. [This](http://introtodeeplearning.com/) is the official website, and it will provide lectures + homeworks. The course lectures will go from 29th April to 24th June. And the schedule is:
+
+![image](https://github.com/user-attachments/assets/376b985f-4322-4ffc-b3a5-8cfba332f683)
+
+As a 'basics lover' I am excited to go over old and hopefully new material. I am excited to see MIT's take on teaching intro to DL. 
+
+### The summary of the 1st lecture is bellow
+
+It started it where DL fits with AI and ML.
+
+![image](https://github.com/user-attachments/assets/2137b92e-7523-4f26-b708-a39921b04ee1)
+![image](https://github.com/user-attachments/assets/065743eb-c99b-4da2-9730-33c4820d9323)
+
+Gave a general and very well put (as expected for MIT) explanation of a neuron/perceptron
+
+![image](https://github.com/user-attachments/assets/b4eabeac-ed6f-4205-81ac-a7cb8d418736)
+
+Provided an example of a simple neural network
+
+![image](https://github.com/user-attachments/assets/76d6302a-9206-40af-b551-4c29f83cb031)
+
+Also, how does the network know how wrong it is? Thanks to the loss
+
+![image](https://github.com/user-attachments/assets/f42468b5-6279-45f2-a1d5-c964aed351ec)
+![image](https://github.com/user-attachments/assets/0870a194-a1d1-46f0-b655-ee58a6cc6eda)
+
+However, using the whole dataset to update the weights W (Gradient Descent) might not be the best idea, especially with the amount of data available these days. That is where Stochastic Gradient Descent comes in.
+
+![image](https://github.com/user-attachments/assets/72823b4f-f75d-4cd7-9529-3115599c89a6)
+
+In the end, we got introduced to the 2 of the most common regularization techniques: dropout, early stopping. 
+
+### As for the CS246: Mining massive datasets
+
+It was on my list for a while, the reason I did not start it earlier was because I was not sure if there were videos on it (even if there were not, I would still end up doing it, but a video lectures just adds a little more to just reading slides/book chapters). I saw today that there are [lecture videos from winter 2019](https://youtu.be/jofiaetm5bY?list=PLoCMsyE1cvdVnCgHk43vRy7PVTVWJ6WVR). Yes it was 4 years ago, but I compared slides from the most recent release (winter 2024) with the slides from winter 2019 and there is little deviation towards the end, when I can just watch the lecture and then by myself read the slides. 
+
+Today I covered lecture 1
+
+Covered topics:
+
+MapReduce, Spark, Spark's RDD, cost measures
+
+![image](https://github.com/user-attachments/assets/f60d448d-7566-4e49-aacd-f6cdee5ae104)
+![image](https://github.com/user-attachments/assets/4f1c1690-7018-4b98-becd-b8fdecbdb7fb)
+![image](https://github.com/user-attachments/assets/b4e99c18-0436-4732-8a6e-ebd937214877)
+![image](https://github.com/user-attachments/assets/f42f0f1e-28ab-4789-b34d-fb46ede8dd55)
+![image](https://github.com/user-attachments/assets/9bf36803-5586-4720-b991-4c155c573b04)
+![image](https://github.com/user-attachments/assets/45412246-e95e-4527-8ac3-22ff5e94afe8)
+
+I also registered for a new event hosted by Global AI for good that's in collaboration with Oxford University - [MLx Generatie AI (Theory, Agents, Products)](https://www.oxfordml.school/genai) on 22-24th August 2024. ^^
+
+Topics to be covered:
+
+Advanced theoretical topics in representation learning (e.g., vision, language, multi-modal, …)
+Agentic AI (e.g., agentic reasoning and design patterns)
+Human+AI alignment
+Building Gen. AI products — from model.fit() to market.fit()
+Using SOTA foundation models (e.g., fine tuning, RAG, RLHF, prompt engineering, …)
+Application of large frontier models in applied domains (e.g., medicine, finance, education, …)
+
+
+That is all for today!
+
+See you tomorrow :)

From 515ff28414d9bba25c86679fd2a5ba708db7a8d6 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:40:13 +0900
Subject: [PATCH 226/638] Update 2024-05-01-day-121.md

---
 _posts/2024-05-01-day-121.md | 105 ++++++++++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-01-day-121.md b/_posts/2024-05-01-day-121.md
index cadf548b6..a3a1a0788 100644
--- a/_posts/2024-05-01-day-121.md
+++ b/_posts/2024-05-01-day-121.md
@@ -1,5 +1,106 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 121) Uncovering the full reason behing multicollinearity + Frequent itemset mining lecture
+categories: [theory,stats,math]
 ---
+
+## Hello :) Today is Day 121!
+A quick summary of today:
+* finally [uncovered the full story behind multicollinearity](https://www.kaggle.com/code/divakaivan12/the-full-story-behind-multicollinearity)
+* covered lecture 2 from Stanford's CS246: Mining massive datasets: [Frequent itemset mining](https://youtu.be/2NyZmnuIicw?list=PLoCMsyE1cvdVnCgHk43vRy7PVTVWJ6WVR)
+
+Whenever I read books, or see blogs, ask an LLM, what happens when there is multicollinearity - the same generic answer is given: multicollinearity causes the estimated parameters to be unreliable. I always wanted to ask why? and had a ton of questions that follow. Well today I finally 'got my hands dirty' and got into the depths of it. Not only does it feel awesome to uncover the truth, but also I am seeing some of the math concepts that I learned about - eigenvalues/vectors, SVD, matrix's rank, determinant and condition number actually being used. 
+
+I wrote my notes, which I shared on r/learnmachinelearning to get opinions and feedback. But also I put it all in a colab/[kaggle notebook](https://www.kaggle.com/code/divakaivan12/the-full-story-behind-multicollinearity).
+
+Here is the gist
+
+### Building up to multicollinearity
+
+* Eigenvalues and Eigenvectors: when an eigenvalue is close to 0, it indicates that the associated eigenvector does not scale much during the applied transformation
+* Minimal scaling: the vector does not scale (much), this means the transformation has very little effect along the direction of that eigenvector (little effect on the features or characteristics of the data that are aligned with that particular eigenvector)
+* Redundancy: if an eigenvector is nearly parallel to the null space of the transformation matrix, this suggests there is a linear combination of the data (features in the design matrix) that has no effect on the outcome
+* Multicollinearity: redundancy (or near-redundancy) in the data is a characteristic of multicollinearity - one predictor variable can be approximately expressed as a linear combination of the others. If we have multicollinearity, there is near linear dependence and the matrix is (near) singular, which makes X^TX ill-conditioned and numerically unstable (sensitive to pertrubations) which inflates the parameter variances, resulting in poor estimates
+
+In the notebook there are some code examples as well showing the eigenvalues, rank, condition number and a logistic regression when THERE IS and there is NOT multicollinearity.
+
+Below are the results from the final linear regression:
+
+* There is multicollinearity
+
+```
+Multicollinearity case:
+
+eigenvalues=array([  0., 275.])
+
+Rank of the matrix: 1
+
+condition_number=inf
+
+
+
+Original Coefficients and Intercept:
+
+Coefficients: [0.80039912 1.60079824]
+
+Intercept: 0.03991352436920792
+
+
+
+Perturbed Coefficients and Intercept:
+
+Coefficients: [-8.6065131   6.30414128]
+
+Intercept: 0.03940363891543086
+```
+
+* There is NO multicollinearity
+
+```
+No multicollinearity case:
+
+eigenvalues=array([  1.9623732, 188.0376268])
+
+Rank of the matrix: 2
+
+condition_number=95.82154225314864
+
+
+
+Original Coefficients and Intercept:
+
+Coefficients: [2.05446569 0.92175784]
+
+Intercept: 0.15618973818845738
+
+
+
+Perturbed Coefficients and Intercept:
+
+Coefficients: [2.05486836 0.92235525]
+
+Intercept: 0.1525767124126478
+```
+
+The above showcases one of the consequences of multicollinearity - making the parameters sensitive to even small changes. For the full code I recommend visiting the kaggle colab (it is a quick read). 
+
+
+
+### Secondly, my notes from the lecture on frequent itemset mining
+
+Covered topics:
+
+Frequent itemsets, association rule mining, finding frequent itemsets, finding frequent pairs, A-Priori algorithm, PCY (Park-Chen-Yu) algorithm, frequent itemsets in <= passes
+
+![image](https://github.com/user-attachments/assets/ecd37966-e226-4539-9227-7ace18db9edc)
+![image](https://github.com/user-attachments/assets/41d3cffb-8d80-474e-b5ef-cf0b8f753a15)
+![image](https://github.com/user-attachments/assets/ca04338f-8d66-4a5f-a719-6627f4370b66)
+![image](https://github.com/user-attachments/assets/b742a0ca-9418-46d2-83fa-ff631b18f222)
+![image](https://github.com/user-attachments/assets/09f4842a-d19e-4cf1-af0e-524afb47f393)
+![image](https://github.com/user-attachments/assets/b5207ea5-efab-47bb-9791-17ad75076b19)
+![image](https://github.com/user-attachments/assets/7b899542-e2a8-47a6-b3e0-24a4d7ea3394)
+![image](https://github.com/user-attachments/assets/a7b5d9e8-d97c-4894-8b70-7b782db55aa6)
+
+That is all for today!
+
+See you tomorrow :)

From 70c1a99fa36b34f2faa1f0eb9e2b2c5f2eabcb32 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:43:53 +0900
Subject: [PATCH 227/638] Update 2024-05-02-day-122.md

---
 _posts/2024-05-02-day-122.md | 58 ++++++++++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-02-day-122.md b/_posts/2024-05-02-day-122.md
index cadf548b6..d26b97bdf 100644
--- a/_posts/2024-05-02-day-122.md
+++ b/_posts/2024-05-02-day-122.md
@@ -1,5 +1,59 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 122) Dive into Deep Learning - Interactive deep learning book with code, math, and discussions
+categories: [theory,deep-learning]
 ---
+
+## Hello :) Today is Day 122!
+A quick summary of today:
+* discovered and read a bit of [Dive into Deep Learning](https://d2l.ai/index.html)
+
+I randomly found the book from some post on r/learnmachinelearning about books on DL (I always manage to find some new cool book from these weekly posts)
+
+It seems a bit crazy that such a book is available for free. It is adopted by so many university courses and so many great practitioners participated in its writing. 
+Every topic contains an executable colab linked to it, and its available in PyTorch, Tensorflow, MXNET and Jax. 
+
+Today I had a read through 3. Linear Neural Networks for Regression
+
+![image](https://github.com/user-attachments/assets/4d08363c-e9c7-4931-8f34-30041bb056bf)
+
+4. Linear Neural Networks for Classification
+
+![image](https://github.com/user-attachments/assets/d35a836b-27df-4611-89fe-354768a4648c)
+
+
+5. Multilayer Perceptrons
+
+![image](https://github.com/user-attachments/assets/95fea018-380e-469d-9344-fe9fbdc16952)
+
+6. Builders’ Guide
+
+![image](https://github.com/user-attachments/assets/98913150-62bd-4f0f-819b-19520d23f6c0)
+
+The book is great because under each topic there is also a live discussion section where one can ask questions.
+
+Overall (so far!), I think An Introduction to Statistical Learning is better. It has code as well and very detailed exercises. The problem I encountered in the above 4 chapters is that sometimes my browser did not read the math notations, so I had to open the page in colab in order to see the proper math notations. The explanations were fine, but maybe I just hold ISL very high in my list haha. 
+
+Dive into DL has so many chapters, but next I am planning to read (and maybe take notes of:)
+
+12. Optimization Algorithms (because it dives into the details of each optimizer)
+
+![image](https://github.com/user-attachments/assets/8bf4062b-f586-46d2-aabb-22df9d3226d1)
+
+21. Recommender Systems (seeing some of the theory from XCS224W:ML with Graphs in practice, and from another point of view)
+
+![image](https://github.com/user-attachments/assets/968278db-6075-4fa9-ab32-f11550602a24)
+
+22. Apendix: Math for DL (will read through it)
+
+![image](https://github.com/user-attachments/assets/5adaccd3-e5b4-48b7-9388-25a2b49a7770)
+
+Tomorrow evening the MLx Fundamentals organised by Global AI for good and Oxford University so I am very excited. 
+
+The full schedule for day 1 (out of 4) is:
+
+![image](https://github.com/user-attachments/assets/76cf49ec-23e4-4598-a019-f54276a737bd)
+
+That is all for today!
+
+See you tomorrow :)

From ff97d8922325a78c2ac8988939eedea5ef07b67e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:47:33 +0900
Subject: [PATCH 228/638] Update 2024-05-03-day-123.md

---
 _posts/2024-05-03-day-123.md | 44 ++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-03-day-123.md b/_posts/2024-05-03-day-123.md
index cadf548b6..a7723753c 100644
--- a/_posts/2024-05-03-day-123.md
+++ b/_posts/2024-05-03-day-123.md
@@ -1,5 +1,45 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 123] Optimization algorithms chapter from Dive into DL
+categories: [deep-learning,theory]
 ---
+
+## Hello :) Today is Day 123!
+A quick summary of today:
+* read about optimization algorithms, and maths for DL from Dive into Deep Learning
+
+Firstly, my notes from the optimization algorithms chapter
+
+Covered topics:
+
+Common challenges in DL optimization, convexity, convexity properties, SGD, momentum, Adagrad, RMSProp, Adadelta, Adam, LR schedulers
+
+![image](https://github.com/user-attachments/assets/aedb5784-5fc2-4714-b310-2c9320367206)
+![image](https://github.com/user-attachments/assets/824d8374-417c-4a0c-9f18-2c54d45c9be0)
+![image](https://github.com/user-attachments/assets/8e2ef566-c914-42ea-8461-f1b65a199ab8)
+![image](https://github.com/user-attachments/assets/e6b6a1d6-3788-4a2b-a797-c3f6ec4a324a)
+![image](https://github.com/user-attachments/assets/f4e2bdf6-24d0-4f60-ae0a-d66861faadd5)
+![image](https://github.com/user-attachments/assets/5dcda9c3-aec0-44b4-b242-67c7057dedd6)
+
+
+### As for the maths for DL
+
+This is actually in the apendix, but I decided to read through it, just to check if there is any interesting math. 
+
+The content is as follows:
+
+![image](https://github.com/user-attachments/assets/439be81a-cc2f-48cd-937c-9721c589f854)
+
+Having this appendix is very nice and definitely helps to answer questions like for example when I went to the roots of multicollinearity (on Day 121) - understanding the math helps understand the root of problems. 
+
+
+
+As for the MLx Fundamentals it started just as I am finishing this blog. The first day is from 9pm to 3am in my local time, so I will write about it in tomorrow's blog. 
+
+Schedule (time is in BST)
+
+![image](https://github.com/user-attachments/assets/b3f715c5-1b08-43e1-b50d-988175561100)
+
+That is all for today!
+
+See you tomorrow :)

From f89d7943e5c9cee4f32f2500bb26e50793f26677 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:55:22 +0900
Subject: [PATCH 229/638] Update 2024-05-04-day-124.md

---
 _posts/2024-05-04-day-124.md | 69 ++++++++++++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-04-day-124.md b/_posts/2024-05-04-day-124.md
index cadf548b6..51ec1e3a9 100644
--- a/_posts/2024-05-04-day-124.md
+++ b/_posts/2024-05-04-day-124.md
@@ -1,5 +1,70 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 124) MLx Fundamentals Day 1 - Intro to ML, Naive Bayes, Factorization methods
+categories: [traditional-machine-learning,deep-learning,applying-knowledge,theory]
 ---
+
+## Hello :) Today is Day 124!
+A quick summary of today:
+* Introduction to Machine learning by Volodymyr Kuleshov from Cornell Univeristy 
+* Naive Bayes practical session by Richard Willis from King's College London
+* Factorization methods by Cho-Jui Hsieh from UCLA
+
+![image](https://github.com/user-attachments/assets/38224cf5-b969-4972-b252-873584d26d3c)
+
+### Introduction to Machine learning by Volodymyr Kuleshov from Cornell Univeristy 
+
+It was really an amazing introduction. Personally it did not cover new stuff for me, but as an intro I believe it was top. What is supervised ML, OLS, Covered Non-Linear Least Squares, Overfitting, Regularization
+
+### Naive Bayes practical session by Richard Willis from King's College London
+
+The interesting bit was implementing a Naive Bayes classification model from scratch. 
+
+I feel like this is one of the simplest explanations of Bayes theorem that I have seen/read so far (of prior, likelohood and posterior). Richard Willis is a Phd
+
+![image](https://github.com/user-attachments/assets/f83f91e9-0d25-4e25-9950-31c2938f7beb)
+![image](https://github.com/user-attachments/assets/3478eca1-5752-412e-9f6c-bd1b957b3c1e)
+![image](https://github.com/user-attachments/assets/f1e23bda-0f7a-4689-9e58-46cfdcec01a4)
+![image](https://github.com/user-attachments/assets/ecee50a6-4bac-48fa-8df1-cd1f4540a0e8)
+![image](https://github.com/user-attachments/assets/cae34390-1bb6-4fe3-87e3-c3a8b1afaf77)
+
+**Implementation**
+
+Computing the prior
+
+![image](https://github.com/user-attachments/assets/25c0d00d-9427-4b63-a4b1-b95cd9d133b9)
+
+Computing the likelihoods 
+
+![image](https://github.com/user-attachments/assets/4995337c-5c32-47c6-bb3d-e3dd965f27f5)
+
+Computing log posterior
+
+![image](https://github.com/user-attachments/assets/639352a9-db2b-4d03-b2ad-0dab6415b127)
+
+Complete NaiveBayesClassifier
+
+![image](https://github.com/user-attachments/assets/dc62e379-50d5-4f17-a62e-acf1a5305d03)
+
+### Factorization methods by Cho-Jui Hsieh from UCLA
+
+I went over the recording and took notes.
+
+Covered topics:
+
+Matrix factorization approach, Altering Least Squares, SGD, extreme multi-label classification, two-tower models, low-rankness for efficient DL, param efficient fine-tuning
+
+![image](https://github.com/user-attachments/assets/f8a5bdb8-65ed-4420-bd16-b62a02811cde)
+![image](https://github.com/user-attachments/assets/9aa7cf54-b0d7-4f5b-8289-1cd905186b34)
+![image](https://github.com/user-attachments/assets/91e58392-4956-4649-8832-67846b210c6f)
+![image](https://github.com/user-attachments/assets/896f3024-d532-41f2-850d-873e0158a5e1)
+![image](https://github.com/user-attachments/assets/b9feb7e5-5c1e-4870-a268-8bea7f13d280)
+![image](https://github.com/user-attachments/assets/590da58a-5535-4a87-a4d7-348a57db5f5d)
+![image](https://github.com/user-attachments/assets/fda11df9-a669-4f0c-b981-47b1d5b77d43)
+![image](https://github.com/user-attachments/assets/12683a6c-a1c7-47e8-9d5b-9c0668a0e794)
+
+I am writing this blog in the break of Day 2. I will share what Day 2 of MLx Fundamentals is about tomorrow.
+
+That is all for today!
+
+See you tomorrow :)

From 33293177de3df59980053374dd9a84de9e393d33 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 20:00:36 +0900
Subject: [PATCH 230/638] Update 2024-05-05-day-125.md

---
 _posts/2024-05-05-day-125.md | 92 +++++++++++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-05-day-125.md b/_posts/2024-05-05-day-125.md
index cadf548b6..a6ef0ad21 100644
--- a/_posts/2024-05-05-day-125.md
+++ b/_posts/2024-05-05-day-125.md
@@ -1,5 +1,93 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 125) MLx Fundamentals Day 2 - Causal representation learning, optimization
+categories: [deep-learning,theory,math,applying-knowledge]
 ---
+
+## Hello :) Today is Day 125!
+A quick summary of today:
+* listened to day 2 lectures on causality and optimization of [MLx Fundamentals](https://www.oxfordml.school/fundamentals)
+* learned a bit of STATA
+
+The schedule was:
+
+![image](https://github.com/user-attachments/assets/b07bed01-f554-4c68-b586-c0b3303e0141)
+
+The recording of the lectures was released about 30 mins ago, so going over it will be my task for tomorrow. 
+
+The 1st lecture from Professor Kun Zhang from CMU was more specifically about causal representation learning. 
+
+For example finding hidden variables. 
+
+![image](https://github.com/user-attachments/assets/76c8d6ac-dcbf-4a14-a6e2-44bc0ef2b7ae)
+
+Here if we look at just the relationship between cholesterol and exercise (right graph) we can see they have a positive relationship. Which is quite weird, and when we incorporate age into the picture, we can see the actual negative relationship. 
+
+![image](https://github.com/user-attachments/assets/c3c40736-7cca-4c40-9e60-8f4af204c323)
+
+In this case there are treatment A and B for kidney stones, and if we just look at the overall, without accounting for stone size, we might conclude that B is better. But if we incorporate stone size into the picture, A is better in both cases. if we understand the problem well, there is no paradox because we know that Both is wrong because there is a hidden variable - in this case size of the stone
+
+### The 2nd lecture by Professor Chi Jin from Princeton University was amazing. 
+
+I got a ton of resources and even though it was math heavy - the explanations were very clear and the TAs helped a lot in the lecture's slack channel. I will take notes and share on a later day when the recording gets uploaded.
+
+Actually from this lecture I got very nice extra material to dive deeper into optimization after I rewatch and take notes of his lecture last night:
+
+* Professors Chi Jin's [lectures on Optimization](https://sites.google.com/view/cjin/teaching/ece539cos512-2021-ver?authuser=0) in Princeton from Spring 2021
+* [Lectures on Convex Optimization](https://shuyuej.com/books/Lectures%20on%20Convex%20Optimization.pdf) (Yurii Nesterov 2018)
+* [Convex Optimization](https://arxiv.org/pdf/1405.4980): Algorithms and Complexity (Sébastien Bubeck 2015)
+
+### The 3rd practical session on Optimization and DNN by Ziyan Wang from King's College London
+
+as it was 2-3.30am for me in Korea I could not attend live but I went over the colab myself (I will need to rewatch later for any extra info that was shared), and below is a summary.
+
+Part I of the practical session (Optimization)
+
+For a simple model:
+
+X = np.linspace(0, 1, 100)
+
+y = 5 * X + 2
+
+We see how the params converge over the epochs
+
+![image](https://github.com/user-attachments/assets/0ada61b6-104d-4005-974d-f8e8d17d3249)
+
+Then using scikit-learn's make_regression function we saw the loss function's surface
+
+![image](https://github.com/user-attachments/assets/aea00ee7-9763-4cb4-ab38-17718cff8bb8)
+
+And how it moves through iterations
+
+![image](https://github.com/user-attachments/assets/60aebabd-3c7a-4444-bd63-c05885329e96)
+
+Here is a case with a smaller learning rate
+
+![image](https://github.com/user-attachments/assets/84a85b3a-5369-43ce-8dd6-d65f8b6c9dc5)
+
+And a case where the lr is too big where the algorithm actually diverges (ending at the top)
+
+![image](https://github.com/user-attachments/assets/4d37d8f2-c957-4a2c-987d-993fa0c3fcb8)
+
+Finally we saw a comparison on the same loss surface of GD, SGD, mini-SGD and Adam (pictures are in the same order)
+
+![image](https://github.com/user-attachments/assets/3aa38292-88e2-447c-9f41-d1dd9ebc4f7e)
+![image](https://github.com/user-attachments/assets/345d8140-feab-475e-9efb-4bfd47aa33c4)
+![image](https://github.com/user-attachments/assets/9a57d528-3270-4fce-8723-c0d4d9eb27c0)
+![image](https://github.com/user-attachments/assets/d70157b9-ea2b-4759-9a71-f89b0662e033)
+
+comparison
+
+![image](https://github.com/user-attachments/assets/31e7086b-b86b-4fda-a8c7-f8cb70e8df9f)
+
+There was Part 2 as well, about building a simple CNN model to classify CIFAR images.
+
+### As for why I wrote that I learned a bit of STATA
+
+For my girlfriend's class, she has to build some kind of linear model using data that she found online. We found some data about [MPI](https://ophi.org.uk/global-mpi) (Multidimensional Poverty Index) from the University of Oxford, and I helped her with some data clearning and manipulation in python. There are probably functions like that in STATA as well, but she wanted to learn a bit of python. We dealt with some missing variables, got dummies for categorical variables, and then I joined her to see how running a regression is stata looks like. We can use the `reg` and the 1st variable after is the y, and the one(s) after that are Xs. Also we ran some VIF (using `estat vif`) to explore multicollinearity. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 17b825fa31a01592e6641b528cf8eeb5bcf28838 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 20:23:03 +0900
Subject: [PATCH 231/638] Update 2024-05-06-day-126.md

---
 _posts/2024-05-06-day-126.md | 58 ++++++++++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-06-day-126.md b/_posts/2024-05-06-day-126.md
index cadf548b6..24808ced5 100644
--- a/_posts/2024-05-06-day-126.md
+++ b/_posts/2024-05-06-day-126.md
@@ -1,5 +1,59 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 126) Optimization lecture by Chi Jin from Princeton University + using Docker for the 1st time
+categories: [applying-knowledge,theory,math]
 ---
+
+## Hello :) Today is Day 126!
+A quick summary of today:
+* went over two of the lectures from Day 2 of MLx Fundamentals - optimization by Chi Jin and practical session on optimization by Ziyan Wang from King's College London
+* deployed a basic ML model using docker
+
+I wanted to go over Professor Jin's lecture again and take notes, because I plan on taking his lecture on optimization and reading some of the suggested books which I shared yesterday.
+
+Covered topics:
+
+Gradient descent, smooth functions, Descent Lemma, convex optimization, accelerated GD, stochastic optimization, SGD, nonconvex optimization.
+
+![image](https://github.com/user-attachments/assets/d59304fd-372b-4477-bcec-9e3fc0cf172e)
+![image](https://github.com/user-attachments/assets/435d9dcf-0da8-4942-8931-99ef78bef219)
+![image](https://github.com/user-attachments/assets/9a0822da-a81e-41e2-bf04-9f2b146efb91)
+![image](https://github.com/user-attachments/assets/d0e65505-4059-4634-a9ec-93a32fb7f8e0)
+![image](https://github.com/user-attachments/assets/4bf5485d-db4c-463d-8ee7-02aff92dca5c)
+![image](https://github.com/user-attachments/assets/50c31aff-41b2-427e-bff6-979e002668b7)
+
+As for the practical tutorial, the material was covered in yesterday's post and nothing extra special was part of the live lecture. 
+
+Tomorrow I will rewatch and take notes on Professor Kun Zhang from CMU's lecture on causal representation learning.
+
+### I also found a simple [tutorial](https://dev.to/pavanbelagatti/a-step-by-step-guide-to-containerizing-and-deploying-machine-learning-models-with-docker-21al) on deploying an ML model with docker.
+
+I registered for this course: DataTalksClub's MLops zoomcamp which starts soon. 
+
+So it has been on my mind to start learning about MLops. Today I searched quickly and found the above nice short tutorial on building a small app.py with an iris prediction model. 
+
+I installed Docker desktop (for the 1st time ever)
+
+![image](https://github.com/user-attachments/assets/0e7f8d20-6bfd-4089-933f-49a1da717bd4)
+
+Wrote this in app.py
+
+![image](https://github.com/user-attachments/assets/a89e16f3-9125-4124-8690-3477256a674e)
+
+Created a requirements.txt
+
+And a Dockerfile 
+
+![image](https://github.com/user-attachments/assets/36eab827-6743-4c86-b6a4-7d6690d2ba60)
+
+And followed the commands to containarize the app, build an image, run the docker container, and tag and push it to Docker hub. 
+
+So now in my Docker desktop I can run, and I get the output which is the accuracy and Model saved! as in my app.py
+
+![image](https://github.com/user-attachments/assets/4a83f1fe-75ea-45e4-ac73-58167b2d188d)
+
+But also, the image is on docker hub as well ([here](https://hub.docker.com/repository/docker/timee98642/ml-docker-app/general)). I am yet to exactly know the potential of what I did today, but I think it is a good 1st step towards learning more about MLops.
+
+That is all for today!
+
+See you tomorrow :)

From 7093cc25aa277b7afd48b531f4023e062a282e7f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 20:28:49 +0900
Subject: [PATCH 232/638] Update 2024-05-07-day-127.md

---
 _posts/2024-05-07-day-127.md | 100 ++++++++++++++++++++++++++++++++++-
 1 file changed, 98 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-07-day-127.md b/_posts/2024-05-07-day-127.md
index cadf548b6..9c31a2657 100644
--- a/_posts/2024-05-07-day-127.md
+++ b/_posts/2024-05-07-day-127.md
@@ -1,5 +1,101 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 127) Serving an API endpoint for news classification + Stanford's CS109
+categories: [theory,applying-knowledge,stats]
 ---
+
+## Hello :) Today is Day 127!
+A quick summary of today:
+* learned a bit more about Docker's potential and deployed a model to classify news text into categories ([github](https://github.com/divakaivan/classify-news-app))
+* started watching Stanford's [CS109](https://www.youtube.com/playlist?list=PLoROMvodv4rOpr_A7B9SriE_iZmkanvUg) Probability for computer scientists and the professor is so amazing
+* saw a [visualization](https://youtu.be/SGknFWqJA8o) of how activations change over epochs
+
+### Firstly, about the mini-project
+
+After yesterday, I wanted to learn more about what Docker can do when it comes to MLops. Luckily, I found [this](https://youtu.be/h5wLuVDr0oc) tutorial which develops a text's language classification model, bundles it with Docker and publishes an endpoint to Heroku. How did I do it?
+
+For a start, I wanted to do a simple model too, so I chose to write a simple model that classifies a piece of news into categories like sport, politics, tech, etc. I found a dataset on Kaggle and trained a simple multinomial naive Bayes that takes a tokenized bag of words and classifies them into sports, tech, politics, entertainment and business (the code is in my [repo](https://github.com/divakaivan/classify-news-app/blob/main/app/model/model-dev.ipynb)). 
+
+Then I packaged the model with pickle, and using FastAPI, I created an endpoint that takes text, sends it to the model and we get a response back like:
+
+Query: 'Chelsea scored a great goal last night'
+
+Response: {'Politics': 0.0001, 'Sport': 99.9926, 'Technology': 0.0, 'Entertainment': 0.0073, 'Business': 0.0}
+
+Using FastAPI, using the API locally was easy, I went to the chosen port
+
+![image](https://github.com/user-attachments/assets/90360912-6b7a-4150-a3e7-763580c91509)
+
+To query the model we use /predict
+
+But instead of installing Postman to send requests to the API, FastAPI provides a /docs endpoint where we can test the functionality (we can see all the available endpoints)
+
+![image](https://github.com/user-attachments/assets/7b2167e2-918b-4aec-8d67-ed65e593ed49)
+
+And we can try out the /predict endpoint right there
+
+![image](https://github.com/user-attachments/assets/f666d854-252b-4888-b781-f4e0e9d5d317)
+
+Write the string, click execute, and 
+
+![image](https://github.com/user-attachments/assets/8210a0c2-24d1-4d8b-a820-5cd0ec8bf0a3)
+
+The only part I did not do, is publish the app to heroku because they require a credit card. But using Docker I saw how that can be done.
+
+### Secondly, about Stanford's CS109 Probability for Computer Scientists course
+
+I randomly stumbled upon it, and wow... Professor Chris Piech is so amazing. His teaching is so engaging and even though I am just sitting in front of a screen, his excitement to teach is making me even more excited to learn. 
+
+Today I watched Lecture 14: Modelling
+
+It was mainly about joint probability distributions, constructing Bayesian networks and using marginalization to obtain the probability distribution of a subset of a set of random variables from the joint probability distribution of all variables.
+
+Lecture 15: General Inference
+
+It introduced covariance/correlation, rejection sampling, using continuous and binary variables in Bayesian nets and sampling,  
+
+Lecture 16: Beta distribution
+
+It talked about the Beta distribution - used as a prior distribution for modeling probabilities
+
+![image](https://github.com/user-attachments/assets/ffdff66e-63d0-40eb-9891-0f42dd47389b)
+
+With the beta distribution instead of having a sinple number to represent a probability, we have a distribution and we can represent our uncertainty.
+In cases like the below, we can say the in the 1st case, the beta distribution is like uniform, but the 2nd we can that the statement is more reliable thanks to the beta distribution. 
+
+![image](https://github.com/user-attachments/assets/399dbf89-3233-4609-9ee7-3b3baa27d69e)
+
+
+I will watch a few more in the next days ^^ Listening to the professor feels relaxing and interesting (even though it is undergrad statistics).
+
+
+### Finally, about the visualization of activation functions in a neural net
+
+In the video linked, we are shown non-linear data (on the left), and the neural net that will fit it (on the right)
+
+![image](https://github.com/user-attachments/assets/e55d7f31-df46-438b-84ae-6486441304b5)
+
+During training, we see how each neuron contributes to the change from a straight line to fitting this curvature. 
+
+At the start:
+
+![image](https://github.com/user-attachments/assets/2de3bdeb-837e-47f6-b75b-96b7f0a4b408)
+
+Middle during training:
+
+![image](https://github.com/user-attachments/assets/34f9ec6d-e1c6-4bdb-a906-7405cef016fb)
+
+Towards the end:
+
+![image](https://github.com/user-attachments/assets/8e01e99f-2772-44ab-9770-37f3e7fcc8ea)
+
+Final fitted data:
+
+![image](https://github.com/user-attachments/assets/5d555803-126a-478c-b53b-1569dd611381)
+
+
+The video has animations as well and it is a very neat way of showing how each neuron controbutes. 
+
+That is all for today!
+
+See you tomorrow :)

From acbc7bc82846744f4479b04a0640cf0ea22266de Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 20:35:08 +0900
Subject: [PATCH 233/638] Update 2024-05-08-day-128.md

---
 _posts/2024-05-08-day-128.md | 108 ++++++++++++++++++++++++++++++++++-
 1 file changed, 106 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-08-day-128.md b/_posts/2024-05-08-day-128.md
index cadf548b6..a149de8b8 100644
--- a/_posts/2024-05-08-day-128.md
+++ b/_posts/2024-05-08-day-128.md
@@ -1,5 +1,109 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 128) IBM Consulting Insights Virtual Careers event + More of CS109
+categories: [theory,stats]
 ---
+
+## Hello :) Today is Day 128!
+A quick summary of today:
+* attended IBM Consulting Insights: [Virtual Careers event](https://www.brightnetwork.co.uk/graduate-events/ibm/ibm-consulting-insights-virtual-careers-event-is-back)
+* watched a few more lectures of Stanford's CS109 Probability for computer scientists
+
+### Firstly, for the IBM consulting event
+
+It was an introduction to the different positions that IBM offers in their Consulting department. It was through webex, and many 'IBMers' showed up. From interns to graduates, to permanent employees and employees from different backgrounds. Below are some pics I took of the presentation. 
+
+![image](https://github.com/user-attachments/assets/273c6d07-3c23-42bd-b652-a539b54da04e)
+![image](https://github.com/user-attachments/assets/aa96ffa0-87c2-40ee-9101-cc4ff3081e67)
+
+They presented how IBM thinks of consulting
+
+![image](https://github.com/user-attachments/assets/41c55ddc-ec5d-44a9-b8ca-dd53d9f2485e)
+
+And the importance of putting clients first, embracing new perspectives, open ecosystems, and collaborative approaches, and restless reinvention and innovation. 
+They shared some habits of success to flourish in our careers:
+
+* build client trust
+* collaborate to succeed
+* grow with endless curiosity
+* embrace diverse perspectives
+* innovate with purpose
+* deliver with impact
+
+What skills do we need for consulting? Here is what IBMers think
+
+![image](https://github.com/user-attachments/assets/cc68179a-da18-4d61-96f5-50ccc1696981)
+
+They also shared about their project development process.
+
+![image](https://github.com/user-attachments/assets/3b453e8b-5ad5-4288-a82a-1d0377ed8aa5)
+
+And how they employ modern techniques like Agile using retrospectives, standups, work breakdown, and planning walls.
+
+![image](https://github.com/user-attachments/assets/a9cd80da-8509-4094-986b-e42a7f86f5d8)
+
+And very important: they want all IBMers to want and be excited to learn. Because they have all of these support systems established for continuous learning.
+
+![image](https://github.com/user-attachments/assets/bc9be564-ecf6-4eae-b117-97ec6e3c905b)
+
+We were also introduced to IBM's learning platform called SkillBuild. And invited to join a specially crafted learning path for the Consulting virtual event
+
+![image](https://github.com/user-attachments/assets/604b1263-edaf-4857-90d9-ae47e59a9cac)
+
+Covering the below courses
+
+![image](https://github.com/user-attachments/assets/20da6421-2094-4524-ab98-69785614e81e)
+
+I will put this on my to-do list ^^
+
+And ~ I got a certificate 
+
+![image](https://github.com/user-attachments/assets/f25dbc0c-0b8b-4c3a-8efc-2d2487d4bf63)
+
+### Secondly, about CS109
+
+Lecture 17: Adding Random Variables
+
+Talked about adding two distributions, adding random vars, or convolutions (if you want to sound fancy, as professor Piech said haha). 
+
+![image](https://github.com/user-attachments/assets/49d85f83-05c1-4014-a9c4-ed47ba1e9a8b)
+![image](https://github.com/user-attachments/assets/a7f9016c-cf9b-422d-a8c6-fba08ea33fa0)
+![image](https://github.com/user-attachments/assets/4f268d3f-b7cb-45ae-9f68-11bdd25567aa)
+![image](https://github.com/user-attachments/assets/a83e495f-a383-4cdc-8852-1b6f79daa778)
+
+Sum of 2 uniforms looks like a triangle
+
+What if we add many of any distribution?
+
+![image](https://github.com/user-attachments/assets/33ba5c93-7174-4101-b010-31b7665dd508)
+
+The Central Limit Theorem
+
+The professor's teaching method is so nice, I wish I found this earlier in my journey haha.
+
+Lecture 18: Central Limit Theorem
+
+Finding sample mean and variance, and the variance of the sample mean. We saw how the distribution of the sample means from a population, regardless of the original distribution of the population, will tend to follow a normal distribution as the samples increase.
+
+![image](https://github.com/user-attachments/assets/7c5cf1e7-6d2d-41e3-9224-1e3f5cf091f0)
+![image](https://github.com/user-attachments/assets/5f5975cc-ef6c-4dfa-8416-62fbf5bf0c54)
+
+Lecture 19: Bootstraping and p-values
+
+![image](https://github.com/user-attachments/assets/a1b9a1e1-ef5a-4a54-a18f-fb748a6fbe3c)
+
+Talked about how to estimate the sampling distribution of a statistic - in this case mean and variance. And once we have some values, how to test if they are good estimaters - p-values.
+
+Lecture 20: Algorithmic analysis
+
+Learned about the law of total expectation
+
+![image](https://github.com/user-attachments/assets/8e17dcd2-8765-4b09-ab02-daf6206ad787)
+
+Tomorrow, the lectures begin with maximum likelihood estimation and ML. I am excited to see how professor Chris Piech will introduce these. I really wish I found these lectures at the start of my journey, but just watching them now and confirming my knowledge still feels good. And also the professor is amazing!
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 344ad2867dfb34cdc6f87cc74f25b4ca18713416 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 22:33:09 +0900
Subject: [PATCH 234/638] Update 2024-05-09-day-129.md

---
 _posts/2024-05-09-day-129.md | 129 ++++++++++++++++++++++++++++++++++-
 1 file changed, 127 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-09-day-129.md b/_posts/2024-05-09-day-129.md
index cadf548b6..eb19de0ad 100644
--- a/_posts/2024-05-09-day-129.md
+++ b/_posts/2024-05-09-day-129.md
@@ -1,5 +1,130 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 129) AI with a Scottish accent? + MLE lecture by Chris Piech (Stanford CS109)
+categories: [applying-knowledge,theory,stats]
 ---
+
+## Hello :) Today is Day 129!
+A quick summary of today:
+* I thought to myself: is there an AI that can speak with Scottish accent and does not sound robotic
+* covered Lecture 21 from Stanford's CS109 on Maximum Likelihood Estimation
+
+
+### Firstly, about the Scottish AI
+
+I was looking for videos to practice my Scottish accent (which is awful). After a few, I thought whether there has been an AI model that given text, can generate that text being said in Scottish accent. Started looking, and the best I could find was [this](https://www.narakeet.com/languages/scottish-accent-text-to-speech/), which tries to do it but it maybe sounds 5% (general) Scottish. 
+
+Then, I started looking on huggingface and kaggle, for models developed by solo devs - nothing. I searched TTS models, most are trained on general south UK English, American, Indian, Australian. But none can speak in Scottish. 
+
+Then, I started looking for a dataset. I could not find any either. That is when I tried something.
+
+I found one video on youtube where there is only Scottish accent used - [The Glasgow Uni accent](https://youtu.be/j11YFdS7hwY). I downloaded the audio file of it, and transcribed it myself. 
+
+The short version is: 'Hello? Hi Shanese. How are you? Who is this? It\'s me Kath from school, silly. Why you talking like that? Like what? Like you\'re changing. Don\'t be daft. Anyway, that\'s me done with uni...' (I did the full video but this is just the start). 
+
+However, while transcribing, though able to understand, there were a few words that for the life of me I could not understand, and youtube CC could not help either. 
+
+![image](https://github.com/user-attachments/assets/bb1a3a75-7029-4160-9515-16da0842b772)
+
+I got this for 1. But what if I could do it for many, and actually transcribe it properly. At the time, I was thinking that this requires manual work, because we need to get the Scottish audio transcribed correctly. A general AI model that was not trained on Scottish accent might not be able to transcribe perfectly the Scottish accent (maybe this is something to test).
+
+I need a Scot! - is what my first thought was. So I made a few posts on r/learnmachinelearning and r/scotland proposing my idea to create a dataset of transcribed audio files so that future generations of AI can also speak in Scottish accent. 
+
+Comments came in and there is a lot of good info about organisations and institutions that aim to preserve Gaelic and promote Scottish, info about potential websites I can use for data. And after a few hours, someone interested replied! I will call them 'M'. M shared with me a youtube video where they do short phrases in a general Scottish accent, as an example of what we can use for our dataset. We connected on LinkedIn, and we will have a chat about it this Saturday. Exciting!
+
+Before then, I will look at some research papers about audio file length when training TTS models. My initial thoughts are to get this dataset going. Audio files + transcription, but we will discuss it with M. 
+
+### Secondly, about MLE by Professor Chris Piech
+
+My thoughts while watching: it's kind of scary how great these explanations are
+
+Jokes aside, wow! So far the best MLE explanation I believe was from the book Intro to Statistical Learning, but now I definitely think this lecture beats it. And it's not like if someone new to AI watches that one lecture they will know MLE, watching the previous lectures - it's like building up to MLE, but without knowing it. And *all* the explanations along the way are so clear, and now with MLE too. Anyway ~ below are some screenshots I took.
+
+We begin with an image of how AI, ML and DL fit in together
+
+![image](https://github.com/user-attachments/assets/5a2db663-2d17-4a5c-bfe7-be997a681df8)
+
+Then, is our path to how to get to DL
+
+![image](https://github.com/user-attachments/assets/ef79ede3-2014-437f-844d-36ce9f59e30a)
+
+In that big box there are 3 mini boxes
+
+![image](https://github.com/user-attachments/assets/35ad5532-6573-4396-9f69-8bb0ed86e91e)
+
+The 1st one, from the lectures so far, we learned how to do
+
+![image](https://github.com/user-attachments/assets/bdc1b7f0-c5a3-49eb-8fce-2540b0d63e79)
+![image](https://github.com/user-attachments/assets/6d8e3d8c-a70b-46aa-9a51-b0a057a96173)
+
+We already know how to get unbiased estimateros (the 1st part in the Path image)
+
+![image](https://github.com/user-attachments/assets/689dde4a-231b-41c7-ab06-b48f952bf881)
+
+Now, for the 2nd cube - MLE: the theory behind a general method for choosing numbers in a model
+
+We begin with trying to guess the mu and sigma of some random data points, and try to find the best parameters that result in the data
+
+![image](https://github.com/user-attachments/assets/b00f67f3-713f-42e5-afe0-d60928abb1a1)
+
+We want to find the likelihood of the data resulting from the params.
+We can find those params by using argmax (find the arguments of a function that maximise the likelihood)
+
+![image](https://github.com/user-attachments/assets/393f928d-3f8a-4133-b9c4-06b8dc6bab99)
+
+While max looks for the max value, argmax is looking for the inpuWe are not really interested in how likely is our data, what we’re really interested in is which input of mean and std make the data look most likely.t that results in the max value.
+
+It turns out that the argmax of a function is the same as the argmax of the log(function). And the log helps us with dealing with very very very small numbers and also cleans up the math (as seen later)
+
+The Holy Grail for MLE
+
+![image](https://github.com/user-attachments/assets/f4175303-1494-49e4-845e-c75c1c6e6cb1)
+![image](https://github.com/user-attachments/assets/81bbd5d0-6d19-4391-8b5b-b6404b247bcd)
+
+(we always assume iid, otherwise nothing works)
+
+![image](https://github.com/user-attachments/assets/bcb5be5e-41b8-4cd6-93c8-231d93ce5dbc)
+
+Then we derived the MLE with Poisson, which is just average the data points.
+
+![image](https://github.com/user-attachments/assets/c7687821-cb36-41a0-a3d0-37f65edf8aab)
+
+Then we tried for Bernoulli
+
+![image](https://github.com/user-attachments/assets/c35fda3c-d4ed-425e-bc97-340db28d7b25)
+
+However, this is not differentiable!!!
+
+![image](https://github.com/user-attachments/assets/6c4a008a-0095-44ba-a0d0-0b63b18bed5b)
+
+The right is a continuous version of the left expression
+
+![image](https://github.com/user-attachments/assets/1ec276bd-7a37-4bae-aaeb-e1d71cc80482)
+
+Result: choose your p to be the sample mean (same as Poisson)
+
+Then for a Normal distribution
+
+![image](https://github.com/user-attachments/assets/c468c49d-6a31-438d-9a24-689effc2e31d)
+![image](https://github.com/user-attachments/assets/6037a4fd-861e-44a1-b720-72cf2fc92a1d)
+
+However, the variance is a biased estimator (no n-1)
+
+Finally, MLE with Uniform dist
+
+![image](https://github.com/user-attachments/assets/58f222bf-78d4-4779-a960-d3681e69c9de)
+
+In uniform, MLE chooses alpha to be the smallest in the data, and beta the largest number. But as seen the original params are 0 and 1. This leads to the problem that small samples cause to MLE (our good ol' friend - overfitting)
+
+![image](https://github.com/user-attachments/assets/7b533e13-9104-4b00-bcdb-7fdf323e79ce)
+
+Finally, summarised with the properties of MLE
+
+![image](https://github.com/user-attachments/assets/ce110359-2e35-4741-9da5-338c6a5c207f)
+
+Just amazing. What a great way to introduce MLE, a key concept.
+
+
+That is all for today!
+
+See you tomorrow :)

From d2a605fc911e06088f0cd09448273b24a28b3d3d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 22:40:44 +0900
Subject: [PATCH 235/638] Update 2024-05-10-day-130.md

---
 _posts/2024-05-10-day-130.md | 155 ++++++++++++++++++++++++++++++++++-
 1 file changed, 153 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-10-day-130.md b/_posts/2024-05-10-day-130.md
index cadf548b6..d207eca3c 100644
--- a/_posts/2024-05-10-day-130.md
+++ b/_posts/2024-05-10-day-130.md
@@ -1,5 +1,156 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 130) CS109 - MAP, Naive Bayes, Logistic Regression
+categories: [theory,stats]
 ---
+
+## Hello :) Today is Day 130!
+A quick summary of today:
+* covered Lecture 22: MAP, Lecture 23: Naive Bayes, Lecture 24: Logistic Regression from Stanford's CS109
+
+From 11pm later, to 3am (I believe) is the 3rd day of MLx Fundamentals, and I will share about the 2 sessions tomorrow. 
+
+### Lecture 22: MAP
+
+We saw new method for optimization: gradient descent (or ascent for now).
+
+![image](https://github.com/user-attachments/assets/5862a2c7-7669-4d81-ad15-43daacf49d66)
+
+We want to choose theta, that is the argmax of some function. Seeing the blue curve is nice, but what can we do if we cannot see it ? Imagine we start with a random theta value, we look at the likelihood of that theta, and we can know the derivative at that point, and based on that we can follow along where the derivative is positive. When we reach the top, we actually don’t know we are at the top but when we check the gradient is 0. So we assume we are at the top. In the real world, because we want to think of minimising the loss, we minimise the negative log likelihood.
+
+![image](https://github.com/user-attachments/assets/4e559c56-7eb2-4f29-a471-2a62610f621c)
+
+MLE is great, but it has a problem. It likes to overfit the data it sees - it chooses the params that makes the data it has seen as likely as possible. It does not think about data it might not have seen.
+Probabilities are beliefs, and incorporating prior beliefs is useful. 
+MAP - allows us to incorporate priors for param estimation.
+
+![image](https://github.com/user-attachments/assets/39345e71-6e88-48c0-8121-400978bbe5f3)
+
+Instead of choosing params that maximise the likelihood of the data, MAP (the Bayesian approach) gives us the params that are most likely given the data.
+
+![image](https://github.com/user-attachments/assets/e9d3df0a-37bd-49a2-9f97-288d1577ca7c)
+![image](https://github.com/user-attachments/assets/23f563d3-e0f9-42d5-b2ed-c7120b180f88)
+
+MAP helps to not overfit. However, MLE works for more than just estimating p.
+
+![image](https://github.com/user-attachments/assets/d1b0c84a-7bef-453b-9780-2cf7c3463dcc)
+
+How can we generalize MAP (and maybe do it ourselves too)?
+
+![image](https://github.com/user-attachments/assets/ef58f7e6-2773-416b-bb9e-775b274286d2)
+
+(the normalization term is a constant so it can go away)
+
+![image](https://github.com/user-attachments/assets/9cbf4343-74a9-45d0-bd1b-fa042303287d)
+![image](https://github.com/user-attachments/assets/35bb023e-fd36-4662-b8ea-a4d2fe6baa27)
+
+Doing MAP for Pareto
+
+![image](https://github.com/user-attachments/assets/1a734be3-acdb-4f6d-a076-4de7387a554b)
+
+alpha ~ N(2, 3)
+
+![image](https://github.com/user-attachments/assets/873e3239-91ee-4724-ac6f-f1fef0ca226e)
+
+If we look at the bottom, if we put alpha = 2, that will make the prior term 0, but if we put a alpha larger than 2,  i.e. 3 (larger than the mean of our prior belief), that term will end up a negative number, ths gradient will pull us closer to 2. And if we put an alpha that’s smaller than 2, it will pull us closer to 2 again but from the other direction.
+
+![image](https://github.com/user-attachments/assets/7383ee5d-cb58-4adb-88c3-f76016e086cf)
+
+we are done with the 1st part. :party:
+
+### Lecture 23: Naive Bayes
+
+Firstly, lets see brute force Bayes
+
+![image](https://github.com/user-attachments/assets/fcdac163-257e-4fc7-9bc5-29e3f5724d10)
+![image](https://github.com/user-attachments/assets/7340a512-bbd6-4eb3-85de-3146afe1e188)
+
+What are the params that we need to learn?
+
+![image](https://github.com/user-attachments/assets/58b2f9a8-ec7a-40da-9959-f70e190200e8)
+
+6 in this classification case (there is a bit of redundancy here (if we know Y=0 is theta5, then we know Y=1 is 1-theta5))
+
+How do we estimate these probabilities?
+
+![image](https://github.com/user-attachments/assets/25586f76-262a-4676-80bc-06fef94415aa)
+
+MLE says: just count
+
+MAP says: count and add imaginary trials
+
+we can use the Laplace prior - which is adding 1 imaginary trial for each outcome
+
+Imagine we have got the probs with MAP now, and a test user comes.
+
+![image](https://github.com/user-attachments/assets/c04356cc-082f-4e5f-a52a-26d2fbf16c8d)
+
+The probs dont add to 1? We got rid of the normalization constant!
+
+So the result is y=1.
+
+The above case was when m=1, we have 1 feature. Now, for m=2
+
+![image](https://github.com/user-attachments/assets/bfd2a836-2654-4623-b80d-45e0281871a1)
+
+m=3 ? same thing
+
+![image](https://github.com/user-attachments/assets/a952b064-34ec-4d2f-bd68-2bf4068395d7)
+
+m=100? Same thing. Thats a lot of params.
+
+Big O is 2^m!!! Brute force Bayes lets us down.
+
+Calculating that P(x|y) is too expensive. Instead, lets assume that every xi is independent, so ~
+
+![image](https://github.com/user-attachments/assets/85edc89f-1600-4840-9d31-0a24c55dae0c)
+
+But that assumption is not great. x2 might totally depend on x1 for cases like Kung Fu panda 2 and Kung Fu panda 1.
+
+![image](https://github.com/user-attachments/assets/67de2265-d5ce-43d5-811a-13844aa0d207)
+![image](https://github.com/user-attachments/assets/6218f406-74d6-4741-9103-9c2509e23a33)
+![image](https://github.com/user-attachments/assets/6ccdcf87-9295-4e61-ae76-9badd9e15072)
+
+How to evaluate a Naive Bayes classifier?
+
+![image](https://github.com/user-attachments/assets/be1bb58b-38c9-4ac1-a6f5-90232177a72a)
+
+### Lecture 24: Logistic Regression
+
+Important notations:
+
+![image](https://github.com/user-attachments/assets/fe6e8fa1-b775-4edc-b439-abed737f305a)
+![image](https://github.com/user-attachments/assets/2a7af249-971c-409e-8be7-4495db024796)
+
+In classification we care about P(Y|X).
+In Naive Bayes we can do that, but we make the dangerous assumption of independence. Can we model P(Y|X) directly? Yes - *logistic regression enters the room*
+
+Logistic regression gives us a machine that’s got some params in it (that are movable), you will choose great values for the params such that your machine when it takes in Xs, just outputs things that are close to the prob that Y = 1 (or 0 for the opposite case).
+
+![image](https://github.com/user-attachments/assets/b3a27263-3df6-4209-a53f-aed4747d0b6b)
+
+It takes the inputs and predict whether or not Y = 1. So you want the prob that Y = 1 given your inputs. It takes all the inputs, each goes through this step where it gets weighted by params and each channel gets summed into z, which then is squashed to get a probability with sigmoid.
+Why is there x0 ? People found that if we include an offset, the model works better, to simplify we set x0 = 1.
+
+![image](https://github.com/user-attachments/assets/3726d024-5539-4332-88b3-30ac7f71ac43)
+![image](https://github.com/user-attachments/assets/4a2aed54-adc5-4b52-b84d-74a435d490dc)
+
+How does that look in code?
+
+![image](https://github.com/user-attachments/assets/89982e51-8486-4cbc-8407-e2c345b4c7b5)
+![image](https://github.com/user-attachments/assets/b939a36b-da20-49de-a0ff-b3f0d3623f7a)
+
+There is something beautiful about sigmoid - its derivative is sigmoid multiplied by 1 - sigmoid
+
+![image](https://github.com/user-attachments/assets/25544688-6561-45f2-94a3-e2d2ad2f28e1)
+![image](https://github.com/user-attachments/assets/8b1f5ee6-824b-408d-918b-aa9092aaa359)
+![image](https://github.com/user-attachments/assets/0fb081ab-4c8d-4d2a-ae05-19e34413662a)
+![image](https://github.com/user-attachments/assets/6e7112a2-6ef9-4c57-8ef3-da53a60e047f)
+
+That is all for today. There is only a few lectures left which is a bit sad :( Professor Chris Piech is so amazing. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 9a7a95c4350aacafbc9b7d14b6a22fc43a703552 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 22:49:03 +0900
Subject: [PATCH 236/638] Update 2024-05-11-day-131.md

---
 _posts/2024-05-11-day-131.md | 104 ++++++++++++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-11-day-131.md b/_posts/2024-05-11-day-131.md
index cadf548b6..7e0cb3a8c 100644
--- a/_posts/2024-05-11-day-131.md
+++ b/_posts/2024-05-11-day-131.md
@@ -1,5 +1,105 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 131) Meeting for the Scottish dataset project + CS109 - Deep learning
+categories: [theory,applying-knowledge,stats]
 ---
+
+## Hello :) Today is Day 131!
+A quick summary of today:
+* Met the enthusiast wanting to join for the Scottish AI dataset (over zoom)
+* Covered lecture 25: deep learning from Stanford's CS109
+* Day 3 of MLx Fundamentals
+
+Tonight it is the final Day 4 of the MLx Fundamentals, so hopefully I will include the info about the last 2 classes in tomorrow, or next day's post.
+
+### Firstly, about the meeting
+
+For now I will call them M, but I guess next time I will ask if I can even just use their first name here. Anyway ~ we linked up on linkedin and we decided to meet through zoom this morning. The meeting went for about ~50mins.
+
+Topics we covered:
+
+* general intro of each other
+* a bit of background
+* M told me a bit more about the Scottish accent and how there are 3 main splits - Highlands, West coast and East coast, and that where they are from (West coast) the accent is the least represented - known as Glaswegian. 
+* We talked about clips, what audio we can use, length
+  * after the meeting I read some more research on that topic and a couple of papers use an audio corpus from 2~100 hours. So I guess we do not need *that* big of a dataset to get some kind of a model going. But there are still many questions we need to answer. But for now, we decided the first step to be just to create a 'raw' dataset with audio and transcriptions. 
+* We decided to meet again next Saturday, and I am excited
+* During the week we will transcribe M's videos and also start with some podcasts as per M's suggestion
+
+
+### Secondly about DL, explain by Professor Chris Piech
+
+![image](https://github.com/user-attachments/assets/dac6519a-92f6-42e1-811b-4152e20d2ffc)
+![image](https://github.com/user-attachments/assets/f9c4532e-0583-4967-8f77-44fcc5da7227)
+
+Now, we can think of each neuron as a single logistic regression.
+
+![image](https://github.com/user-attachments/assets/65a5e0ad-dfff-49ba-bdf3-df2c2e516222)
+
+Giving a number to a simple logistic regression will overwhelm it, and it does not work. An image is *a lot* of pixels(inputs). And the interaction between the inputs and and output is much more complicated. What if we stacked these onto each other?
+
+![image](https://github.com/user-attachments/assets/9ffd4123-929c-4397-bf23-05af458c744c)
+
+Every single circle in the hidden layer is a logistic regression connected to the inputs. Once we have this hidden layer, we have a logistic regression that takes the hidden layer as its input and predicts the output. Every circle is a logisic reg that will take the inputs, weight them in its own way with its own params, sum it, squash it and the circle will turn on or off.
+
+Definitions:
+
+Deep learning is maximum likelihood estimation with neural nets. 
+
+A neural network is (at its core) many logistic regression pieces stacked on top of each other.
+Deep learning gets its intelligence from its thetas (aka params)!
+
+So ~ how do we train?
+
+We want to choose thetas that maximise the likelihood of our dataset
+
+![image](https://github.com/user-attachments/assets/a9c18881-5095-4aa1-829d-01a86d014708)
+
+In the pic, we are predicting handwritten 0s and 1s only.
+
+How do we learn again? A DL model gets its intelligence from its thetas. How to find these good thetas? By looking for ones that maximise the likelihood of our train data. How to do that? We use optimization techniques like gradient descent. In order to do gradient descent - we need to calculate partial derivatives of likelihood w.r.t. its thetas.
+
+![image](https://github.com/user-attachments/assets/18f495ec-5656-4966-bea8-51fe73000cad)
+![image](https://github.com/user-attachments/assets/e518d5ee-d348-4148-96c3-4ae42019a051)
+
+It’s all chain rule!
+
+### Finally, a general overview of the material covered on Day 3 of MLxFundamentals.
+
+Schedule was:
+
+![image](https://github.com/user-attachments/assets/ed328871-f76f-467a-9c90-ccec6317ed82)
+
+Yuki M. Asano from University of Amsterdam provided a general intro to deep learning
+
+Covered topics were: history of DL, activation functions (ReLU, ELU, GELU, Softmax), backprop (Andrej Karpathy's video mentioned ^^ ), optimization, CNN, self-supervised learning. Definitely a lot of material for a 2hr limit, but it was a good overall intro lecture.
+
+Karsten Kreis from NVIDIA talked about generative models for vision applications
+
+![image](https://github.com/user-attachments/assets/9f913092-be0f-40fd-85ed-37412b037af8)
+![image](https://github.com/user-attachments/assets/905d4849-2bf5-47a9-a2dd-2de583eb6411)
+![image](https://github.com/user-attachments/assets/f661fd69-3fce-4d3f-a225-d585f8467b1e)
+![image](https://github.com/user-attachments/assets/64286615-e9d3-47da-bd86-58350b4d7e94)
+
+If we add a discriminator (as in GANs) to the above, the quality improves significantly
+
+![image](https://github.com/user-attachments/assets/4d15d572-457a-4d63-b938-0d22af2623ac)
+
+Example
+
+![image](https://github.com/user-attachments/assets/27bf5c84-67ce-450b-9372-911c12e7a9e3)
+![image](https://github.com/user-attachments/assets/2c394d91-125b-431a-8bc8-4e8fb9bb222d)
+
+He also introduced briefly GANs, VAEs, Autoregressive models, energy-based models, and normalizing flows
+
+![image](https://github.com/user-attachments/assets/10060f9a-b997-487d-a503-a317d132fd2d)
+![image](https://github.com/user-attachments/assets/917fdbf4-5e81-4f8b-8f9b-23771befeda6)
+
+Finally, provided a quick intro to video diffusion models (like Sora)
+
+![image](https://github.com/user-attachments/assets/0a92845e-01d9-4198-8ae8-63fb8a8dc241)
+![image](https://github.com/user-attachments/assets/a7386e93-1a0d-4cbe-9a86-75acbe6e7b2a)
+
+That is all for today!
+
+See you tomorrow :)

From e812f37aa8c52fc19cb19132e082fafc1cf68399 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 22:55:02 +0900
Subject: [PATCH 237/638] Update 2024-05-12-day-132.md

---
 _posts/2024-05-12-day-132.md | 58 ++++++++++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-12-day-132.md b/_posts/2024-05-12-day-132.md
index cadf548b6..a14e48308 100644
--- a/_posts/2024-05-12-day-132.md
+++ b/_posts/2024-05-12-day-132.md
@@ -1,5 +1,59 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 132) MLx Fundamentals Day 4(1) + CS109 - Fairness in AI
+categories: [theory,deep-learning]
 ---
+
+## Hello :) Today is Day 132!
+A quick summary of today:
+* last night was the final day 4 of MLxFundamentals
+* covered lecture 26: fairness from Stanford's CS109
+
+Schedule was (time is in BST):
+
+![image](https://github.com/user-attachments/assets/d41f588d-e05e-43f0-b9c8-19967ba4f4d4)
+
+The practical session on GenAI (Vision&NLP) was too late for my local time so I will rewatch the recording and share about it in tomorrow's post.
+
+Firstly, Meng Fang from the University of Liverpool
+
+The lecture was a general intro to LLMs
+
+![image](https://github.com/user-attachments/assets/e976b5d2-8b9b-45c7-bc17-0cb53492c932)
+
+Examples for coding, art, Q&A. Then talked about GPT, and the transformer architecture. 
+Then, some prompt engineering strategies (zero-shot, few-shot, chain-of-thought, self consistency)
+
+![image](https://github.com/user-attachments/assets/4ccc8d6b-230e-414d-8f67-288a36e88a7f)
+
+Then, a brief intro to retrieval-based models. It was mostly based on the ACL 2023 talk which I covered around Day 74.
+
+Finally, language agents. It was a lot of content for just 2 hours, but it was a nice overview.
+
+![image](https://github.com/user-attachments/assets/df0445f8-2f08-43b7-baad-d81555bcc929)
+
+### Secondly, about CS109 Lecture 26 on Fairness:
+
+Professor Piech introduced the topic of fair AI, and how we need to be aware of the problems it can cause if it is biased.
+
+![image](https://github.com/user-attachments/assets/c95e72bb-3332-41e0-9818-ed2b4f11d474)
+![image](https://github.com/user-attachments/assets/87e1955f-99e2-4c9d-b3f5-c84835620bad)
+
+There are two philosophical views on fairness
+* Procedural (fairness through unawareness turns out to be hard)
+  * focuses on the decision-making or classification process, ensures that the algorithm does not rely on unfair features
+* Distributive
+  * focuses on the decision-making or classification outcome,  ensures that the distribution of good and bad outcomes is equitable
+
+![image](https://github.com/user-attachments/assets/ba6b952a-fc34-44f7-8814-084241ff115f)
+![image](https://github.com/user-attachments/assets/8d0050f9-7fab-4b18-b477-c6c19cd7095d)
+![image](https://github.com/user-attachments/assets/6d3055a1-4dba-450f-a011-a724d3634407)
+
+Fairness through awareness still fails to capture certain aspects
+* if the classifier is significantly less good at identifying candidates (i.e. for a surgery in a minority group, relative to the data), the candidates accepted might have worse outcomes, leading to future bias
+* quality of service disparity might then lead to an allocation disparity (looking at differences in groups and assume there is a stronger differences than there are between the groups, which can change people's mindsets to think that one group is stronger than the other)
+
+
+That is all for today!
+
+See you tomorrow :)

From 6b6f92ea4c4fba2ee5830b467ba18c522c1b2064 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Wed, 7 Aug 2024 22:59:52 +0900
Subject: [PATCH 238/638] Update 2024-05-13-day-133.md

---
 _posts/2024-05-13-day-133.md | 123 ++++++++++++++++++++++++++++++++++-
 1 file changed, 121 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-13-day-133.md b/_posts/2024-05-13-day-133.md
index cadf548b6..652d7eade 100644
--- a/_posts/2024-05-13-day-133.md
+++ b/_posts/2024-05-13-day-133.md
@@ -1,5 +1,124 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 133) Gathering data for the Scottish dataset project + Factor analysis + Grokking ML + MLxFundamentals Day 4 (2)
+categories: [theory,deep-learning,applying-knowledge]
 ---
+
+## Hello :) Today is Day 133!
+A quick summary of today:
+* started putting together some audio clips + transcription for the [Scottish dataset project](https://50daysml.blogspot.com/2024/05/day-129-ai-with-scottish-accent-mle.html)
+* saw how eigenvalues play a role in factor analysis
+* started reading Grokking Machine Learning on Manning.com
+* finished [MLx Fundamentals](https://www.oxfordml.school/fundamentals)' final session
+
+### Firstly, about the audio data
+
+My Scottish partner for this project has recorded various phrases in Glaswegian in the past and uploaded them to youtube. Today I did 4 of the 10 videos. 
+
+To cut the clips I ended up using an app called VideoPad, and even though it is a paid app, it allows me to just cut an audio clips in smaller pieces and save them as new files. 
+
+This is a sample audio waveform of one of the 4 videos
+
+![image](https://github.com/user-attachments/assets/e4ba1d62-ac6f-4eff-a0c5-966a9d1b0e91)
+
+What I did was, make short clips around each expression. I am not sure what these waves are called when there is speech. So for example from the above, I ended up with 36 clips, and uploaded them all (along with the other 3 videos' audio clips) to our project's drive. And the total amount we have so far is 4.75 minutes.  
+
+### Secondly, today I read about eigenvalues' role in factor analysis
+
+In my stats class at uni, we learned about factor analysis, and at the end of the chapter I saw the word eigenvalues, and I am glad because once again I will see their real world impact (after my dive into multicollinearity). 
+
+Firstly, about factor analysis, here are the results after using unstandardized variables
+
+![image](https://github.com/user-attachments/assets/0eccd38a-2aea-4e7d-b4b9-d1dec0ec6f20)
+
+And after standardization
+
+![image](https://github.com/user-attachments/assets/47e30f74-f69b-42b3-87ac-8367a7efcda3)
+
+Why standardize?
+
+* result interpretability
+* helps with linearity
+* treats variables equally
+
+Where are the eigenvalues? The overall under each loading. 1.981 and 1.008. Which are sum of the square of each of the 3 values above it. 
+
+![image](https://github.com/user-attachments/assets/07fe6713-40b1-447c-8b6f-d68ea1aec256)
+
+To interpret this, we take the 1st row and F1: 0.00089222 - this is F1 accounts for 0.0089% of the variance in Y1 which is Finance, while F2 accounts for 99.90%. And in total, the 2 factor space accounts for 99.99% of the variance in Y1 Finance. 
+
+ The eigenvalues can help to determine which factors to keep (i.e. using scree plots).
+Love it when I see the math I studied for ML being used in practice, and *where* it is used.
+
+### Thirdly, about Grokking ML
+
+I decided to subscribe to manning.com and the 1st book I decided to read was Grokking ML (as it is one of the most recommended and popular ones). Today I managed to read the 1st 4 chapters(what is ML, types of ML, linear regression, optimization), and I can definitely see why it is popular for beginners, and I am excited to keep reading. 
+
+### Finally, the last session from MLxFundamentals was delivered by Wenhan Han a PhD candidate from TU Eindhoven.
+
+It was about loading and using an LLM, and a diffusion model
+
+Some interesting bits from both parts are:
+
+Question to an LLM:
+
+How many kinds of human beings are there in the history?
+
+We saw the top answers from the model. 
+
+![image](https://github.com/user-attachments/assets/825a4c31-1f23-40ad-87ef-527517ffe9a9)
+
+Various prompt strategies
+Zero-shot
+
+![image](https://github.com/user-attachments/assets/bb3bc5d6-fe04-49ab-8361-12cc93a57eff)
+
+Few shot
+
+![image](https://github.com/user-attachments/assets/28e0cbad-5f40-4cb4-a8f6-f7a38c915e2d)
+
+Chain-of-thought
+
+![image](https://github.com/user-attachments/assets/e5f39894-ac9f-4425-af28-2d499d52a1d1)
+
+Then, how to finetune an LLM with 'unsloth'
+
+![image](https://github.com/user-attachments/assets/9e641220-6cdf-4722-bea7-80b5e68d8440)
+
+We add LoRA adapters
+
+![image](https://github.com/user-attachments/assets/24f4cbd3-3e4a-4ce3-b43f-214abbd5a2b8)
+
+Prepare the data
+
+![image](https://github.com/user-attachments/assets/70b57dff-4493-44ec-9308-6e5116cb96ef)
+
+Data was from huggingface
+
+![image](https://github.com/user-attachments/assets/32f629b3-4a2c-4d80-a07d-dea72d53be6e)
+
+Train the model
+
+![image](https://github.com/user-attachments/assets/a0d3ac66-04b2-4a23-a779-8f72348b650f)
+
+And then inference
+
+![image](https://github.com/user-attachments/assets/e510e563-d72b-411d-af74-e6b5f409d9a2)
+
+For finetuning I need to try unsloth by myself. For some part of the tutorial, an openai api key was required which was unfortunate because I do not have one, so I just watched that part. 
+
+As for diffusion models
+
+![image](https://github.com/user-attachments/assets/ee417c96-8777-427e-afa0-cbe420160585)
+
+I saw the library 'diffusers' was used which I did not know.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgflXnxK36P1DZbSvRH14jYJ-jcomrsGNeMrV-J1lPczAqwZJlIIC5oL-xlSMxVJGz-wMVEhLZrxD9Cko_f2xM2CfWDrHyOcDNFF5sL1-RSYTywH_poMSNtJ8F4pMsllY1ECPuH2pvn8J-JlW6PBA39ILlqCrj_lhYWbowhl_zXloy9qUZorQOltcolta08)
+
+And I played around with it a bit, a picture I tried to get and failed was 'a horse riding an astronaut'
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhv9fWL3JUd3w50xfBAE5dTnd86eOly7-gX2CF1dSeqkFeoK1s2pnvTKRqaSXwGI4BRc1hw2905wcrLiiSOkzT3sfrl06hpAmfw4FGvSllGbIrXAlz8dxCz7boBaSkEMiIPZoP8PtuqWmSKtppAW7drYeEv-2zJopeQkD23Uc0A3mBIqybFgVV7pbFuNMr4)
+
+That is all for today!
+
+See you tomorrow :)

From 8792dad7279bb20b124470607d53a54fb19b25cb Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 10:54:37 +0900
Subject: [PATCH 239/638] Update 2024-05-14-day-134.md

---
 _posts/2024-05-14-day-134.md | 152 ++++++++++++++++++++++++++++++++++-
 1 file changed, 150 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-14-day-134.md b/_posts/2024-05-14-day-134.md
index cadf548b6..aac90ff2b 100644
--- a/_posts/2024-05-14-day-134.md
+++ b/_posts/2024-05-14-day-134.md
@@ -1,5 +1,153 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 134) Finished CS109 + Scottish dataset project + Started MLOps zoomcamp by DataTalks club
+categories: [applying-knowledge,theory,nlp]
 ---
+
+## Hello :) Today is Day 134!
+A quick summary of today:
+* covered the last 2 lectures of Stanford's CS109
+* processed 4 more videos for the Scottish accent dataset
+* started MLOps zoomcamp by DataTalks club
+* watched a nice video comparing the roles of data scientist vs AI engineer
+
+### Firstly, on lecture 27: advanced probability and lecture 28: future of probability from Stanford's CS109
+
+What a course! Sad it is over. Chris Piech - what an extraordinary professor! 
+
+Both lectures were more or less an overview of the course and professor Piech's hopes for the future of probability and his hopes for his students.
+
+Lecture 27
+
+![image](https://github.com/user-attachments/assets/3db79aa6-4b84-4d65-956a-cb053d6804c5)
+
+Around 2013, autograd was introduced for the first time and this allowed for backprop to be automated, instead of manual calculations. This was one of the reasons for deep learning to explode.
+
+The professor also talked a bit about diffusion models
+
+![image](https://github.com/user-attachments/assets/3d3a5e98-6e92-4657-a763-8f9e3caf0cc2)
+
+#1 remove the noise
+
+#2 predict where the noise is
+
+And a bit about language models (at the time of recording ChatGPT had just come out). And he introduced a basic version of RNN/LSTM. 
+
+![image](https://github.com/user-attachments/assets/76075ec5-a935-4289-b2b6-ec02f15696e4)
+
+Lecture 28
+
+Besides an overview of what was covered throughout the previous 27 lectures, he talked about potential usage of AI in education and how it can help provide feedback to students, and in medicine. 
+
+### Secondly, about the Scottish dataset project
+
+Today I cut videos 'Scottish phrases 5-8' ([4 videos](https://www.youtube.com/@Cannaekickaroonhere/videos)). And now the total time of the audio clips we have in the dataset is 10 minutes :party: We are improving slowly. 
+
+### Thirdly, started MLOps zoomcamp by DataTalks club
+
+Last night was the opening day for the MLOps camp that I signed up for. It is one (if not the) most popular open courses for MLOps, so I was excited to start it. Well today I covered module 1 and its homework. Below are notes from the intro lecture + module 1 + the homework. 
+
+Intro lecture
+
+The course will cover the following topics
+
+![image](https://github.com/user-attachments/assets/9af4e154-735c-4254-ba3e-b10f4508c675)
+![image](https://github.com/user-attachments/assets/84a991f1-6183-48c1-8bad-42a4cb83ac75)
+
+Module 1
+
+I think the coolest thing was learning about the different MLOps maturity levels (introduced by [Microsoft](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/mlops-maturity-model))
+
+Level 0: no MLOps
+
+* no automation
+* all code in jupyter
+* good for experiments
+
+Level 1: DevOps, No MLOps
+
+there are experienced devs helping the data scientists
+some automation
+releases are automeated
+unit & integration tests
+CI/CD
+ops metrics
+no experiment tracking
+no reproducability
+data scientists separated from engineers
+
+Level 2: Automated training
+
+* training pipeline
+* experiment tracking
+* model registry
+* low friction deployment
+* data scientists work with engineers
+* good level if we have 2-3 ML cases
+
+Level 3: Automated deployment
+
+* easy to deploy models
+* pipeline is: data prep, model traning, model deployment 
+* A/B tests between models
+* some model monitoring
+
+Level 4: Full MLOps automation
+
+* automatic training
+* automatic retraining
+* automatic deployment
+* A/B tests
+* approaching a zero-downtime system
+
+Not all orgs need to be on level 4. Level 3 is still fine because we can have a human making that final decision whether a model goes live. So we need to judge what level is best for a particular project.
+
+The homework
+
+I uploaded all my work to my [github](https://github.com/divakaivan/MLOps-camp).
+
+The homework followed the module 1 materia on creating a linear regression model that predicts the taxi travel time from point A to point B. It involves some data preprocessing, model creation and exporting the model with pickle. 
+
+Q1: loading the data and how many columns there are
+
+![image](https://github.com/user-attachments/assets/d5fc2c7b-9889-4d30-8d2b-d856df94e49d)
+
+Q2: What's the standard deviation of the trips duration in January?
+
+![image](https://github.com/user-attachments/assets/05cec8df-1107-4c9b-98b6-71f193fa641b)
+
+Q3: What fraction of the records left after you dropped the outliers?
+
+![image](https://github.com/user-attachments/assets/5feef4e6-714f-452f-93f0-7cfa755d584c)
+
+Q4: What's the dimensionality of the matrix (number of columns) after using DictVectorizer?
+
+![image](https://github.com/user-attachments/assets/317e30b5-773a-4c91-8234-52e039021c08)
+
+Q5: What's the RMSE on train?
+
+![image](https://github.com/user-attachments/assets/dc18c4c0-2ffc-46d6-8ab1-acd07a4369ae)
+
+We can interpret this as being 7.6 minutes wrong on average in our ride duration predictions.
+
+Q6: What's the RMSE on validation?
+
+After loading the val dataset (data from February 2023, train is January 2023), I got the result
+
+![image](https://github.com/user-attachments/assets/edfe6bbd-b29c-4819-8015-1171182aade4)
+
+Overall, pretty satisfied, and excited for the future modules and homeworks. 
+
+
+
+### Finally, the comparison between a data scientist and an AI engineer
+
+[This](https://youtu.be/Vxw0nE1qfZc) IBM video was in my recommended on youtube so I clicked on it to see. And the below pic is the final explanation.
+
+Some abbreviations: FM: foundation model, FE: feature eng, CV: cross-val, HPT: hyperparam tuning, PEFT: param efficiant finetuning. Of course the below is not static, DS might work on prescriptive cases, and AI eng can work with structured data.
+
+![image](https://github.com/user-attachments/assets/74564126-a8b4-4eb4-9665-8c5931a21acd)
+
+That is all for today!
+
+See you tomorrow :)

From 858f49ec868f306881e0e69dfcd88ef22ccfda09 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:05:20 +0900
Subject: [PATCH 240/638] Update 2024-05-15-day-135.md

---
 _posts/2024-05-15-day-135.md | 233 ++++++++++++++++++++++++++++++++++-
 1 file changed, 231 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-15-day-135.md b/_posts/2024-05-15-day-135.md
index cadf548b6..3385fff37 100644
--- a/_posts/2024-05-15-day-135.md
+++ b/_posts/2024-05-15-day-135.md
@@ -1,5 +1,234 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 135) Going deeper into MLOps
+categories: [mlops,applying-knowledge,theory]
 ---
+
+## Hello :) Today is Day 135!
+A quick summary of today:
+* covered [module 2](https://github.com/DataTalksClub/mlops-zoomcamp) of the mlops-zoomcamp by DataTalks club about experiment tracking and model management
+* cut 2 more videos for the Scottish dataset project
+
+### Firstly, about using mlflow for MLOps
+
+Maybe this is because I am starting to learn about MLOps for the 1st time and I don't know other tools, but WOW mlflow is amazing. Below are my notes from the module 2 lectures. Full code on my [github](https://github.com/divakaivan/MLOps-camp/tree/main/Module%202) repo.
+
+First, some important concepts
+* ML experiment: the process of building an ML model
+* experiment run: each trial in an ML experiment
+* run artifact: any file that is associated with an ML run
+* experiment metadata
+
+What's experiment tracking?
+
+* the process of keeping track of all the relevant info from an ML experiment (could include source code, environment, data, model, hyperparams, metrics, other - these can vary)
+
+Why is experiment tracking important?
+
+* reproducability
+* organization
+* optimization
+
+Why can't we just use an excel spreadsheet?
+
+* error prone
+* no standard format
+* visibility and collaboration
+
+_MLflow enters the stage_
+
+![image](https://github.com/user-attachments/assets/09d776ec-9a73-4a19-9c0d-91fb1abd832b)
+![image](https://github.com/user-attachments/assets/590e6179-c361-41c4-9c8d-09a9fa4d957c)
+
+We can spin up mlflow with: mlflow ui --backend-store-uri sqlite:///mlflow.db
+
+And the sqlite db will keep artifacts and metadata inside of it. 
+To create tracking:
+
+![image](https://github.com/user-attachments/assets/03206a8a-6b0f-4f5c-af14-0cf0186fa8f2)
+![image](https://github.com/user-attachments/assets/a84360c8-323a-438d-9f5d-8cf352bbd974)
+
+With the above we can log a run, its params, metric, more info. 
+In the UI, we see
+
+![image](https://github.com/user-attachments/assets/163b2de3-5760-4ac2-9f2f-ec47666adc30)
+![image](https://github.com/user-attachments/assets/c44482d7-cf6d-4aeb-b192-2f214ce57230)
+
+Run another model with alpha=0.1, and we can compare the two
+
+![image](https://github.com/user-attachments/assets/f1b09289-f217-4d15-8f12-e4dbf55d5fab)
+
+Next, doing hyperparam optimization using xgboost and pyperopt
+
+![image](https://github.com/user-attachments/assets/3384c32b-3d90-423e-b7c6-a1da9b6fd3db)
+
+we need to define an objective
+
+![image](https://github.com/user-attachments/assets/b239e87a-0c97-4e3e-8474-a3c937e884e8)
+
+Search space and then run
+
+![image](https://github.com/user-attachments/assets/e3f6e4e4-be84-4426-a4bd-6a7c823e1046)
+
+After this ran (for around 30 mins), in mlflows’ ui we can visualise all the runs that use the xgboost model
+
+![image](https://github.com/user-attachments/assets/c306e7ed-dcb4-426b-808f-5b84fd1b79c6)
+
+it also shows which hyperparam combinations lead to which results
+
+![image](https://github.com/user-attachments/assets/f9ef6908-4302-4e60-a6b6-23ac9dee7ef2)
+
+For example if we see a scatter between rmse and min_child_weight we ca see somewhat of a pattern where smaller min_child_weight results in smaller rmse
+
+![image](https://github.com/user-attachments/assets/117e35c2-a7f4-480e-affb-6ad7f79aa048)
+
+we can also see a countour plot where we can see 2 hyperparams and their effect on the rmse
+
+How do we go about model selection?
+
+A naive way would be to choose the model with lowest rmse
+
+![image](https://github.com/user-attachments/assets/b0620d28-96e8-4db7-a9c8-f83f7068f7f3)
+
+and after clicking on it we can check more info about it.
+
+![image](https://github.com/user-attachments/assets/316d21f4-0749-4134-a0ed-08a9412394cb)
+
+And we can make further judgements like run duration, data used, etc.
+Instead of writing lines of what to log, we can use autolog, and when we run the model with the best params from the hyperparam search
+
+![image](https://github.com/user-attachments/assets/443eb924-2d72-41f2-99d1-bcf90634a611)
+
+we get a bit info:
+
+![image](https://github.com/user-attachments/assets/b0f4d7fc-3256-4bce-92b8-82dec8de3875)
+![image](https://github.com/user-attachments/assets/0f3e7b79-e7c7-43f5-b9af-5f807093002b)
+![image](https://github.com/user-attachments/assets/3294b909-9e57-4c2a-a823-db52450234de)
+
+In model metrics:
+
+![image](https://github.com/user-attachments/assets/995c6c2a-cfb1-4e2a-b990-f3ad5b64d333)
+
+We have more content in the ‘Artifacts’ tab as well
+
+![image](https://github.com/user-attachments/assets/0b5ff4a8-8e96-499e-9996-d3d98a26b16b)
+
+Like model info, dependencies, how to run the model
+
+![image](https://github.com/user-attachments/assets/99cf93ec-a38f-402b-9402-ddaec087260e)
+
+Next, I learned about model management
+
+![image](https://github.com/user-attachments/assets/4847b322-ba76-4c49-a3ab-1b38b1074c34)
+
+What's wrong with using a folder system?
+
+* error prone
+* no versioning
+* no model lineage
+
+To begin with, we can use to save the model in the artifacts tab
+
+![image](https://github.com/user-attachments/assets/3817e9d4-94f0-4e21-b495-17db80dba674)
+
+And when we run the code again, we get
+
+![image](https://github.com/user-attachments/assets/38d481e2-845c-46a7-9ccc-8369f5ef717d)
+
+Instead of just getting the bin file, we can get better logging using
+
+![image](https://github.com/user-attachments/assets/5392f373-8bd2-4deb-b5c4-4603ca635780)
+
+Which results in
+
+![image](https://github.com/user-attachments/assets/16895f8c-2eed-42da-9938-1810eca6827a)
+
+We can also log the DictVectorizer used in the data preprocessing step, and get it from mlflow
+
+![image](https://github.com/user-attachments/assets/34ea8953-f5fc-4294-82dd-4e6042d65116)
+![image](https://github.com/user-attachments/assets/536c929b-e4ca-431d-98b5-a4be96060cae)
+
+Using the saved model for predictions mlflow provides this sample code when a model is logged in the artifact tab
+
+![image](https://github.com/user-attachments/assets/3255932f-d6ee-432f-85ed-33e972f80a91)
+
+If we use the sample pandas code to load the model, we can check out the model
+
+![image](https://github.com/user-attachments/assets/73d56c87-aba0-4cad-892d-81f2ca22c2eb)
+
+Since it is a xgboost model, we can load it with mlflow’s xgboost and then use it as normal
+
+![image](https://github.com/user-attachments/assets/00bc7504-8c19-4b63-a35e-b91509bdc821)
+
+### Next, model registry
+
+Imagine a scenario where a DS sends you a developed model and asks for me to deploy it. My task is to take it and run it in prod. But before I do it, I should ask what is different, and what is the purpose. What are the needed dependencies and their versions which are needed for the model’s environment 
+
+![image](https://github.com/user-attachments/assets/b624fb6f-fde5-45e5-aa3b-8f60b0a901e3)
+
+This email is not very informative, so I might start to look through my emails for the previous prod version. If I am lucky enough someone that was in my place before me or a DS, has put the info of the current prod version in a tool for experiment tracking (like mlflow).
+
+![image](https://github.com/user-attachments/assets/ef66c815-4529-4068-a8b3-aff822421fce)
+
+The pictures above with lots of model runs is the tracking server. A model registry lists models that are in different stages, production and archive. To do actualy deployment, we need some CI/CD. When analysing which model is best for production, we can consider model metrics, size, run time (complexity), data. To promote a model to the model registry, we can just click on the ‘register model’ button in a model’s artifact tab.
+
+![image](https://github.com/user-attachments/assets/ee377027-601d-4b4f-ada2-323ed4f87b9b)
+
+We can go to the models tab now in mlflow
+
+![image](https://github.com/user-attachments/assets/a3f22094-3bb7-437f-8b1f-cdc4d41daaa3)
+
+and see the registered models (I ran and registered a 2nd model)
+
+![image](https://github.com/user-attachments/assets/39fd6a98-115d-47fd-b069-8f42b737c2d1)
+
+In the newest version of mlflow, they have removed the staging/deployed/archived flags, and it is more flexible, so I assigned champion and challenger (custom) aliases
+
+![image](https://github.com/user-attachments/assets/12062937-add7-42fc-a05e-e24d1d7be64a)
+
+Now the deployment engineer (assuming we understand the meaning of the aliases) can decide what to do. All of the above actions can be done through code as well, no need to use the UI (I learned some of the functions and used them in the github notebook and using the mlflow client docs)
+
+Next, I learned about mlflow in practice
+
+![image](https://github.com/user-attachments/assets/3c04d71d-08aa-4aff-8585-ac96f92db5f4)
+
+1 - no need to use mlflow, saving locally is fine
+
+2 - running mlflow locally would be enough. Using model registry would be a good idea to manage the lifecycle of the models. But it is not clear if we can just run it locally or need a host.
+
+3 - MLflow needed. We need a remote tracking server. Also model registry is important because there might be different ppl with different tasks.
+
+![image](https://github.com/user-attachments/assets/6853f9f1-2c16-4b62-94ee-c812aacfd1e8)
+
+For the 3rd scenario I wanted to follow along and create an AWS instance, but I don’t have an account and for some reason my card keeps getting rejected. So I guess I will have to do it some other time 😕
+
+![image](https://github.com/user-attachments/assets/87ec114b-a520-4584-82ae-0de84e2e3158)
+![image](https://github.com/user-attachments/assets/5adb4202-7b5b-461e-bfec-265a76170507)
+![image](https://github.com/user-attachments/assets/6cc435f5-a2fa-4f5d-a3b4-8e4bd3a172e0)
+
+[This website](https://neptune.ai/blog/mlops-tools-platforms-landscape) shows comparison between mlflow and paid/free alternatives.
+
+
+Secondly, about the Scottish dataset
+
+Today I finished the last 2 videos from my collaborator
+
+![image](https://github.com/user-attachments/assets/3c23565c-5c93-47e6-ae9f-f6b735016b8f)
+
+And now the total audio length in our dataset from all 10 videos is 770 seconds. I need to find a better way to store our audio files and the dataset itself. It all lives on my google drive at the moment. 
+
+I also added some other metadata for each clip
+
+![image](https://github.com/user-attachments/assets/88643658-d1f6-450b-9bad-2a805e6125aa)
+
+And this is a describe of the length_seconds column
+
+![image](https://github.com/user-attachments/assets/8f03ffa7-9a8c-486c-96d2-3c9992887fc0)
+
+Tomorrow and Friday is [AWS Summit Seoul 2024](https://aws.amazon.com/ko/events/summits/seoul/), and there are a lot of panels each day (2 days in total). So I will share my experience and attended panels. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 235927cf657d4aaea0af080d50dad643776f4124 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:06:00 +0900
Subject: [PATCH 241/638] Update 2024-05-14-day-134.md

---
 _posts/2024-05-14-day-134.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-05-14-day-134.md b/_posts/2024-05-14-day-134.md
index aac90ff2b..ce3cb6384 100644
--- a/_posts/2024-05-14-day-134.md
+++ b/_posts/2024-05-14-day-134.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 134) Finished CS109 + Scottish dataset project + Started MLOps zoomcamp by DataTalks club
-categories: [applying-knowledge,theory,nlp]
+categories: [applying-knowledge,theory,nlp,mlops]
 ---
 
 ## Hello :) Today is Day 134!

From 4b442026dd4c29d2289dd589fcd7131489f301cc Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:09:45 +0900
Subject: [PATCH 242/638] Update 2024-05-16-day-136.md

---
 _posts/2024-05-16-day-136.md | 53 ++++++++++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-16-day-136.md b/_posts/2024-05-16-day-136.md
index cadf548b6..eb1c02851 100644
--- a/_posts/2024-05-16-day-136.md
+++ b/_posts/2024-05-16-day-136.md
@@ -1,5 +1,54 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 136) AWS Summit Seoul Day 1
+categories: [cloud]
 ---
+
+## Hello :) Today is Day 136!
+A quick summary of today:
+* Attended day 1 of AWS summit Seoul and sat in the 'banking, finance and fintech' talks - everything was related to AI (not surprisingly haha)
+
+![image](https://github.com/user-attachments/assets/3599f969-08da-4337-83ed-cdfaec72651a)
+![image](https://github.com/user-attachments/assets/6f2638fb-6625-4e89-a9e2-1ca6da202025)
+
+It was all in Korean, so below I will try to give a general overview of the attended sessions + some extra activities that I did. 
+
+Something funny and interesting. There was a booth where you took a picture and your face was put on a medieval/pre-historic/renaissance figure. It is an event in Korea so I was curious whether it can recognise white people faces, and below is supposed to be me
+
+![image](https://github.com/user-attachments/assets/ede60302-9796-46ba-8463-867e8d657851)
+
+Haha. Interesting ^^
+
+There was also a football kick booth where I queued for around 30mins to shoot twice and to try to kick and score with the highest speed (I placed rank 24 with 72km/h kick on target). They also had a model that takes a picture and puts a football jersey on the person
+
+![image](https://github.com/user-attachments/assets/d4b34497-3f60-4793-b1ff-76a80a2cc252)
+
+Next, about the sessions. 
+
+They were all in Korean (the slides as well) and even though I took pictures putting them here would be no good
+
+![image](https://github.com/user-attachments/assets/b0144aed-04c2-474b-8a74-a40cf1ab8d5b)
+
+It says finance and fintech
+
+There are also many boots with various company and got plenty of free goodies
+
+![image](https://github.com/user-attachments/assets/0c8c209b-25ee-4c06-ae47-98441fa40286)
+
+The sessions I attended were:
+
+1. How Hyundai card uses AWS' big data platform
+2. Using the cloud in health insurance businesses
+3. (From a different topic) AI and ML in manufacturing
+4. How Shinhan Bank uses the cloud
+5. How KBank moved to and uses the cloud
+
+Some covered topics were: on-premise vs the cloud, AWS devops services, MLOps services, data storage, architectures (picture), customer services through AWS, AI banking, 6 checks whether a service is well-architectured (operating excellence, protection, safety, efficiency, cost reduction, maintenance), SageMaker.
+
+There is plenty to learn about AWS. I felt like a kid today that had just heard the names of some of AWS' services like SageMaker.
+
+Tomorrow is Day 2. Looking forward to it!
+
+That is all for today!
+
+See you tomorrow :)

From 280af6c842f781fc5e7327e37da15478b654a926 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:23:06 +0900
Subject: [PATCH 243/638] Update 2024-05-17-day-137.md

---
 _posts/2024-05-17-day-137.md | 180 ++++++++++++++++++++++++++++++++++-
 1 file changed, 178 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-17-day-137.md b/_posts/2024-05-17-day-137.md
index cadf548b6..cc2a3157a 100644
--- a/_posts/2024-05-17-day-137.md
+++ b/_posts/2024-05-17-day-137.md
@@ -1,5 +1,181 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 137) AWS Summit Seoul Day 2
+categories: [cloud]
 ---
+
+## Hello :) Today is Day 137!
+A quick summary of today:
+* attended 'data analysis' and 'business tech' sessions during Day 2 of AWS Summit in Seoul
+
+![image](https://github.com/user-attachments/assets/8fc01acf-1f4f-487a-8ebc-fbcf0fe72f8b)
+
+I tried to take more notes with my laptop during the session today.
+
+
+
+### First session - 'Data strategy for successful GenAI on AWS'
+
+A GenAI app is like an iceberg. 
+
+Under the water it consists of among other things data storage, database, data lake, data manipulation, data governance. 
+
+The session introduced us to RAG, and how we can either use an out-of-the-box language model, pre-train a model, or train a model from scratch. 
+
+As for a vector search AWS offers plenty of options: Amazon OpenSearch Service, OpenSearch Serverless, Aurora PostgreSQL, RDS for PostgreSQL, DocumentDB, DynamoDB via zero ETL, MemoryDB for redis, Neptune.
+
+As for a database, there are plenty of structured and nonstructured dbs. 
+
+Amazon Bedrock - allows for using out-of-the-box models for GenAI applications. It gives us the ability to use a simple API for a model, allows for model customizing, RAG, Agents, protection, privacy and security. 
+
+If we use Amazon Bedrock, the vector databases we can use are Amazon OpenSearch Serverless, Redis Enterprise Cloud, Pinecone, Amazon Aurora, MongoDB.
+
+### Second session - 'How to best use vector databases on AWS'
+
+Start with a pdf - chunk - embed with Amazon Tital - store in Amazon Aurora PostgreSQL (or other db)
+
+![image](https://github.com/user-attachments/assets/4a200e2f-6085-4af8-9de7-641268b3b3cd)
+
+Things we need to look for when embedding. 
+
+* embedding time
+* size
+* query time
+
+Questions to ask when choosing a db system
+
+* for my workflow, which vector storage is the best?
+* how much data am I going to store?
+* among storage, efficiency, connectiviy, cost which is important to me?
+* what are the pros and cons of the different indexing, query time and schema designs?
+
+What is pgvector?
+
+* offers open source similarity search, vector storage, indexing and metadata
+
+From pgvector's github it has 2 strategies for indexes: 
+
+HNSW: An HNSW index creates a multilayer graph. It has better query performance than IVFFlat (in terms of speed-recall tradeoff), but has slower build times and uses more memory. Also, an index can be created without any data in the table since there isn’t a training step like IVFFlat.
+IVFFlat: An IVFFlat index divides vectors into lists, and then searches a subset of those lists that are closest to the query vector. It has faster build times and uses less memory than HNSW, but has lower query performance (in terms of speed-recall tradeoff).
+
+
+
+### Third session - 'Successful data stories from the business'
+
+Presented many cases (companies) where after integrating Snowflake into their operations, the companies reduced cost, increased customer satisfaction, profits, amount of data, customer service. Cases like KFC, AT&T, Pfizer, SCANIA.
+
+![image](https://github.com/user-attachments/assets/99e7c03f-d229-4f42-a2bb-515a71efa28f)
+
+Below is what kind of services are offered
+
+Basicaully, it was a sales pitch for snowflake haha
+
+
+### Fourth session - Improving GenAI search performance through Amazon's OpenSearch
+
+What is Amazon OpenSearch? The most optimized search and log solution
+
+* offers similarity search
+* seamless data analysis and visualisation
+* connects data sources
+* low cost solution
+* easy of use
+
+Introduced vector search and TF-IDF scoring, and the process of giving a PDF, chunking, embedding, saving, finding similar to a query passages and retrieving them.
+Using the ANN algorithm - Hierarchical navigable small world (HNSW), and IVF
+
+Showed an example of semntic search:
+
+1. external model hosting and service connection
+2. creating an OpenSearch document gathering using NN search pipeline
+3. search request through the API gateway
+4. call the backend through the API gateway
+5. search the backend for related documents and after give them to the client
+
+![image](https://github.com/user-attachments/assets/a7dc8a4a-b036-422e-96d4-c454e69e0462)
+
+We saw a demo on using Amazon OpenSearch and Bedrock for text2sql - similar to my [chat with your PDFs](https://github.com/divakaivan/RAG_with_your_PDF_for_free_in_colab) but of course on a much higher level using OpenSearch and using streamlit. Also every query, response and extra metadata gets saved in the backend.
+
+Below is what happens on a higher level
+
+![image](https://github.com/user-attachments/assets/3158dd36-03ac-4221-ba46-7eb23603ade0)
+
+### Fifth session (from the business tech sessions because there is no data analysis session in this time slot) - Easy data lake creation using AWS
+
+What is a data lake? A place that stores all kinds of data (structured and unstructured)
+
+Data lake vs Data warehous
+
+Data saved in any format (raw) vs fixed format
+
+Schema is not needed vs Schema is needed
+
+Low trustability vs High trustability
+
+Storage and volume cost are better vs Provides fast and efficient queries
+
+Data lake key features (example - Amazon S3)
+
+* data catalogue and searching
+* user access
+* data gathering
+* manipulation and analysis
+
+There are also data lake houses that combine the benefits of both lakes and houses
+
+![image](https://github.com/user-attachments/assets/ec1dd352-9f6a-43e6-9644-2dbc0b654826)
+![image](https://github.com/user-attachments/assets/184741f4-0e87-4b8c-818d-47d47c024deb)
+
+Things to consider for data lakes in finance
+
+* regulation compliance
+* multi-user support
+* user certification and ability to use/maintain
+* authorization
+* protection against attackers
+* private network connection and traffic
+
+Data lake security levels
+
+* identity and access management
+* application security
+* data protection
+* infrastructure security
+* network security
+* identity and access management (platform)
+* multi-account management
+
+AWS lake formation
+
+![image](https://github.com/user-attachments/assets/aef074ab-f8e7-40d8-825c-2eb72be8ae29)
+
+### Sixth session - 'Using Amazing SageMaker Canvas for GenAI'
+
+Data prep challenges
+
+* too many different tools
+* various code dependancies
+* scalability
+* operations management
+
+Data prep/manipulation to get a clean version of the data for a model can take 60-80% of the time of an ML project
+
+SageMaker Canvas has us covered - new feature - chat for data prep
+
+* run code with plain text, edit/remove/add data, get insights, data prep process one with no code and ready to make a model
+
+![image](https://github.com/user-attachments/assets/f971bff1-ee02-4a71-a416-d91e35aea5ee)
+
+After our data is ready, there are ready to use models for text, image, audio content. 
+
+Finetuning? Can be done without writing any code as well. Selecting a model (or up to 3), data, and we get training and validation results and extra info if we want to finetune with python.
+
+
+
+Great day. I am glad I attended ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 9c35b6b0db1b6760d62bfa0996761eb1ebf66ec7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:40:32 +0900
Subject: [PATCH 244/638] Update 2024-05-18-day-138.md

---
 _posts/2024-05-18-day-138.md | 44 ++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-18-day-138.md b/_posts/2024-05-18-day-138.md
index cadf548b6..ddc30480e 100644
--- a/_posts/2024-05-18-day-138.md
+++ b/_posts/2024-05-18-day-138.md
@@ -1,5 +1,45 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 138) Fine-tuning Speech T5 using a very small Glaswegian dataset
+categories: [applying-knowledge,glaswegian,project]
 ---
+
+## Hello :) Today is Day 138!
+A quick summary of today:
+* fine-tuned microsoft's speech T5 to create a Glaswegian TTS and published it on [huggingface](https://huggingface.co/microsoft/speecht5_tts) (using our [project's dataset](https://huggingface.co/datasets/divakaivan/glaswegian_audio))
+
+I started the day by meeting with my collaborator (as I am writing this I remembered I forgot to ask them if I can mention them by name today ...) for an hour discussing my progress last week on the Scottish phrases audio clips and how we can move forward.
+
+After the short clips, next we can move forward with transcribing podcasts, or him/friends doing voice recordings for our dataset because the 1st option might be too much manual labour.  
+
+During the call M asked me how the whole model operation happens and about embedding the text and audio, and after the meeting I decided to dig a bit and see how *once* we have a good dataset, how do we actually use it.
+
+I found [this](https://huggingface.co/learn/audio-course/chapter0/introduction) course from huggingface that introduces working with audio data. One of the units is specifically about TTS. From there I learned about a library called [TTS](https://pypi.org/project/TTS/) and tried to somehow transform the data we have so far to put init one of their models, but with no success. From that I found a public [colab notebook](https://colab.research.google.com/drive/1GiI4_X724M8q2W-zZ-jXo7cWTV7RfaH-?usp=sharing#scrollTo=zd2xo_7a8wyj) by them that involves basic steps of uploading .wav files -> clicking a create dataset button -> clicking train model button -> clicking generate button. This was using gradio so it was some kind of a low-code demo solution, but I did not completely like it because the generated audio was slow (probably data problems), but also not something that I can easily share on huggingface/a public platform. 
+
+Before doing any kind of model I wanted to see what it's like uploading a dataset to huggingface. Using their docs I finally managed to push a simple version. As for the data, instead of having each clip being 3 times each (as in the [original youtube videos](https://www.youtube.com/watch?v=LkWplv39-80&t=171s&pp=ygUSc2NvdHRpc2ggcGhyYXNlcyA1)) I included each phrase only once - mainly for simplicity. After some hurdles ~
+
+![image](https://github.com/user-attachments/assets/e2b54524-8ff3-4580-bc67-c32e493b350f)
+
+[I did it](https://huggingface.co/datasets/divakaivan/glaswegian_audio) ^^
+
+Finally, I randomly found [this notebook](https://colab.research.google.com/drive/1i7I5pzBcU3WDFarDnzweIj4-sVVoIUFJ#scrollTo=wm7B3zxrumfF) that fine-tunes Speech T5 using huggingface's API.
+
+It goes through cleaning and preparing the dataset (I actually finished the dataset's upload to huggingface after seeing that this notebook allows for that), tokenization, training (which took about 1 hour), and then evaluating. 
+
+After some trial and error, I managed to [push a fine-tuned model](https://huggingface.co/divakaivan/glaswegian_tts) to huggingface ^^
+
+![image](https://github.com/user-attachments/assets/132c9586-8ffe-48f5-a716-3181fd51cb1e)
+
+Sadly the inference and compute button generate some error unfortunately, but when I used python in a notebook, I managed to get a clip out. 
+
+Important note! The dataset is terribly small, 118 audio clips, with a total of 250 seconds haha. But it is a nice POC of how we can do it after we have a sensible dataset. 
+
+At the end of the notebook there is a way to make some kind of a [demo](https://huggingface.co/spaces/divakaivan/glaswegian-tts-demo) with gradle on huggingface, and I finally managed to get it to work!!! :party:
+
+As a final note, when I woke up I read a bit more of Grokking ML - about classification models' metrics and naive Bayes' model.
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From db638572818fe64a2f82b0857fd7d2ab864de91d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:44:23 +0900
Subject: [PATCH 245/638] Update 2024-05-19-day-139.md

---
 _posts/2024-05-19-day-139.md | 83 +++++++++++++++++++++++++++++++++++-
 1 file changed, 81 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-19-day-139.md b/_posts/2024-05-19-day-139.md
index cadf548b6..dccd02794 100644
--- a/_posts/2024-05-19-day-139.md
+++ b/_posts/2024-05-19-day-139.md
@@ -1,5 +1,84 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 139) MLFlow (MLOps) on AWS
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 139!
+A quick summary of today:
+* learned how to host a mlflow server on AWS
+
+I managed to get my card working and finally finished the set up on my AWS account, so I could finish the last bit of module 2 from the [MLOps zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp).
+
+First, go to EC2 console
+
+![image](https://github.com/user-attachments/assets/a86a634b-11e3-4fe5-8dfa-9f90204eebfd)
+
+And set up an instance
+
+![image](https://github.com/user-attachments/assets/0c44fca9-59c7-4bf1-a31d-00d52e5a62f0)
+
+We need to select instance type (for me it is the free one)
+
+![image](https://github.com/user-attachments/assets/acbdc809-a03c-46f3-8d92-09e8d8de1ea1)
+
+Then we need to create a key-pair for logging in
+
+![image](https://github.com/user-attachments/assets/47afd555-1f48-48f5-b615-727a3ae55eb3)
+
+And it is done
+
+![image](https://github.com/user-attachments/assets/7e58f8ac-a674-4ec8-a334-cc0eb33a7cd0)
+
+Next we need to create an S3 bucket
+
+![image](https://github.com/user-attachments/assets/2a9ed907-e95d-4dc2-9afa-71f34e7ddae9)
+
+Next, we create a postgresql db
+
+![image](https://github.com/user-attachments/assets/7e90329b-18f5-4374-918d-185e618aee9b)
+
+Some configs to choose
+
+![image](https://github.com/user-attachments/assets/6fb6b427-71a1-4f6f-95db-a50a47957e78)
+
+And I got a link for the db on port 5432 (which I later use for the mlflow server). We need to give access to EC2 to the database
+
+And we have the console
+
+![image](https://github.com/user-attachments/assets/fec98946-e223-448f-8237-dfa12955b6b9)
+
+Where we have to install some dependencies
+
+Install dependencies: pip3 install mlflow boto3 psycopg2-binary
+
+I had to create a user and add some permissions so that it can connect to S3, EC2 and RDS
+
+![image](https://github.com/user-attachments/assets/2e88e7d8-0cd5-402c-834c-3ab29c6336bf)
+
+Then I got it running
+
+![image](https://github.com/user-attachments/assets/3c6d69f1-d6f6-4f2c-b4cd-df3becf93975)
+
+So just like when doing it locally we have the db (ofc run on aws), and the mlflow hosted as well
+
+![image](https://github.com/user-attachments/assets/d59561c4-b515-4b81-a7fe-7b23690d1a5f)
+
+To make sure I do not get any surprising charges on my card, I deleted/stopped all the instances I created.
+Looking forward to Module 3 of the MLOps zoomcamp, and also the 2024 version of the Module 2 homework should be uploaded some time in the upcoming week.
+
+
+
+On another note, today I found 4 interesting books
+
+![image](https://github.com/user-attachments/assets/5d4da124-5698-4848-b3d5-8a249798483b)
+![image](https://github.com/user-attachments/assets/74c0380f-15ca-4b0a-a0b8-9ef73007397b)
+![image](https://github.com/user-attachments/assets/dc5ac69d-60d1-428c-877d-6c7c2fee4390)
+![image](https://github.com/user-attachments/assets/6a17dcae-74a2-4b06-afe3-15f8402e86d4)
+
+I added them to my list to read/study into ^^
+
+
+That is all for today!
+
+See you tomorrow :)

From 346e9295658546846ff4f4cd9b9104c44487a6c1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 11:54:39 +0900
Subject: [PATCH 246/638] Update 2024-05-20-day-140.md

---
 _posts/2024-05-20-day-140.md | 46 ++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-20-day-140.md b/_posts/2024-05-20-day-140.md
index cadf548b6..3038f2e0a 100644
--- a/_posts/2024-05-20-day-140.md
+++ b/_posts/2024-05-20-day-140.md
@@ -1,5 +1,47 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 140) First 3 chapters of A Primer For The Mathematics Of Financial Engineering by Dan Stefanica
+categories: [math,theory]
 ---
+
+## Hello :) Today is Day 140!
+A quick summary of today:
+* Ch 1 - calculus review and plain vanilla options
+* Ch 2 - improper integrals, numerical integration, interest rates and bonds
+* Ch 3 - probability, Black-Scholes formula, Greeks and hedging 
+
+From the books I found yesterday I thought that starting from this one is the best because of its pure math + real application format. Each chapter consists of some math/stats and also how they are used in the real world.
+
+
+
+### Ch 1 - calculus review and plain vanilla options
+
+![image](https://github.com/user-attachments/assets/955157ad-0ee8-4f39-ab80-8e1f41f165a1)
+![image](https://github.com/user-attachments/assets/d4638b75-0dff-4e48-bffa-c89914814420)
+![image](https://github.com/user-attachments/assets/da731d57-db29-47e5-a107-efc084602f63)
+![image](https://github.com/user-attachments/assets/a7d1b663-eade-4fd1-b066-d12cbb38483e)
+![image](https://github.com/user-attachments/assets/0f44ebe8-4145-4d85-ad04-e2791935e276)
+![image](https://github.com/user-attachments/assets/1a9bfa43-2721-42c3-a5f8-92a819e153e2)
+![image](https://github.com/user-attachments/assets/c52698fd-51a5-4507-915c-db7492f1df03)
+
+### Ch 2 - improper integrals, numerical integration, interest rates and bonds
+
+![image](https://github.com/user-attachments/assets/286fbe9a-8a95-4582-a8c1-5e78948ae745)
+![image](https://github.com/user-attachments/assets/77603f42-3dad-41e1-9023-d86df36b1cb9)
+![image](https://github.com/user-attachments/assets/cee9aadc-859e-45e9-8ee4-d99e8154c490)
+![image](https://github.com/user-attachments/assets/850a5186-d419-4f21-b819-558db699a43d)
+
+### Ch 3 - probability, Black-Scholes formula, Greeks and hedging 
+
+![image](https://github.com/user-attachments/assets/1d608767-99e6-4ced-a896-d86f48038968)
+![image](https://github.com/user-attachments/assets/b08664ba-9406-4ec5-9193-dc12b90cd770)
+![image](https://github.com/user-attachments/assets/818e4274-bb27-49fd-93d4-42eefc119f70)
+![image](https://github.com/user-attachments/assets/30ed878d-dd36-454a-a062-150a33ef11af)
+
+On another note ~ today I realised that calling this whole blog project self-study into AI might not be the best. Because math, probability and stats are at the root of AI. So maybe some kind of renaming would be appropriate. 
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From 9436d8f469e376d0637d83868822376518c930e7 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:05:09 +0900
Subject: [PATCH 247/638] Update 2024-05-21-day-141.md

---
 _posts/2024-05-21-day-141.md | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-21-day-141.md b/_posts/2024-05-21-day-141.md
index cadf548b6..f2b690edb 100644
--- a/_posts/2024-05-21-day-141.md
+++ b/_posts/2024-05-21-day-141.md
@@ -1,5 +1,32 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 141) Lognormal random variables and looking for a TA position
+categories: [theory,math]
 ---
+
+## Hello :) Today is Day 141!
+A quick summary of today:
+* read a bit more of A Primer For The Mathematics Of Financial Engineering by Dan Stefanica
+* applied (sent an email) about becoming a TA for one of the undergraduate big data/ML/stats courses in my uni
+
+I like the Primer book because it is pure math - just Lemmas and Theorems and proving them (+ application in finance).
+
+Below are my notes from Ch 4 - lognormal random variables
+
+![image](https://github.com/user-attachments/assets/2ebd239e-0f5d-4f1b-a554-28bd29d53181)
+![image](https://github.com/user-attachments/assets/f8e0b9e8-af36-41ec-8e49-ad1628b64eb9)
+![image](https://github.com/user-attachments/assets/a4bccbfd-c047-4c2b-b469-3e43822cd30c)
+
+I saw that a deadline for the 4th assignment from Stanford's [XCS224W](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs): ML with Graphs is due on the 28th so I will likely start doing it in the afternoon after a quiz I have at uni. 
+
+Also I saw that the official 2nd homework on model management and experiment tracking from the [MLOps zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp) is up, but let's see if I can do it tomorrow.
+
+
+
+As for the 'application' to be a TA. I believe teaching is the highest form of knowing something. Being able to explain a concept to someone who does not know it and them understanding it is one of the biggest certifiers that someone really know's something. So I wrote an email in Korean about that I know python, some math, some stats, have been studying and blogging here about ML/DL, so let's see if I can atleast meet with a professor even just for a chat.
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From b7c05448e29a798c482635d4a8bddafa26eb0eb3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:07:17 +0900
Subject: [PATCH 248/638] Update 2024-05-22-day-142.md

---
 _posts/2024-05-22-day-142.md | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-22-day-142.md b/_posts/2024-05-22-day-142.md
index cadf548b6..21b57578e 100644
--- a/_posts/2024-05-22-day-142.md
+++ b/_posts/2024-05-22-day-142.md
@@ -1,5 +1,21 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 142) Stanford's XCS224W - ML with Graphs - assignment 4 completed
+categories: [applying-knowledge,gnn]
 ---
+
+## Hello :) Today is Day 142!
+A quick summary of today:
+* did my 4th assignment for the XCS224W:ML with Graphs course
+* interviewed for a lab assisstant at my university's Big Data department
+
+Unfortunately, as with the previous assignments, I am not allowed to share anything from them. I can only share about the topic which was heterogeneous graphs (which I learned about on Day 103). It took a bit of review, coding and debugging torch/cuda version disparity, but ~ I got the max score in the end.
+
+![image](https://github.com/user-attachments/assets/6aa469ab-2530-4ca2-8c4c-51bd5a992673)
+
+As for the interview ~ yesterday I sent an email to the head of the department of the Big Data department at my uni - Professor Minju Park, last night he responded and invited me today at 12 at his office. We had a chat, answered questions, ambitions about research, projects, and at the end ... he said there is 1 space left in his lab and that I can join. The lab is all undergrads and myself (the only non-korean) and I am excited to study, learn, share about anything related to AI - from math to probability to stats to ML to DL. I said I can start from tomorrow, and I will start tomorrow. 
+
+
+That is all for today!
+
+See you tomorrow :)

From 0883d25dd66488cd19d0a9c4c1180d4120cdfb00 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:12:41 +0900
Subject: [PATCH 249/638] Update 2024-05-23-day-143.md

---
 _posts/2024-05-23-day-143.md | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-23-day-143.md b/_posts/2024-05-23-day-143.md
index cadf548b6..db02dcf0b 100644
--- a/_posts/2024-05-23-day-143.md
+++ b/_posts/2024-05-23-day-143.md
@@ -1,5 +1,37 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 143) Forward, backward prop and param update by hand
+categories: [math,applying-knowledge,deep-learning]
 ---
+
+## Hello :) Today is Day 143!
+A quick summary of today:
+* did forward and backprop of a toy neural net by hand and confirmed with python
+* set up my station at the student lab at uni
+
+The idea of doing backprop manually again came back yesterday night. But this time I wanted to do it 'as in the old days'. In one of the lectures of CS109 Professor Chris Piech said he used to do forward/backprop manually back before autograd, so I wanted to experience that as well (in a toy example). 
+
+Below are my handwritten notes of the forward, backprop and param update for the 1st iteration of a toy neural net. 
+
+![image](https://github.com/user-attachments/assets/6b907786-0641-4d38-a63f-107ccca7ae5c)
+![image](https://github.com/user-attachments/assets/4bbc4170-40ce-4894-b8f6-28403d3de154)
+![image](https://github.com/user-attachments/assets/ccbb8356-4895-4437-9abc-d0c5e56a4ab0)
+![image](https://github.com/user-attachments/assets/7748559c-ecdd-4e00-b0bf-033a3e3e7b70)
+
+My tablet battery died towards the end so I wrote it on the board.
+
+![image](https://github.com/user-attachments/assets/7998ab2e-38c9-4690-935a-a21166d9dba0)
+![image](https://github.com/user-attachments/assets/19f7149b-ece8-4e97-9976-077aff861927)
+
+I confirmed all the numbers with python - the used notebook is [here](https://github.com/divakaivan/neural-network-from-scratch/blob/main/toy_nn_from_scratch.ipynb).
+
+As for my 1st day at the lab
+I liked it. It was mostly waiting for downloads and setting up cuda/torch/python/etc because it was a new PC. There was a seminar where 2 of the students talked a bit about what they are doing.
+
+At the end of the day, I had a chat with Professor Park about potential competitions I can join and he can help me. So it is something to look into tomorrow. 
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From a7af2f8f4a31eb3843573ef810b30a87ae557108 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:35:30 +0900
Subject: [PATCH 250/638] Update 2024-05-24-day-144.md

---
 _posts/2024-05-24-day-144.md | 57 ++++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-24-day-144.md b/_posts/2024-05-24-day-144.md
index cadf548b6..abf950e5c 100644
--- a/_posts/2024-05-24-day-144.md
+++ b/_posts/2024-05-24-day-144.md
@@ -1,5 +1,58 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 144) Using Graph Neural Networks to predict taxi passenger demand and origin/destination
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 144!
+A quick summary of today:
+* started reading papers on using graphs for taxi demand and origin/destination predictions
+
+One of the papers my professor wrote is on Measuring taxi ridesharing effects and its spatiotemporal pattern in [Seoul, Korea](https://sciencedirect.com/science/article/pii/S2214367X22001016) and because I am learning about GNNs from Stanford's [XCS224](https://online.stanford.edu/courses/xcs224w-machine-learning-graphs): ML with Graphs I remembered that one of the most popular use cases of graphs is on traffic prediction. 
+
+I read 2 papers:
+
+* [GNN-based Passenger Request Prediction](https://arxiv.org/pdf/2301.02515)
+* [Origin-Destination Matrix Prediction via Graph Convolution: a New Perspective of Passenger Demand Modeling](https://zheng-kai.com/paper/kdd_2019_b.pdf)
+
+I actually found the 2nd paper from the 1st paper because it was referenced.
+
+There are these 4 models used as comparison vs the proposed GNN-based model
+
+![image](https://github.com/user-attachments/assets/c09f6c34-901b-4cce-8327-85ac0aa5b500)
+
+As for the models, the 1st paper kind of goes off from the 2nd one on the idea to separate a city into grids. 
+
+![image](https://github.com/user-attachments/assets/be1aee7f-ef4c-46bd-892d-333a045d100a)
+
+and construct an OD (Origin-Destination/Adjacency) matrix based on that with requests from one cell to another
+
+![image](https://github.com/user-attachments/assets/a65e0801-e14e-45fd-b9e9-61b87cbf63cb)
+
+Below is a pic from the 2nd paper on grids
+
+![image](https://github.com/user-attachments/assets/2f9a297c-397f-4166-b509-1e0e19f21f4d)
+
+Compared to the 4 models, the proposed GNN-based one performs the best on MAPE and MAE scores. 
+
+![image](https://github.com/user-attachments/assets/fb456240-ebf3-452b-adb2-a5d814d7819a)
+
+There are a few more papers that I have noted to read during the weekend:
+
+Passenger Mobility Prediction via Representation Learning for Dynamic Directed and Weighted Graph
+
+ST-LoRA: Low-rank Adaptation for Spatio-Temporal Forecasting - this one I want to read is because the 2 papers from above that I read - they use this grid cell for origin/destination demand and what if we can use LoRA on that matrix? Might idea might be totally off but I want to explore it.
+
+A greedy approach for increased vehicle utilization in ridesharing networks
+
+Fair and Efficient Ridesharing: A Dynamic Programming-based Relocation Approach
+
+
+
+I hope I can think of something to research on these topics! Especially since one of professor Park's interests are in transportation.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 67a815db8d147687cb5bd052098d2cf9007deec1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:41:06 +0900
Subject: [PATCH 251/638] Update 2024-05-25-day-145.md

---
 _posts/2024-05-25-day-145.md | 85 +++++++++++++++++++++++++++++++++++-
 1 file changed, 83 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-25-day-145.md b/_posts/2024-05-25-day-145.md
index cadf548b6..bdafaf400 100644
--- a/_posts/2024-05-25-day-145.md
+++ b/_posts/2024-05-25-day-145.md
@@ -1,5 +1,86 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 145) Build & Modernize AI Applications with Azure (prep for Microsoft Azure x NVIDIA hackaton in Seoul)
+categories: [cloud,applying-knowledge,theory]
 ---
+
+## Hello :) Today is Day 145!
+A quick summary of today:
+* covered [course 1](https://learn.microsoft.com/en-us/collections/d8n2fg08dzem5m#msdynttrid=iHvSccBLIkdQzWBPCP8qwIT6Y9pOFXk1Nf7dsa5Ke6Y) for the hackaton: Azure Kubernetes and the cloud
+
+I don't think I shared before but on Wednesday I got confirmation that I can attend [Microsoft Azura x NVIDIA](https://msevents.microsoft.com/event?id=3190798606) hackaton in Seoul held on Monday and Tuesday 27th and 28th of May. It will be an amazing opportunity to visit the Seoul Microsoft office, learn from Microsoft professionals and in general participate in a hackaton. On the website they mention that if we want to prepare, they recommend covering two courses: the one I did today on 1. [building and modernizing apps](https://learn.microsoft.com/en-us/collections/d8n2fg08dzem5m#msdynttrid=iHvSccBLIkdQzWBPCP8qwIT6Y9pOFXk1Nf7dsa5Ke6Y) and 2. [improving productivity with github and Azure for devs](https://learn.microsoft.com/en-us/collections/mpn1f5od5rq525#msdynttrid=gCwNPCvw0x5f1WGnbugk1rfy8Ly1ehyeyvzpBQGnnnc) - which I will do tomorrow. 
+
+Before the material, there are some nice articles that I will review further
+
+![image](https://github.com/user-attachments/assets/ba99fcda-c715-4a86-90e8-fc5d0f71ef61)
+
+But as for the taught material, below is the outline
+
+![image](https://github.com/user-attachments/assets/73f0b3e0-04cb-429e-9387-771917f4dadb)
+
+Notes I took:
+
+K8r (Kubernetes) secrets:
+
+* Store sensitive information (like passwords) safely not in plain text
+* Encoded in base64 which makes them less visible in command outputs
+* Scoped to a single namespace to limit exposure
+
+ConfigMaps
+
+* Store nonsensitive data using a key value structure
+* Decouple configuration data from application code
+* Limited to 1 MiB of data to maintain simplicity
+* Scoped to a single namespace and can be referenced via environment variables or mounted as files
+
+On Azure Kubernetes Service (AKS)
+
+Containers:
+
+* Atomic units of software packaging code, dependencies and configuration
+* Offer benefits like immutability, smaller size, lightweight nature, and fast startup
+
+Container management:
+
+* Involves deploying, upgrading, monitoring and removing containers
+* Kubernetes helps automate these tasks, providing a portable, extensible platform for managing containerized workloads
+
+Kubernetes:
+
+* Abstracts container management complexity with a declarative configuration.
+* Functions as a cluster of primary and worker machines for deployment, scaling, and load balancing.
+* Does not provide middleware or data-processing frameworks; these run as containers.
+
+Cloud-native apps:
+
+* Built using modular, loosely coupled components
+* Allow for independent technology choices and frequent updates
+* Utilize cloud services like Kubernetes for scaling and Cosmos DB for data storage, minimizing the need for custom code
+
+Practical on implementing Azure services to an app
+
+![image](https://github.com/user-attachments/assets/d502e3ee-33cd-4d8d-b9e0-c9da3e090523)
+
+Tasks were:
+
+* download sample app
+* create web app
+* update and redeploy the app
+
+And I had to execute some commands in the provided terminal
+
+![image](https://github.com/user-attachments/assets/5d29b905-36b0-4776-9de1-ac05083f88d4)
+![image](https://github.com/user-attachments/assets/a9f90c0e-4ac7-414a-bace-1d3fdc700561)
+
+The deployed app was some static HTML website
+
+![image](https://github.com/user-attachments/assets/0c69acf2-54e3-4b87-84fb-473d46d66ff0)
+
+Also learned about autoscaling on Azure - a cloud process that dynamically adjusts resources based on demand. It involves scaling in (reducing resources) and scaling out (increasing resources) to maintain optimal performance and efficiency.
+Tomorrow I am planning to do my Module 2 homework from the [MLOps zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp) and some of the 2nd preparation course for the Microsoft x NVIDIA hackaton. 
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From fc2b34662458065c8431f2adff053c268a2f1897 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:48:11 +0900
Subject: [PATCH 252/638] Update 2024-05-26-day-146.md

---
 _posts/2024-05-26-day-146.md | 129 ++++++++++++++++++++++++++++++++++-
 1 file changed, 127 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-26-day-146.md b/_posts/2024-05-26-day-146.md
index cadf548b6..7bc016dbd 100644
--- a/_posts/2024-05-26-day-146.md
+++ b/_posts/2024-05-26-day-146.md
@@ -1,5 +1,130 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 146)] MLOps zoomcamp module 2 homework + some more prep for Microsoft x NVIDIA's hackaton
+categories: [theory,applying-knowledge,cloud]
 ---
+
+## Hello :) Today is Day 146!
+A quick summary of today:
+* did 2nd homework from MLOps zoomcamp on experiment tracking
+* covered some material on devops on Azure
+
+### MLOps zoomcamp Module 2 homework on experiment tracking
+
+The homework link is [here](https://github.com/DataTalksClub/mlops-zoomcamp/blob/main/cohorts/2024/02-experiment-tracking/homework.md). The homework followed the below outline:
+
+1. Install mlflow
+2. Preprocess NYC taxi data
+3. Train a model with autolog
+4. Launch a tracking server locally
+5. Tune model hyperparams
+6. Promote the best model
+
+My code for it is on my [github](https://github.com/divakaivan/MLOps-camp).
+
+The used model was a random forest regressor using rmse as the main evaluation metric. The homework was more focused on running mlflow commands rather than the preprocessing/training (which were done in Module 1 of the course). 
+
+Below is a graphical comparison of some of the models I ran
+
+![image](https://github.com/user-attachments/assets/1321df10-9e42-4ae8-9d1b-14e734f60112)
+
+I had to set up logging and mlflow db for a sample run
+
+![image](https://github.com/user-attachments/assets/8de59d5c-290d-4b63-b334-af373a98390d)
+
+Run hyperparam optimization 
+
+![image](https://github.com/user-attachments/assets/fb581664-e6a5-4bd1-8a92-688c254544f8)
+
+And finally choosing and registering the best model
+
+![image](https://github.com/user-attachments/assets/96f5ead5-3448-421f-b138-9ccfe0ffc387)
+
+For a bit I faced an issue with the params passed to the model because they were missing (initially not logged in the hyperparam opt code), and then I kept registering the *worst* performing model because i had [0] at the end of that best run, but when I put [-1] it was all good. Below are the top 5 and then best among the 5 models is registerered. 
+
+![image](https://github.com/user-attachments/assets/66f9a840-9848-4219-8026-4beff7ee3318)
+
+As for the next module - [module 3](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/03-orchestration), I saw we will use Mage to learn Orchestration and ML Pipelines and saw that the youtube videos were uploaded today. 
+
+### 2nd prep course for Microsoft x NVIDIA hackaton in Microsoft's office in Seoul
+
+It was a bit long so I covered just the bits I thought were new, and below are some notes I took along the way.
+
+Azure Boards: tools for agile planning, work item tracking, visualization, and reporting
+
+Azure Pipelines: a CI/CD platform that is language, platform, and cloud-agnostic, supporting containers and Kubernetes
+
+Azure Repos: cloud-hosted private Git repositories with two types of version control: distributed (Git) and centralized (Team Foundation Version Control, TFVC). Features include
+
+* Free private Git repositories
+* Support for any Git client
+* Web hooks and API integration
+* Semantic code search
+* Git code reviews with threaded discussion
+* Continuous integration for each change
+* Branch policies to ensure code quality
+* Integration with favorite tools and editors
+
+Azure Artifacts: Integrated package management supporting Maven, npm, Python, and NuGet packages from public or private sources
+
+Azure Test Plans: An integrated solution for planned and exploratory testing
+
+Workflows: Define automation processes, detailing trigger events and jobs to run. Workflows are written in YAML and located within the GitHub repository at .github/workflows. Example:
+
+![image](https://github.com/user-attachments/assets/fa0ab85d-dbc8-41b2-84f2-832f9d5cbd29)
+
+GitHub runners: compute resources that execute GitHub Actions workflows, performing build, test, and deployment tasks directly within GitHub repositories
+
+Continuous integration with actions example
+
+![image](https://github.com/user-attachments/assets/8dd18562-3a0d-4b30-89dd-32e23fcf9e6b)
+
+* On: Specifies what will occur when code is pushed
+* Jobs: There's a single job called build
+* Strategy: It's being used to specify the Node.js version
+* Steps: Are doing a checkout of the code and setting up dotnet
+* Run: Is building the code
+
+Next, Azure Pipelines is a robust service for creating cross-platform CI/CD workflows, integrating with your preferred Git provider and deploying to major cloud services. Key features
+
+* End-to-end flow of value: Focus on delivering continuous value from concept to customer, avoiding siloed processes
+* Automation and reliability: Establish a repeatable, reliable process for software delivery
+* Quality verification: Each pipeline stage verifies feature quality to prevent user-facing errors
+* Team feedback and visibility: Ensure all team members have feedback and visibility into the delivery process
+* Frequent small changes: Promote frequent, smaller changes for better flow and optimization
+* Continuous improvement: Continuously monitor and resolve obstacles to improve pipeline efficiency
+
+Pipeline stages:
+
+* Build automation and Continuous Integration: Automatic building and integrating of changes
+* Test automation: Automated testing for functionality and quality
+* Deployment automation: Automated deployment to various environments
+
+Orchestration:
+
+* Release and pipeline orchestration: Manage and control the entire pipeline, providing top-level insights
+* Value stream mapping: Identify and improve inefficiencies
+* Infrastructure efficiency: Efficient infrastructure is crucial for an effective pipeline
+
+![image](https://github.com/user-attachments/assets/c9707515-7c94-4cd3-8c9b-1dbe89ad2e33)
+
+Key terms in Azure pipelines
+
+![image](https://github.com/user-attachments/assets/192d19d3-2c62-437e-bd8e-04f324a46760)
+
+* Agent: software that runs a build or deployment job
+* Artifact: collection of files or packages published by a build, used for distribution or deployment
+* Build: one execution of a pipeline, collecting logs and test results
+* Continuous Delivery (CD): process of building, testing, and deploying code to various stages, ensuring quality through multiple stages and constant monitoring
+* Continuous Integration (CI): practice of automating the testing and building of code to catch issues early. CI systems produce artifacts used in CD pipelines for automatic deployments
+* Deployment target: destination for application deployment, such as a virtual machine, container, or web app
+* Job: a set of steps run on an agent within a build, representing an execution boundary
+* Pipeline: script defining the CI/CD process, composed of tasks for testing, building, and deploying
+* Release: execution of a release pipeline, including deployments to multiple stage
+* Stage: primary divisions in a pipeline, such as 'build,' 'test', and 'deploy'
+* Task: building block of a pipeline, performing specific jobs like building, testing, or deploying
+* Trigger: configuration that specifies when the pipeline should run, such as on code push or schedule
+
+That is all for today!
+
+See you tomorrow :)

From 33331b0b26a4ea82946308b555f34d7730ecbd28 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:52:01 +0900
Subject: [PATCH 253/638] Update 2024-05-27-day-147.md

---
 _posts/2024-05-27-day-147.md | 67 ++++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-27-day-147.md b/_posts/2024-05-27-day-147.md
index cadf548b6..f08aae34c 100644
--- a/_posts/2024-05-27-day-147.md
+++ b/_posts/2024-05-27-day-147.md
@@ -1,5 +1,68 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 147) Microsoft Azure hackathon Day 1
+categories: [applying-knowledge,cloud]
 ---
+
+## Hello :) Today is Day 147!
+A quick summary of today:
+* attended day 1 of Microsoft Azure hackathon
+
+![image](https://github.com/user-attachments/assets/9e94ed44-28f0-4fa3-8a21-3ca9a7dcf73b)
+
+The learning objectives of the hackathon:
+
+* Understand GenAI concepts and how they can be used to build chat applications
+* Learn how to configure the Azure OpenAI service and use Azure AI Search to build private OpenAI with your own data
+* Gain practical experience in implementing a chat app architecture with Terraform
+* Explore features of chat apps, such as multilingual queries and advanced chat response settings
+* Apply GenAI to real-world scenarios, including increased user interactions and dynamic document processing
+* Operationalize AI-enabled applications with enterprise-level monitoring and logging
+* Scale AI-enabled applications with enterprise-level load balancing
+
+We are assinged to a table and the people on a table work together as a team. It is all about using Azure and its services.
+
+There are in total 6 challenges, 3 for Day 1 and the others for Day 2. Below are some pics/notes from the first three challenges. 
+
+We are given access to a VM, and have to do all the challenges in there
+
+![image](https://github.com/user-attachments/assets/a1971532-fc8d-4e20-aa8f-797ce132abb1)
+
+Challenge 1: Deploy Azure OpenAI Service and LLM Models
+
+* Deploy Azure OpenAI Service and Large Language Models (LLMs) such as gpt-35-turbo and text-embedding-ada-002 to improve Contoso Ltd.'s customer support operations by handling inquiries efficiently
+
+![image](https://github.com/user-attachments/assets/c442421d-1eb4-49e6-8145-b9240387aedd)
+
+Challenge 2: Implement Document Search with Azure AI Search
+
+* Leverage Azure AI Search and Azure OpenAI (GPT-3.5-Turbo) to create a document search solution that makes support documents searchable and understands customer queries, enhancing Contoso's support services
+
+Had to set up Azure AI search, AI service, a Storage account, upload documents to azure, create an indexer. The result:
+
+![image](https://github.com/user-attachments/assets/e50a7261-7ef8-45af-a17d-a0a1edcdd51c)
+
+A simple html with all the reviews (pdf files uploaded) and a way to search using key words. 
+
+Challenge 3: Deploy an AI-Powered Chat App
+
+* Deploy an AI-powered chat application for Contoso Electronics. This app showcases advanced features like chat and Q&A interfaces, augmented by AI capabilities, and integrates Azure OpenAI Service with the GPT-3.5 Turbo model and Azure Cognitive Search for efficient data indexing and retrieval
+* 
+Actually this one we were not able to complete because non of us had used terraform before and we faced various issues related to running scripts. At the end of the day, even though the final deployment script ran - there were plenty of errors, but our team stayed after everyone as so there was no mentor to help us. Hopefully we can run and complete it tomorrow (along with the rest of the challenges).
+
+Architecture diagram of the implemented app
+
+![image](https://github.com/user-attachments/assets/aee0c04e-bcea-439e-aff3-444b798e29ee)
+
+Here are some pics of the view (hackathon room is on the 13th floor)
+
+![image](https://github.com/user-attachments/assets/514ffd36-6867-47df-9d51-c6cabfd0e238)
+![image](https://github.com/user-attachments/assets/9773780b-3187-4161-9fd0-9ba0a48fa04c)
+
+Tomorrow is Day 2 with some more challenges ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From d779b9cf3930cfe6baf8942cba9a88df51280859 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:54:08 +0900
Subject: [PATCH 254/638] Update 2024-05-28-day-148.md

---
 _posts/2024-05-28-day-148.md | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-28-day-148.md b/_posts/2024-05-28-day-148.md
index cadf548b6..eeae2e818 100644
--- a/_posts/2024-05-28-day-148.md
+++ b/_posts/2024-05-28-day-148.md
@@ -1,5 +1,34 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 148) Microsoft Azure hackathon Day 2
+categories: [cloud,applying-knowledge]
 ---
+
+## Hello :) Today is Day 148!
+A quick summary of today:
+* attended day 2 of the Microsoft Azure hackathon in the Microsoft office in Seoul
+
+![image](https://github.com/user-attachments/assets/c1f32482-5c97-4e50-a7c4-46632c6e2586)
+
+Today ~ we tried. The 3rd challenge from yesterday - deploying an AI-powered chat app... well we could not do it today either. Turns out all the teams were facing the same issue and I was trying to fix errors on that 3rd task. The main issue was at the last step - deployment and people were getting some kind of os error. At some point I got that too, and after trying to fix it I started getting another error in regards to access (felt like I went a step back maybe haha). In the end ~~~ It was not possible, and the 4th challenge which was about testing the app from the 3rd challenge was not possible to complete either. 
+
+Then I moved to the 5th one. Part of it was setting up an indexer that reads documents, then setting up a model that recognizes and read different parts of invoices (pics below)
+
+![image](https://github.com/user-attachments/assets/ea5d3969-52fb-4b53-8681-75666e1e4652)
+![image](https://github.com/user-attachments/assets/c192687d-1eaf-4bf7-881c-7332b85aad7a)
+
+Actually I realise now as I am writing that I did not take much pics while working today because we were just trying to get something finished while fixed tons of errors coming our way. Also from our team (3, including myself) noone had any Azure experience so we were diving into the deep by ourselves. After the model was trained and tested. We had to set up a 'pipeline with BPA'. BPA stands for business process automation but there was no such service on Azure and we just kept trying to see how to do a normal pipeline with data, the model and deployment. At this point the 5th and 6th challenges came together because they both involved deploying a model + pipeline but time was pushing so we were just looking at how to deploy a 'BPA pipeline' on Azure. I found some online tutorial about using Azure's Document intelligence and I was following it but I kept getting errors again haha. Another part of the challenge was to deploy an OpenAI service resource and contrary to the rest of the day - this actually worked without any problems haha. 
+
+At the end of the 2 days ~ I learned a lot about Azure and its ecosystem - storage, models, AI service, access - and some issues someone might face when using it. 
+
+As a goodbye, we were given some gifts too (pen, cup, tshirt of the hackathon)
+
+![image](https://github.com/user-attachments/assets/d65460ee-7127-411c-8170-bc7af704e34b)
+![image](https://github.com/user-attachments/assets/4bf968c4-4cdc-433e-a5e1-12c42bd29cad)
+
+Thank you to Microsoft Korea for inviting me :)
+
+
+That is all for today!
+
+See you tomorrow :) 

From c7bf2d3eefbbc8888f4ca5fa6978221c197beb64 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:59:20 +0900
Subject: [PATCH 255/638] Update 2024-05-29-day-149.md

---
 _posts/2024-05-29-day-149.md | 105 ++++++++++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-29-day-149.md b/_posts/2024-05-29-day-149.md
index cadf548b6..36ae50a6d 100644
--- a/_posts/2024-05-29-day-149.md
+++ b/_posts/2024-05-29-day-149.md
@@ -1,5 +1,106 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 149) Learning about the Origin-Destination Matrix Prediction problem in passenger prediction tasks
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 149!
+A quick summary of today:
+* back in the lab and read 2 papers about Origin-Destination Matrix Prediction via Graphs
+
+I decided to start using obsidian to take notes and it is amazing. It is also good practice writing math using latex. Also for some reason I cannot copy paste the math formulas, so some of the text below is in pictures so that the formula (math notation) can be included for clarity. I need to figure out a way to better share these notes. 
+
+### Paper 1: Gallat - Passenger Mobility Prediction via Representation Learning for Dynamic Directed and Weighted Graph (DDW)
+
+Introduction
+
+At first, studies concentrated on passenger demand as a function of starting location and time period. However, this failed to account for the passengers' destination. This was tackled by the definition of Origin-Destination Matrix prediction problems, where there are different time slots with their own OD matrix describing travel demand from region i to region j. Based on this ODMP, Graph Neural Networks have emerged as the go-to solution because of their ability to capture complex relationships and compared to previously applied CNNs, generalise to non-Euclidean graph topologies. 
+
+![image](https://github.com/user-attachments/assets/4857a671-fb4f-4d30-b636-de4270170d13)
+
+Definitions
+
+![image](https://github.com/user-attachments/assets/d3c40ed3-9f0e-46b5-840a-7fab7b6cc7da)
+
+Methodology
+
+![image](https://github.com/user-attachments/assets/6b8ffc92-1db7-4bea-8356-cb3aa37ceb86)
+![image](https://github.com/user-attachments/assets/72eed545-91c4-4f18-a502-5b4cade587e2)
+![image](https://github.com/user-attachments/assets/675c9e6a-0964-4129-8f14-b168b753077f)
+![image](https://github.com/user-attachments/assets/d59562cd-c733-4915-8037-b6bf0bf6dd8b)
+
+* pre-weighted functions - helps with the data sparsity. In addition to node semantic similarity, the obtained weights also incorporate real-time popularity of neighbour regions
+
+Temporal Attention Layer
+
+This layer can capture the sequential dependencies among embeddings and generate spatiotemporal representation for predicting the passenger demand in the next time slot. A naive approach to this would be to gather information from P most recent and consecutive DDW graphs. This might not work that well since in reality, a graph at time T will have similarity with graphs in close time slots, and if the time difference is too big this will introduce noise into the learned graph representation. To improve on that, the paper uses P historical DDW graphs form the same time slow of each day.
+
+Transferring Attention Layer
+
+Focusing on the next time slot, we can predict passenger demands between regions. Initially a vector with outbound passenger demands at the next time slot T+1 is created. Then it is distributed to different regions using a probability distribution that indicates the likelihood of a passenger trip from one region to every other region in the next time slow.
+
+![image](https://github.com/user-attachments/assets/5b22c62b-0a79-4d7c-a9f9-732d6f0e5528)
+
+Experiment
+
+Dataset
+
+![image](https://github.com/user-attachments/assets/34d780f3-3a3a-4374-9e8a-543040d7f3a3)
+
+Baseline models
+
+History Average (HA), LSTNet, GCRN, GEML
+
+Results
+
+0,3,5 are thresholds for demand. I.e. threshold 0 includes all regions, threshold 3 includes regions where there is at least 3 passenger demands 
+
+![image](https://github.com/user-attachments/assets/850bd33f-3c68-4302-bc04-9b59217a40f3)
+
+### Paper 2: GEML - Origin-Destination Matrix Prediction via Graph Convolution. A New Perspective of Passenger Demand Modeling
+Introduction
+
+The ODPM problem is essential to solve for large taxi companies, as it will give information about customers' demands from one place to another. ODPM provides info about combinations of different origins and destinations, and the demand for each origin-destination combination. The target is to predict the number of orders from one place to another in a given time slot.
+
+OD matrix construction
+
+![image](https://github.com/user-attachments/assets/f1616666-93cd-44c2-a3c2-720e90b0e2f0)
+![image](https://github.com/user-attachments/assets/5e996cf8-c0a7-4bb2-afc7-d3c8a1ad1e36)
+![image](https://github.com/user-attachments/assets/0d376537-ea08-4442-a037-a4a5e3f4e4ee)
+
+Methodology
+
+![image](https://github.com/user-attachments/assets/ac27c895-8e73-4bdb-91e3-d2115317c787)
+
+Grid-Embedding based Multi-task Learning (GEML) 
+OD matrices are extracted from mobility records of ride service providers. Then the grid embedding part of GEML learns the embedding vector for each grid through aggregating geographical and semantic neighbours information. Next, the vector representation is put through a multi-task neural network which learns the grid for the most recent time slot. Finally, the result is used to for a predicted OD matrix. In the authors' proposed model, both spatial information - using a neighbour-based grid embedding to learn each grid's representation by aggregating neighbour information, and temporal information - using the multi-task neural network that can learn dynamic trends of passenger movement over time, is captured.
+
+Grid embedding
+
+GCNs are limited when it comes to sparse graphs - which can happen in low demand areas. To tackle this, the authors propose two kinds of neighbourhoods for better neighbourhood aggregation - geographical and semantic neighbours. This achieves to measure the closeness between a grid and its neighbours and in addition, the traffic flow's semantic strength between origin and destination.
+
+![image](https://github.com/user-attachments/assets/325b72fd-be4d-4dee-89c0-27845efa6fa3)
+![image](https://github.com/user-attachments/assets/bd13effc-751f-44bb-a92e-5950c5a6d697)
+![image](https://github.com/user-attachments/assets/437ee298-6745-46d7-ba11-dfd971cd6ba4)
+
+Evaluation
+
+Compared with History Average (HA), LSTM, LSTNet, GCRN 
+Where GEML-XX are modifications to the model.
+
+![image](https://github.com/user-attachments/assets/1fdd3f27-2041-4447-92fb-1c570047b423)
+
+Visualisation
+
+![image](https://github.com/user-attachments/assets/5be250ca-6a52-4e84-9f5f-a0bf10d12bab)
+
+(a) Morning mobility: people leave home and go to different destinations. g202, g205, g189, g169 are residential areas, while g267, g290 are software parks
+
+(b) Post-lunch mobility: there are still many passengers going to work/entertainment venues.
+
+(c) Evening mobility: there are people going home, but also going to entertainment venues
+
+
+That is all for today!
+
+See you tomorrow :)

From 5c956ce431e270644cb38469a8d71df56520cbb0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 14:05:09 +0900
Subject: [PATCH 256/638] Update 2024-05-30-day-150.md

---
 _posts/2024-05-30-day-150.md | 168 ++++++++++++++++++++++++++++++++++-
 1 file changed, 166 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-30-day-150.md b/_posts/2024-05-30-day-150.md
index cadf548b6..2e6836e73 100644
--- a/_posts/2024-05-30-day-150.md
+++ b/_posts/2024-05-30-day-150.md
@@ -1,5 +1,169 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 150) Learning more about taxi OD matrix prediction + Scottish dataset update
+categories: [glaswegian,applying-knowledge,reading-research]
 ---
+
+## Hello :) Today is Day 150!
+A quick summary of today:
+* read a few more papers about taxi demand and Origin-Destination matrix prediction using Graph Neural Networks
+* added more audio clips to the [Scottish (Glaswegian) dataset](https://huggingface.co/datasets/divakaivan/glaswegian_audio)
+
+Obisdian makes this nice graphs of papers referencing each other.
+
+![image](https://github.com/user-attachments/assets/1d5587fd-2e53-4c86-b6dc-9c9234ce1c2e)
+
+### First paper is Modeling Long- and Short-Term Temporal Patterns with Deep Neural Networks
+
+(some will be pictures because I am yet to find a better way to share the math notations)
+
+Introduction
+
+Multivariate time series forecasting can learn traffic jam patterns ahead of time. However, a common challenge is capturing dynamic dependencies across multiple variables. For example the traffic on a highway can experience two patterns - daily (morning vs evening traffic) and weekly (weekday vs weekend). If a model is not able to handle complex data, then time-series forecasting is not feasible. To this end, the authors propose the usage of deep neural networks (RNN and CNN), and in particular - Long- and Short-term Time-series Network (LSTNet).
+
+![image](https://github.com/user-attachments/assets/55f34e88-99f4-48f6-ad4c-8b80fdb574ba)
+
+ It combines the strength of convolutions to discover local dependency patterns among multi-dimensional input variables and the recurrent layer which can capture complex long-term dependencies. It also uses a 'Recurrent-skip', which helps with capturing long-term patterns and makes optimisation easier. Finally, a traditional autoregresive linear model is used side-by-side with the non-linear deep learning model for more robustness when the data has violate scale changing.
+
+Related Background
+
+Autoregressive integrated moving average (ARIMA) is one of the most famous univariate time series models due to its statistical properties and Box-Jenkis methodology. ARIMA models are adaptive, flexible and subsume other models like autoregression (AR), moving average (MA) and Autoregressive Moving Average (ARMA). However none of the above are used in high dimensional multivariate time series forecasting due to high computational costs.
+
+Vector autoregression (VAR), on the other hand, is one of the most used models in multivariate time series due to its simplicity. However, the model capacity grows linearly over the temporal window size, and quadratically over the number of variables which results in VAR models overfitting easily.
+
+Linear methods are often preferred for multivariate time series forecasting due to the availability of high-quality off-the-shelf solvers in the machine learning community. However, similar to VAR models, these linear approaches may struggle to capture the intricate non-linear relationships present in multivariate signals. Consequently, while linear models offer efficiency, they may sacrifice performance when faced with complex non-linear dynamics.
+
+Another type of method is the non-parametric Gaussian Processes (GP). However, it comes at a high price in terms of computational complexity.
+
+Methodology
+
+![image](https://github.com/user-attachments/assets/309c9bde-aee8-4fee-b89c-954151be3489)
+
+Recurrent Component
+The above output is fed through the Recurrent component and Recurrent-skip component in parallel. The Recurrent component is a GRU and the output is the hidden state at each time step.
+Autoregressive Component
+
+The autoregressive (AR) component is integrated into the LSTNet architecture to counteract the scale insensitivity of neural network models like CNNs and RNNs, particularly when faced with non-linear and non-periodic changes in input signals. AR models excel at capturing linear trends and dependencies in time series data, providing a complementary approach to the non-linear components of LSTNet. By combining both AR and non-linear components, LSTNet can effectively capture both linear and non-linear patterns in the data, enhancing forecasting accuracy.
+
+Evaluation
+
+Methods for comparison
+
+AR, LRidge, LSVR, TRMF, GP, VAR-MLP, RNN-GRU, LSTNet-skip, LSTNet-Attn
+
+Data
+
+![image](https://github.com/user-attachments/assets/6a391f09-7fc5-4fb2-b805-616a7af2663f)
+
+Results
+
+![image](https://github.com/user-attachments/assets/98d668d5-f441-4943-ab5b-0ea90478252e)
+
+### 2nd paper: Structured Sequence Modeling with Graph Convolutional Recurrent Networks
+
+Introduction
+
+Graph Convolutional Recurrent Network (GCRN) is a generalization of  RNNs to data structured as a graph. 
+
+Combinations of CNN and RNN models have been explored for videos and images, where CNN is used for visual feature extraction followed by a RNN for sequence learning. 
+
+This paper leverages the above to propose the GCRN model for predicting time-varying graph-based data, and at the core is merging CNN for graph-structured data and RNN to identify simultaneously meaningful spatial structures and dynamic patterns.
+
+![image](https://github.com/user-attachments/assets/c5e17dee-d6b6-4274-ac3b-817f437a811f)
+
+Proposed GCRN Models
+Model 1
+
+Stacking a graph CNN for feature extraction, and an LSTM for sequence learning
+
+![image](https://github.com/user-attachments/assets/703d425b-b7d8-4634-84e4-50e4de444878)
+![image](https://github.com/user-attachments/assets/46574087-c2be-4385-9d61-cd7b2452237f)
+
+Model 2
+
+To generalise the convLSTM model to graphs, the Euclidean 2D convolution * is replaced by a graph convolution *G
+
+![image](https://github.com/user-attachments/assets/fd6abd4e-e04e-4735-8ff8-4fa8647be16a)
+
+Comparison between models
+
+![image](https://github.com/user-attachments/assets/23a4076e-3048-4cdf-b71b-1e244bd9507c)
+
+Results
+
+![image](https://github.com/user-attachments/assets/0d82a6a4-179e-4519-b25c-bd78b30c2b68)
+![image](https://github.com/user-attachments/assets/8126a992-d33d-4db2-9087-2500c42f1977)
+
+### 3rd paper: A Baselined Gated Attention Recurrent Network for Request Prediction in Ridesharing
+
+Introduction
+
+![image](https://github.com/user-attachments/assets/f875d3f5-478f-49d7-a466-41f4e38f9acd)
+
+This papers proposes a new method to tackle the ODMP (RSODP - ridesharing ODP as referred in this paper) problem - called Baselined Gated Attention Recurrent Network (BGARN). It uses graph convolutions with multi-head gated attention for spacial features, a recurrent part for temporal features, and a baselined transferring layer to calculate final results.
+Related work
+
+Majority of work uses GNNs over CNNs to capture the non-Euclidean nature of the space and a LSTM to capture the temporal dynamics. Typically, the request data is preprocessed into a (OD) grid map, then neighbour data is aggregated (for spatial features), such as Semantic Neighborhood and haversine distance. These grids from the OD Graph form a new feature graph, where one directed edge represents the propagation tendency of the features from one grid to another. With the support of Graph Neural Networks (GNNs), the feature propagation among the nodes is performed, and the output grid matrix specifies the extracted request forwarding pattern. A time series of these grid matrices is then pushed into a recurrent module like Long Short-Term Memory (LSTM) to integrate the temporal features, such as tendency and periodicity (also referred to as trend and period). Finally, the output spatial-temporal features are translated back to a new OD Graph to predict the future request flow.
+
+[[GCRN]] does not use spatial features, instead it processes the OD matrix directly as a spatial feature. However, this increases the initial feature dimensions. This problem is solved by [[Gallat]] and [[GEML]] by specifying semantic and geographical neighbourhoods. Gallat further splits the Semantic neighbourhood into forward and backward which shows that grids have different importance as an origin and destination. Furthermore, the above two models also use pre-weighted functions that take the contributions of different neighbours into account.
+
+Simple GNNs do not account of for different importance along different edges and this is important in this OD problem because the connections between grids vary due to spatial dynamics. To this end, GAT was proposed which leverages edge weights using Attention.
+
+From a time point of view, there are papers which replaced the LSTM with GRU for reduced computation cost, others with a Transformer for capturing long-term temporal dynamics.
+
+![image](https://github.com/user-attachments/assets/9581c8b7-769c-4a13-9745-dce7a83d6d65)
+
+Problem definition
+
+![image](https://github.com/user-attachments/assets/b929f024-2f6c-42f5-96f1-ce14167b9c81)
+![image](https://github.com/user-attachments/assets/1735f2e7-b3ef-4f26-af57-2f1d7280850c)
+
+System Architecture
+
+![image](https://github.com/user-attachments/assets/9cfb89fe-f5db-4636-ac77-e062e73418bb)
+
+Preprocessing module
+
+![image](https://github.com/user-attachments/assets/13a4feb3-0576-46a7-9a54-1632a457e791)
+
+Spatial Attention Layer
+
+Originially introduced in GEML and Gallat, below are the definitions for spatial analysis.
+
+* Forward neighbourhood
+* Backward neighbourhood
+* Geographical neighbourhood
+
+Temporal Recurrent Layer
+
+![image](https://github.com/user-attachments/assets/af05caa0-2c18-4098-a87a-eb8c47872f2c)
+
+Transferring Layer
+This layer uses the previously produced spatial-temporal embedding matrix for two tasks: demand prediction for the next time slot, and the OD matrix for the next time slot. Baseline models: HA and AR (Auto-Regressive) Using the baseline results and the prediction results from the spatial-temporal embedding matrix, the Transferring 
+
+Layer obtains the final prediction outputs. Equations for task: 
+
+![image](https://github.com/user-attachments/assets/ad0a3ab8-11de-46c5-9443-415347c70212)
+
+where Aggr(.) specifies the tuning approach which can be sum, weighted sum or multiplication (by default).
+
+
+Dataset and Results
+
+![image](https://github.com/user-attachments/assets/c6addfe9-8d65-4581-9e3b-a67cd4bae479)
+![image](https://github.com/user-attachments/assets/30859f88-c369-452b-97f0-256523f525a3)
+
+### As for the Scottish dataset
+
+I got home and I preprocessed the ~150 clips (postprocess count of audio clips that my Scottish collaborator had sent me). And now we have a total of ~16 minutes of audio clips in Glaswegian. I also tried to finetune microsoft T5 again but my colab ran out of free credits. So tomorrow while working, I will use my PC. 
+
+![image](https://github.com/user-attachments/assets/33022e57-5e93-4fcf-b17c-cfa9cbab48db)
+
+This is a pic of the dataset on huggingface and I am so excited to see it grow. FYI I am not sure why but if I do not have this empty column, the whole thing breaks. But for now it is fine. 
+
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From cd334efc64ecd7253504c7980a333c6de3dcd7c5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:16:38 +0900
Subject: [PATCH 257/638] Update 2024-05-31-day-151.md

---
 _posts/2024-05-31-day-151.md | 175 ++++++++++++++++++++++++++++++++++-
 1 file changed, 173 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-05-31-day-151.md b/_posts/2024-05-31-day-151.md
index cadf548b6..c684c8f14 100644
--- a/_posts/2024-05-31-day-151.md
+++ b/_posts/2024-05-31-day-151.md
@@ -1,5 +1,176 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 151) Reading more about taxi OD matrix prediction architectures + more Scottish dataset audio included
+categories: [glaswegian,reading-research,applying-knowledge]
 ---
+
+## Hello :) Today is Day 151!
+A quick summary of today:
+* my collaborator recorder more audio and the Scottish (Glaswegian) dataset is growing
+* read more papers related to OD (origin-destination) matrix prediction for taxis
+
+### Fristly, about the [Glaswegian dataset](https://huggingface.co/datasets/divakaivan/glaswegian_audio)
+
+![image](https://github.com/user-attachments/assets/8085d6b4-2437-4a7b-9335-cd92c6dec505)
+
+The process of adding data to huggingface is faster now, so it is going great. We are up to ~18 mins.
+
+### Secondly, about the papers from today
+
+First paper is GNN for Traffic Forecasting - A survey
+
+
+Problems
+
+Traffic flow
+
+Traffic flow is the number of vehicles that pass through a spatial unit, such as a road segment or traffic sensor point in a given time slot. Three types are considered: road-level, region-level and station-level.
+
+Traffic speed
+
+This is the average speed of vehicles passing through a spatial unit in a given time slot. Traffic speed problems are categorized into road-level and region-level issues, including travel time and congestion predictions. Traffic congestion prediction aids in optimizing road conditions and vehicle flow. Challenges vary between urban roads and freeways, with urban roads posing more complexities due to intricate connection patterns, speed limit variations, and spatial dependencies.
+
+Traffic demand
+
+Refers to the potential demand for travel. An example of demand is the amount of requests sent through taxi hiring apps (whether all the demand is met is different).
+
+![image](https://github.com/user-attachments/assets/4f0c0114-7836-4ea5-ae99-a694ed2b5ec2)
+
+### Graph Neural Networks
+GNNs are currently the SoTA in traffic forecasting. Variations include recurrent GNNs, convolutional GNNs, graph autoencoders, and spatiotemporal GNNs, the last being used in this case since traffic forecasting is a spatiotemporal problem.
+
+### Open Data and Source Codes
+
+**Open data**
+- Three types: graph-related data (transportation network data), historical traffic data (historical traffic state records), and external data (weather and calendar data). Links to traffic sensor data, taxi data, ride-hailing data, bike data, and subway data used in surveyed studies.
+
+**Open source codes**
+- Links available to repositories.
+
+### Challenges
+
+**Heterogeneous data**
+- Data quality
+- Incomplete data
+- Traffic anomalies data is difficult to collect to train DL models
+- Data privacy
+
+**Multi-task performance**
+- Multi-task frameworks in ITSs are needed to predict demand for multiple transportation modes simultaneously.
+- Knowledge adaptation helps transfer information from data-rich to data-sparse sources for demand prediction.
+- Challenges include data format incompatibilities and differing spatial or temporal patterns.
+- Most surveyed models are designed for single-task training, though some handle multiple tasks like traffic flow and speed prediction on the same road segment.
+- Multi-task forecasting is challenging in graph-based modeling due to different graph structures for different tasks (e.g., road-level vs. station-level problems).
+- Some GNN-based models have tackled multi-task prediction (e.g., taxi departure/arrival flow, region-flow, transition-flow, crowd flows, OD flows).
+- Existing attempts typically use the same graph with multiple outputs via feed forward layers.
+- Significant further development is needed for GNN-based multi-task prediction, especially for tasks requiring multiple graph structures.
+
+**Practical implementation**
+- **Data Bias**: Existing GNN-based studies often use less than one year of data, introducing bias and limiting applicability to different times or places. Using longer traffic data requires updating underlying traffic infrastructures, increasing costs and complexity.
+- **Computation Scalability**: GNNs face scalability issues with large-scale traffic network graphs, often leading to consideration of only subsets of nodes and edges. Solutions like graph partitioning and parallel computing have been proposed, but they provide only marginal performance improvements over simpler models.
+- **Infrastructure Changes**: Real-world network graphs change with modifications in transportation infrastructure (e.g., new road segments, bus lines, points-of-interest). Static graph formulations are inadequate. Solutions include dynamic Laplacian matrix estimators and Data Adaptive Graph Generation (DAGG) modules to handle changing spatial dependencies.
+
+**Model interpretation**
+- Interpreting "black-box" machine learning or deep learning models, including GNNs, is a major criticism in traffic forecasting.
+  - **Early-Phase Techniques**: Techniques for explaining GNN predictions are still in the early stages of development and have not yet been applied to traffic forecasting.
+  - **Severity in Transportation**: The lack of model interpretation is particularly problematic in transportation due to complex and heterogeneous traffic data, making it harder to design interpretable models compared to simpler data formats like images and text.
+  - **Current Efforts**: Some efforts, such as incorporating state space models, have been made to improve model interpretation in traffic forecasting, but the issue remains largely unresolved, especially for GNN-based models.
+
+### Future directions
+- Creating a centralized data depository
+- Designing a transportation knowledge graph that leverages the traffic semantic information to improve forecasting performance
+- Using other techniques
+  - Data augmentation
+  - Transfer learning
+  - Meta learning
+  - Generative Adversarial Networks (GANs)
+  - AutoML
+  - Bayesian Networks
+
+### Applications in real-world ITS systems
+- Most GNN-based studies are based on simulations with historical traffic data and not used in real-world applications to test their validity.
+
+## Second paper: Taxi Origin and Destination Demand Prediction Based on Deep Learning: A Review
+
+### Introduction
+- When it comes to graphs and taxi demand prediction - node-level predictions are for the number of trips for that particular node (region), while edge-level predictions are for demand relationships between two nodes (regions).
+
+### Mathematical statistical methods
+- Statistical models are based on historical and time series data. The most widely applied ones are HA, ARMA, MA, ARIMA, and the Kalman filtering model. However, such models lack the ability to work with high-dimensional, spatiotemporal data such as taxi data.
+
+### Traditional machine learning methods
+- Common ones are linear regression, SVM, decision trees, RF, artificial neural networks. In these models, feature selection directly influences their accuracy, and while they improve upon the statistical methods, they do not effectively solve the nonlinear correlation of complex multidimensional data.
+
+### Deep learning background
+- CNN, RNN, GCN are commonly used for extracting temporal and spatial information. Some papers also use multi-task learning, residual networks, and other methods to get better accuracy. To make demand and OD matrix predictions, historical regional taxi data with both temporal and spatial information is needed. Below, the review focuses on four areas: spatial topology construction, spatial-dependent modeling, time-dependent modeling, and other factors.
+
+### Spatial topology construction
+
+**Raster data**
+- For CNNs, the area of interest is partitioned into non-overlapping grids of predetermined size. However, transportation data has spatiotemporal attributes and non-Euclidean structural characteristics which renders the grid structure powerless. In addition, if the raster data is small, the area of interest may be split and result in higher data volume that increases the difficulty of prediction. If the raster is too large, then it might be harder to extract demand features and we will end up with a reduction in accuracy.
+
+**Graph**
+- For GCNs, the travel demand data is transformed into images (non-Euclidean spatial data). There are two graph structures: static and dynamic graphs.
+  - **Static graph**: The model assumes that the graph structure is constant. To construct the OD matrix we can use methods like distance measures or Gaussian kernels. Otherwise, we can construct a binary adjacency matrix using information about which nodes are connected. Studies have also looked into adding auxiliary features like connectivity maps, semantic function maps, weather, distance maps, traffic connectivity maps.
+  - **Dynamic graph**: There are two types: (i) nodes and edges continually change over time, and (ii) node and edge properties vary over time. Traditional graph representation learning frameworks generate static representations and overlook the dynamic nature of transportation data. Taxi demand data is spatiotemporal so dynamic graphs can better represent its nature. There are two construction methods:
+    - **Discrete-time dynamic graphs (DTDG)**: Defines a fixed length r, and updates the embedding at each t time unit. The result is a dynamic adjacency matrix or a sequence of multiple graphs. Each graph is like a snapshot at a particular time slot. DTDG relies on the length r. A loose r can result in missing useful information like trends, whereas a more tight r can lead to extra noise. Therefore, the DTDG method suffers from loss of information due to the fixed (discrete) segmentation of the OD stream information.
+    - **Continuous time dynamic graphs (CTDG)**: Here the node representation is updated based on event data which can include type of event, location, and its time. An example of an OD request can be described by a tuple (e, l, t), where e is the type of OD request, l is the location, and t is the timestamp. This method can only capture the time dependence of finite time steps. Also, due to the spatiotemporal demand imbalance, the OD matrix can be very sparse in some regions.
+
+### Spatial dependency
+- Traditional CNNs work on Euclidean structured data and lack the ability to handle the opposite. Transportation networks have spatiotemporal attributes which are well-handled by GCNs.
+
+### Temporal dependency
+- Three main architectures used: RNNs, Transformer, and TCN (Temporal Convolutional Network).
+
+### Other factors
+
+**External characteristics**
+- Taxi demand can be affected by various auxiliary factors like weather.
+
+**Model helper methods**
+- Papers have used methods like attention, multi-task learning, and ResNet networks.
+
+### Challenges
+
+**Challenge 1: Representation of dynamic correlations in OD flow**
+- It is common for the relationship between two regions to change over time (peak vs non-peak hours). Static graphs cannot capture this, so research has looked into different ways of using dynamic graphs for that purpose.
+
+**Challenge 2: Spatial-temporal correlation**
+- Spatiotemporal correlation means that each node can influence its neighbors at the next time slot. Spatiotemporal heterogeneity comes from the fact that the OD flow is different in cases like morning, evening, downtown, or city outskirts. At the time of writing, using two independent components to capture both spatial and temporal dependencies in a chained prediction often fails to capture this correlation and heterogeneity.
+
+**Challenge 3: Differenti
+
+ation of different semantics of origin and destination**
+- In complex and irregular transportation networks, passenger demand between OD pairs can be both geographically and semantically correlated, with directed and bidirectional relationships. Modeling demand separately for origins and destinations ignores the flow relationship between OD pairs and lacks practical application. Moreover, considering only the distance and flow information between grids, without distinguishing between origin and destination, overlooks the directionality of OD flows and the varying attraction relationships at different times.
+
+**Challenge 4: Time window selection**
+- The predominant method for predicting OD flows is the discrete dynamic graph approach, which aggregates historical transactions into demand snapshots within fixed time windows. This method results in disconnected OD flows and lacks rigor due to the arbitrary choice of time granularity, which can introduce noise or overlook important information. Continuous-time dynamic graph methods offer a solution by maintaining dynamic state vectors for traffic nodes, but they face challenges in updating and maintaining representations for numerous continuous-time nodes. Recent advancements include using a spatiotemporal attention network and developing a continuous-time dynamic graph framework to improve prediction accuracy and capture complex time patterns.
+
+**Challenge 5: Data sparseness problem solving**
+- The time sequence associated with each OD pair involves intricate spatial dependencies. Discrete dynamic graph-based prediction methods often result in information loss and a high frequency of zero values. Continuous-time dynamic graph-based prediction methods face challenges due to sparse data for certain OD pairs, which is exacerbated by the quadratic increase in predicted OD demand.
+
+### Going forward
+
+**Spatiotemporal dynamic correlation**
+- An individual node is affected by interactions with its surroundings, and randomness of the network itself. Currently, this is tackled using attention mechanisms, but this can be explored further.
+
+**External information addition**
+- External factors like holidays, weather, points of interest, large events, and traffic accidents can also significantly affect taxi demand. Introducing such auxiliary information is rare in current research.
+
+**Regional division**
+- Current papers split the region of interest into a set of grids or traffic zones. But this method can lack in understanding all relationships between the data, so a more rigorous approach can be developed.
+
+**OD data sparseness and data overload**
+- Data sparsity affects the model, and future research can look into how to handle such sparsity in the OD matrix.
+
+**Technical related issues**
+- Most papers use RNNs, GCNs, a few use GATs and GAEs. Future research can look into how we can use GNNs for OD demand prediction problems better.
+
+### On another note
+
+Tonight I have to stay up till 2am because I signed up to give a short PPT on [text2chart](https://github.com/divakaivan/text2chart) (which I created some time ago) in a project group organized by Stanford AI professional certificate students. I uploaded the PPT PDF to the GitHub repo.
+
+
+That is all for today!
+
+See you tomorrow :)

From 246764c2c9d84e87f218e740df4e6419629b94e5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:20:33 +0900
Subject: [PATCH 258/638] Update 2024-06-01-day-152.md

---
 _posts/2024-06-01-day-152.md | 132 ++++++++++++++++++++++++++++++++++-
 1 file changed, 130 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-01-day-152.md b/_posts/2024-06-01-day-152.md
index cadf548b6..a313d5bc5 100644
--- a/_posts/2024-06-01-day-152.md
+++ b/_posts/2024-06-01-day-152.md
@@ -1,5 +1,133 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 152) SVR & STTCM - Two architectures for taxi demand prediction
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 152!
+A quick summary of today (papers read):
+* Optimization of spatial-temporal graph: A taxi demand forecasting model based on spatial-temporal tree
+* Predicting Taxi-Out Time at Congested Airports with Optimization-Based Support Vector Regression Methods
+
+Updated graph of read papers from Obsidian:
+
+![image](https://github.com/user-attachments/assets/3fdbda4f-de92-43ff-bc84-33da6c0041d2)
+
+### The first paper introduces Spatial-Temporal Tree Convolution Model (STTCM)
+
+Introduction
+
+Spatial-temporal dependencies in taxi demand reflect changes across different locations and times. Temporally, demand follows city rhythms, being higher during rush hours and lower late at night, with variations between weekdays and weekends. Spatially, demand varies by area type, such as high demand in business districts during rush hours and in nightlife zones late at night. These dependencies interact; for instance, high morning demand in business districts is linked to the time of day, just as late-night demand in hotspots depends on it being late night. Taxi demand is influenced by multiple historical sequences and spatial dependencies. Temporally, demand is related to previous adjacent moments and shows periodic changes. Historical inflows also impact future demand, as passengers often return by taxi. Spatially, regions close to each other tend to have similar demand patterns and more taxi flows (picture). However, demand can also be affected by distant areas with potential spatial dependencies, such as high morning demand from residential to work areas and evening demand in the opposite direction on weekdays.
+
+![image](https://github.com/user-attachments/assets/de7e6bd5-0d6a-4503-bb59-3c250398d1cc)
+![image](https://github.com/user-attachments/assets/eb804108-514d-4958-9c1e-77b4a8996864)
+![image](https://github.com/user-attachments/assets/8d0378b4-698d-4ed4-b24b-2ed132070652)
+![image](https://github.com/user-attachments/assets/810a5415-691b-43e9-ad8d-96b946cb12eb)
+![image](https://github.com/user-attachments/assets/1c1d9712-5f53-4337-9dcd-94ecd15fa067)
+
+Definition of a tree matrix
+
+![image](https://github.com/user-attachments/assets/0e29226a-d8c3-4a48-896a-6f8adaf68195)
+![image](https://github.com/user-attachments/assets/38f28027-3373-4310-88a9-4109c3929c6a)
+![image](https://github.com/user-attachments/assets/58fd3fba-c19b-4799-a9ce-5d9a66c529ba)
+![image](https://github.com/user-attachments/assets/d3f912db-b544-42ca-877a-985c9e049b71)
+
+Model design
+
+The design consists of four modules that capture the above defined four spatio-temporal characteristics.
+
+![image](https://github.com/user-attachments/assets/3b9288a2-7a18-45a8-8379-3ebd04b47576)
+![image](https://github.com/user-attachments/assets/bb68e9f4-1d6f-4081-94eb-8712e6d250f5)
+
+Spatial-temporal convolution module
+
+For spatial characteristics, GCN is widely used but its performance falls off when it comes to discerning the local path features and the hierarchical features among different regions. In terms of temporal characteristics - the LSTM is widely used. However, its iterative training process is time-consuming.
+This paper introduces a novel block - spatio-temporal convolutional block. It consists of a temporal gate convolution layer (for temporal characteristics) and a tree convolution layer (for spatial characteristics).
+
+Temporal Gated Convolution Layer
+
+The CNN captures the influence of adjacent time-step features on the current time-step features, and then the GLU further improves the non-linearity of the data.
+
+Spatial Tree Convolution Layer
+
+Tree convolution aims to construct a multi-layer convolutional structure for tree matrices to fully explore spatial relationships between nodes. The spatial tree matrix consists of multiple plane tree matrices, each processed independently by convolution. The process starts with inputting the bottom two rows of a plane tree matrix into a CNN to obtain hidden layer node features. These features are then weighted and fused with the original features to preserve information. This process repeats until all layers' features are fused, resulting in high-level features for each plane tree matrix. The final output combines these high-level features from all plane tree matrices.
+
+Tree convolution uses a [2 × 1] convolution kernel to translate from the bottom to the top of a tree matrix, capturing local path and hierarchical features among nodes. Each column in the tree matrix, generated by the BFS algorithm, stores connectivity paths from the root to leaf nodes. Each path undergoes an independent convolution process, transforming connectivity features into high-level features without intersecting with other paths. The high-level features from all paths are concatenated to represent local path features. This method ensures independent convolutional space for each path, preventing noise from other paths and enabling accurate analysis of latent attributes. This approach offers a more precise understanding of potential features compared to traditional aggregate-style convolution methods. 
+
+Tree convolution captures hierarchical features using a plane tree matrix generated by BFS. This matrix clearly stores hierarchical relationships among nodes, with the root node in the first row, directly connected nodes in the second row, and indirectly connected nodes in subsequent rows based on the number of indirect connections. The convolution kernel translates row-by-row, starting from layers with more indirect connections. This process allows tree convolution to interpret both direct and indirect hierarchical relationships effectively. 
+
+During conventional convolution, some information loss is inevitable, especially noticeable when processing data from root nodes and their direct connections in a graph structure. Root nodes and directly connected nodes retain more relevant information due to their limited participation in the convolution process, leveraging the convolution's inherent properties to reduce information loss and enhance accuracy and robustness. This hierarchical processing strategy forms higher-level features through tree convolution, incorporating varying degrees of hierarchical relationships between nodes. This approach allows for granular feature extraction, effectively revealing complex hidden information within node data in graph structures.
+
+Forecasting process
+
+![image](https://github.com/user-attachments/assets/a3a5f81a-aece-4490-b8b3-b00cf162a7d1)
+
+Experiment
+
+![image](https://github.com/user-attachments/assets/bc2edce0-2d29-4e6f-ad6e-c56dd61f3b2b)
+
+Results
+
+![image](https://github.com/user-attachments/assets/2cdbb1cc-10dd-4946-8031-178ca1f40ee0)
+
+Future research
+
+* There is inherent sparsity in the constructed tree matrix.
+* Incorporating auxiliary factors like holidays and weather conditions, can also form ideas for future improvements upon STTCM.
+* Incorporating new structures that can better capture and reflect time-varying trends in inter-regional travel flows and traffic flows within urban road networks.
+
+### The second paper was referenced in the above one so I decided to read it as well - talking about Support Vector Regression
+
+Introduction
+
+Taxi-out time refers to the duration an aircraft spends moving on the ground from the departure gate to the takeoff runway. It starts when the aircraft is pushed back from the gate and ends when the aircraft's wheels lift off the ground during takeoff.
+
+This period includes various activities such as taxiing to the runway, waiting in line for takeoff clearance, and any other ground movements necessary before departure. It's an essential factor in understanding airport operations and flight efficiency since longer taxi-out times can contribute to delays and congestion, affecting both airport capacity and overall flight punctuality.
+
+Taxi-out time prediction techniques
+
+Generalised Linear Model
+
+GLM is a flexible generalisation of OLS that allows for response variables with error-distribution models other than normal distribution. GLM relates the linear model to the response variables through a link function and by allowing the magnitute of the variance of each measurement to be a function of its predicted value.
+
+Softmax Regression Model
+
+SR is a generalisation of logistic regression allowing for multiple (more than two) output classes.
+
+Artificial Neural Network
+
+Improved Swarm Intelligence Algorithm Based Prediction Approaches
+
+Support Vector Regression
+
+SVR is a non-linear regression forecasting method where the input variables are mapped into a high-dimensional linear feature space, commonly through a kernel function. In this higher dimensional space, the training data can be approximated to a linear function, and the global optimal solution is obtained by training of the finite sample.
+
+Particle Swarm Optimization
+
+PSO solves an optimization problem by moving the particles (candidate solutions) over those particles' velocities and positions according to simple mathematical formulae. The position of each particle is updated towards the better-known position driven by its neighbours’, and the global, best performance.
+
+Improved Firefly Algorithm Optimization
+
+The Firefly Algorithm (FA) is a new approach for solving optimization problems efficiently. It works by simulating the behavior of fireflies. In this algorithm, fireflies are attracted to each other based on their brightness and attraction levels. Brightness depends on their location and target value, with higher brightness indicating a better location. Fireflies with higher brightness also have a stronger attraction. If fireflies have similar brightness, they move randomly. Overall, FA is effective in solving optimization problems and can outperform traditional algorithms like Genetic Algorithms (GA).
+
+PSO/IFA Based Support Vector Regression
+
+This study combines swarm intelligence algorithms with SVR for better prediction accuracy. Since the learning data has many more samples than features, the input variables are transformed into Hilbert space using the RBF kernel, which is more effective than other kernels. To accurately predict departure taxi-out time, SVR models need optimal values for penalty factor, RBF kernel parameter, and epsilon. PSO and IFA are used to determine these values beforehand. Incorrect settings could affect training errors, induce overfitting, or lead to less effective learning. 
+
+Performance Measures
+
+RMSE, MAPE, squared correlation coefficient, prediction accuracy
+
+
+
+Data
+
+![image](https://github.com/user-attachments/assets/5f2c7f44-97c4-4a9a-8256-2e4d3c3e34a4)
+
+Results
+
+![image](https://github.com/user-attachments/assets/047f2e9b-43b0-45d6-bb49-c86ef0a69ed6)
+
+That is all for today!
+
+See you tomorrow :)

From 32e4ef99166d41f4f525c475a8ae32d5dd7d4ad3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:24:05 +0900
Subject: [PATCH 259/638] Update 2024-06-02-day-153.md

---
 _posts/2024-06-02-day-153.md | 56 ++++++++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-02-day-153.md b/_posts/2024-06-02-day-153.md
index cadf548b6..383d93198 100644
--- a/_posts/2024-06-02-day-153.md
+++ b/_posts/2024-06-02-day-153.md
@@ -1,5 +1,57 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 153) First steps into orchestration and ML pipelines (module 3 from MLOps zoomcamp)
+categories: [applying-knowledge,mlops]
 ---
+
+## Hello :) Today is Day 153!
+A quick summary of today:
+* started [Module 3](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/03-orchestration): Orchestration and ML pipelines from the MLOps zoomcamp
+
+It is my 26th birthday today, so my study time was pretty limited and studied during my 2 hour bus rider home from Seoul. 
+
+The below is the full outline
+
+![image](https://github.com/user-attachments/assets/fc4f7a3c-bbf3-4a2a-84b8-0e9429135e72)
+
+I only got to complete 3.1. Data preparation, and below are some pics/notes I took.
+So before this, in the MLOps zoomcamp we covered mlflow, experiment tracking and model management. I guess orchestration is the next step which automates the whole data prep, training, testing process (note: I am not sure what else so far). 
+
+The 2024 cohort of the camp, uses [mage.ai](http://mage.ai/) as a free-to-use platform, so today's and the next steps will be done on its platform. 
+
+First it was setup using docker. 
+
+git clone https://github.com/mage-ai/mlops.git
+
+cd mlops
+
+./scripts/start.sh
+
+And a local mage.ai webapp was started
+
+![image](https://github.com/user-attachments/assets/6618f7af-e36e-4282-86e9-e393ff6f774a)
+![image](https://github.com/user-attachments/assets/0854b138-84d1-44a5-80e3-0858ff6cccde)
+
+In the 3.1 part we created a data preparation pipeline, and here is the final version:
+
+![image](https://github.com/user-attachments/assets/f43a4810-3568-4581-b14d-99e8ce7ab867)
+
+It is not hard to use, and fairly easy to set up. Each block is a piece of code. For example the ingest block is:
+
+![image](https://github.com/user-attachments/assets/7852e581-cdb4-4237-a5bb-ccb99c0f9320)
+
+It downloads taxi data. Then when we create a next block, following it, it seems to automatically chain outputs from previous to inputs of current block. Below is the second 'prepare' block:
+
+![image](https://github.com/user-attachments/assets/adc72274-11f4-426b-942a-b050cd7db5a0)
+
+And the final 'build' block is where we have the dataset split and data vectorized (using util functions)
+
+![image](https://github.com/user-attachments/assets/d057de79-1b8d-493d-9736-1ce479c69b6e)
+
+I will try to complete the rest of the module in the coming week ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 3d4c41b47a41ecbb3d3ef23e32725e1240d4983b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:28:50 +0900
Subject: [PATCH 260/638] Update 2024-06-03-day-154.md

---
 _posts/2024-06-03-day-154.md | 152 ++++++++++++++++++++++++++++++++++-
 1 file changed, 150 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-03-day-154.md b/_posts/2024-06-03-day-154.md
index cadf548b6..33634caef 100644
--- a/_posts/2024-06-03-day-154.md
+++ b/_posts/2024-06-03-day-154.md
@@ -1,5 +1,153 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 154) Diving deeper into Graph Neural Networks used in taxi demand prediction
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 154!
+A quick summary of today:
+* STGCN - Spatio-Temporal Graph Convolutional Networks: A Deep Learning Framework for Traffic Forecasting
+* CACRNN - Predicting taxi demands via an attention-based convolutional recurrent neural network
+
+Thanks to a lab mate, I found a way to sync my Obsidian notes using google drive. So I don't need to use github anymore! But still I have to upload text with math notations as pictures.
+
+![image](https://github.com/user-attachments/assets/71632e4e-57bf-4598-922c-c82121540d26)
+
+### Frist paper, STGCN
+
+Introduction
+
+The paper presents methods to effectively capture the temporal and spatial patterns in traffic flow. Instead of viewing the traffic network as separate grids or segments, it represents it as a general graph to better leverage spatial data. To address the shortcomings of recurrent networks, a fully convolutional structure along the time axis is used. The key contribution is the development of a novel deep learning model, called spatio-temporal graph convolutional networks, specifically designed for traffic forecasting. This model consists of spatio-temporal convolutional blocks that integrate graph convolutional layers with convolutional sequence learning layers, enabling the simultaneous modelling of spatial and temporal dependencies. This approach is, to our knowledge, the first to apply purely convolutional structures for extracting spatio-temporal features from graph-based time series in traffic analysis.
+
+Preliminaries
+
+![image](https://github.com/user-attachments/assets/b6b91a6e-9563-465b-b929-d3419683a937)
+![image](https://github.com/user-attachments/assets/f908a05e-ab58-46eb-9aec-0bf3f24473b5)
+
+Proposed model
+
+Network architecture
+
+![image](https://github.com/user-attachments/assets/d16bdd9d-8537-44a7-935a-398551d36465)
+
+Graph CNNs for extracting spatial features
+The Theta formula from the previous section, can be expensive as O(n^2), but two approximation strategies are applied to overcome this issue.
+
+Chebyshev polynomials approximation
+From O(n^2) we can go to O(K|E|).
+
+1st-order approximation
+
+They are a simplified way to perform graph convolutions, making them efficient for large-scale graphs. They simplify the convolution operation by merging two parameters into one and normalizing the graph's adjacency matrix. The result is a deep architecture can be created to capture spatial information efficiently, with each layer using information from neighbouring nodes up to a certain depth (defined by the number of layers, K).
+
+Generalisation of graph convolutions
+
+![image](https://github.com/user-attachments/assets/3b515894-8d47-462f-bc5a-9424ab9f1977)
+
+Graph convolutions can handle multi-channel and multi-frame data, making them suitable for complex inputs like traffic prediction data.
+
+Gated CNNs for extracting temporal features
+The popular for time-series analysis, suffer when it comes to traffic predictions due to their time-consuming nature and inability to respond to dynamic changes. CNNs, on the other hand, have fast training, simple structure, and no dependency constraints to previous steps.
+
+Spatio-temporal convolutional block
+This block fuses both spatial and temporal features. The sandwich structure (middle picture from the network architecture) helps the network sufficiently apply bottleneck strategy to achieve scale compression and feature squeezing by downscaling and upscaling of channels C through the graph convolutional layer.
+
+Model summary
+* STGCN is a universal network that can handle any type of spatio-temporal sequence learning task
+* The spatio-temporal block combines graph convolutions and gated temporal convolutions, which extract spatial and temporal features, respectively
+* The model is entirely composed of convolutional structures and therefore achieves parallelisation over input with fewer parameters and faster training speed. Also, due to the approximations included, large-scale networks can be handled as well.
+
+Experiment
+
+Dataset
+
+Using two real-world traffic datasets - BJER4 (Beijing) and PeMSD7 (California).
+
+![image](https://github.com/user-attachments/assets/5aaf942e-65a9-47ef-8976-bb214c85bd6f)
+
+Settings
+
+Only workday traffic data is used to eliminate atypical traffic (Li et al., 2015). Grid search is used to locate the best parameters on validations. All tests use 60 minutes as a historical time window (i.e. 12 observed data points; M=12 are used to forecast traffic conditions in the next 15, 30, and 45 minutes)
+
+Evaluation metrics
+
+MAE, MAPE, RMSE
+
+Baseline models
+
+HA, Linear SVR, ARIMA, FNN, FC-LSTM, GCGRU
+
+Results
+
+![image](https://github.com/user-attachments/assets/68b56a27-19b8-4b88-a9d1-889ee5d9239b)
+![image](https://github.com/user-attachments/assets/67db3122-49f2-4554-b75d-74f9596ca88d)
+![image](https://github.com/user-attachments/assets/59a36208-688c-4475-9ea6-cc0adce20201)
+
+### Second paper, CACRNN
+
+Introduction
+
+This paper proposes a context-aware attention-based convolutional recurrent neural network (CACRNN) for predicting fine-grained taxi demand in an urban area.
+
+Key features include urban area partitioning: using morphology-based map segmentation, the urban area is divided into fine-grained regions; and for region, three taxi demand predictions are made, considering various spatio-temporal dependencies and external factor. The proposed model combines local convolutional layers and GRUs to capture spatial and temporal data characteristics. It also uses RNNs to identify short- and long-term periodic taxi demand patterns.
+
+Problem formulation
+
+Definition 1: road network
+
+A road network of an urban area is composed of a set of road segments. Each road segment is associated with two terminal points (i.e. intersections of crossroads), and connects with other road segments by sharing the same terminals. All road segments compose the road network in the format of a graph.
+
+Time is split into equal interval time slots, and the whole urban area is divided into disjoint regions based on its road network. Each region is an irregular polygon, encompassed by several road segments.
+
+![image](https://github.com/user-attachments/assets/2ad769c8-93b4-45f1-b1a2-da20528a4e70)
+
+Data
+
+Road network data
+
+![image](https://github.com/user-attachments/assets/c6e4faa2-a213-4601-8841-d26fd401c464)
+
+Taxi trip data
+
+![image](https://github.com/user-attachments/assets/b1916606-a772-4fb8-a8f6-e5b13a2dd8fb)
+![image](https://github.com/user-attachments/assets/6466ef2b-e0ba-4748-9160-c77a2f856fac)
+![image](https://github.com/user-attachments/assets/ee420e07-de4d-41b7-9d07-5ef505b83164)
+
+POI data
+
+![image](https://github.com/user-attachments/assets/10fb615f-7917-4a5b-b628-93a1077dd213)
+![image](https://github.com/user-attachments/assets/b3d46ad1-cc38-4ed9-ac1a-0b7abebfa095)
+
+Meteorological and holiday data
+
+![image](https://github.com/user-attachments/assets/78b314a7-c0be-4b03-b0f9-e775119f25b7)
+
+Methodology
+
+![image](https://github.com/user-attachments/assets/63515a3c-8e89-4b1c-9738-eca335960bd9)
+![image](https://github.com/user-attachments/assets/96f238da-c06e-47b5-9764-7b2ef5e4fcb9)
+
+Definition 6: Functional similarity
+
+Two regions evaluated whether they are similar based on cosine-similarity of their embedding vector (containing significance of POIs in a specific category c, measured by TF-IDF).
+
+![image](https://github.com/user-attachments/assets/c059a87f-1d70-4a9f-92f9-4fd92f5d463d)
+![image](https://github.com/user-attachments/assets/9aa72adb-484e-4d2f-b3e1-b9dc779d5ee8)
+
+Experiments
+
+Baseline models
+HA, ARIMA, LSTM, DCRNN, STGCN
+Results
+
+Also compared with different variations of the CACRNN
+
+![image](https://github.com/user-attachments/assets/69a9b48c-59f2-40fe-bc82-8e5f9b67f9c7)
+
+Finally, thanks to my lab professor and lab mates for the birthday cake
+
+![image](https://github.com/user-attachments/assets/b1028c19-7838-48c7-ba65-4a4d48c45cb2)
+
+That is all for today!
+
+See you tomorrow :)

From 9eba26e08b2d60f9b3b21ca3457e6b3fbba6232c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:37:42 +0900
Subject: [PATCH 261/638] Update 2024-06-04-day-155.md

---
 _posts/2024-06-04-day-155.md | 276 ++++++++++++++++++++++++++++++++++-
 1 file changed, 274 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-04-day-155.md b/_posts/2024-06-04-day-155.md
index cadf548b6..da901d354 100644
--- a/_posts/2024-06-04-day-155.md
+++ b/_posts/2024-06-04-day-155.md
@@ -1,5 +1,277 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 155) Reading more about 'historic' (used as baseline) models for spatio-temporal predictions using graphs
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 155!
+A quick summary of today's papers:
+* Real-time Prediction of Taxi Demand Using Recurrent Neural Networks
+* Graph WaveNet for Deep Spatial-Temporal Graph Modeling
+* GMAN: A Graph Multi-Attention Network for Traffic Prediction
+* DNN-Based Prediction Model for Spatio-Temporal Data
+
+The updated graph with reference connections from Obisdian is:
+
+![image](https://github.com/user-attachments/assets/5c24291c-a61d-4258-a3c7-38a2d1146e13)
+
+### Real-time Prediction of Taxi Demand Using Recurrent Neural Networks
+
+Introduction
+
+This paper proposes a real-time method for predicting taxi demands in different areas of a city. A big city is divided into smaller areas and during a pre-set period of time, the number of taxi requests in each area is aggregated. This way taxi data becomes a sequence and a LSTM is applied. LSTM is capable of learning long-term dependencies by utilising some gating mechanisms to store information. Therefore, it can for instance remember how many people have requested taxis to attend a concert and after a couple of hours use this information to predict that the same number of people will request taxis from the concert location to different areas. Simply learning the average to predict will not result in a very good model, so the paper adds Mixture Density Networks (MDN) on top of the LSTM. This way, instead of directly predicting a demand value, the output is a mixture distribution of the demand. 
+
+Mathematical model
+
+GPS data encoding
+
+A city is split into 153m x 153m area sizes. Then past taxi data is converted into week-long data sequences, which are the input to the LSTM.
+
+RNNs
+
+![image](https://github.com/user-attachments/assets/1453ac9d-cfb1-4aa4-a590-3bffb080bba3)
+
+Taxi demand prediction model
+
+Mixture density networks
+
+MDNs can be used in prediction applications in which an output may have multiple possible outcomes. This paper's model outputs the parameters of a mixture model - the mean and variance of each Gaussian kernel and also mixing coefficient of each kernel which shows how probable the kernel is.
+
+potential gold
+
+LSTM-MDN sequence learning model
+
+![image](https://github.com/user-attachments/assets/23dc4c65-a056-4cd4-83c1-2f51e2376568)
+![image](https://github.com/user-attachments/assets/e20e5cf8-7700-4375-a2c8-bb71c22de925)
+![image](https://github.com/user-attachments/assets/2117d8fc-573c-481f-919a-9af670f86869)
+
+LSTM-MDN-Conditional model
+
+In the LSTM-MDN model, the probability distribution of taxi demands in all areas are predicted at the same time-step. This means that prediction in each area is conditioned on all areas of all previous time-steps. However, the taxi demand in an area might not only be related to the past, but also to the taxi demands of other areas in current time-step. So instead of outputting a joint distribution for all areas in a single time step, the network predicts the conditional distribution of each area at a single time-step.
+
+![image](https://github.com/user-attachments/assets/843cf4d2-8b3b-4f40-8559-8a090b7613a7)
+
+This approach is called Conditional, and it has the same input as the LSTM-MDN, but each x only leads to one area taxi demand output. This model predicts the demand for a particular area in a sequential manner.
+
+![image](https://github.com/user-attachments/assets/a4e4d6ed-8b3b-422d-bbca-ca66fe49e6ee)
+![image](https://github.com/user-attachments/assets/0bb89990-4fc2-4a94-84ee-d6fbe6d44db7)
+
+Experiments
+
+Metrics
+sMAPE, RMSE
+
+Baselines
+FC-FFNN, Naive statistic average predictor
+
+Results
+
+![image](https://github.com/user-attachments/assets/1deecbc4-913d-47a6-8738-5b21c372324f)
+![image](https://github.com/user-attachments/assets/a1fd612d-5b23-4c06-9ef1-eba2dcfea043)
+![image](https://github.com/user-attachments/assets/fc6f013a-a676-4dbe-a8f8-dbf46a10873c)
+
+### Graph WaveNet for Deep Spatial-Temporal Graph Modeling
+
+Introduction
+
+![image](https://github.com/user-attachments/assets/e03d3332-bba4-4981-87ee-8d010ed46f40)
+
+A basic assumption behind spatial-temporal graph modelling is that a node’s future information is conditioned on its historical information as well as its neighbours’ historical information. Paper contributions:
+
+* A self-adaptive adjacency matrix is constructed to preserve hidden spatial dependencies. This matrix automatically uncovers unseen graph structures from the data without prior knowledge. Experiments show that this method improves results when spatial dependencies exist but are not provided.
+* Additionally, an effective and efficient framework is presented to capture spatial-temporal dependencies simultaneously. The core idea is to combine the proposed graph convolution with dilated causal convolution, allowing each graph convolution layer to address the spatial dependencies of nodes’ information extracted by the dilated causal convolution layers at different granular levels.
+
+Methodology
+
+Problem definition
+
+Given a graph G and S historical time steps, predict the next T time steps.
+
+Graph convolution layer
+
+The paper proposes a novel self-adaptive adjacency matrix, which does not require any prior knowledge and is learned end-to-end through SGD. This way the model can learn hidden spatial dependencies by itself.
+
+![image](https://github.com/user-attachments/assets/de32af54-615f-4477-bd4d-3371bea74d1b)
+
+where E_x are randomly initialised two node embedding dictionaries.
+Temporal convolution layer
+
+Dilated causal convolution are used as TCN to capture temporal trends. Dilated causal convolution networks expand the receptive field exponentially by increasing layer depth. Unlike RNN-based methods, these networks can effectively manage long-range sequences in a non-recursive manner, enabling parallel computation and reducing the risk of gradient explosion. Dilated causal convolution maintains temporal causal order by padding inputs with zeros, ensuring that predictions at the current time step only use historical data. As a variant of standard 1D convolution, the dilated causal convolution operation skips values at regular intervals while sliding over the inputs.
+
+![image](https://github.com/user-attachments/assets/a78f772e-4013-4e8f-ad09-8852b21d878e)
+
+Graph WaveNet framework
+
+![image](https://github.com/user-attachments/assets/c5691fdd-b62b-4bd7-9c0d-659c3dc690d8)
+
+By stacking multiple spatial-temporal layers, Graph WaveNet is able to handle spatial dependencies at different temporal levels.
+Experiments
+
+Datasets
+
+METR-LA and PEMS-BAY
+
+Baseline models
+
+ARIMA, FC-LSTM, WaveNet, DCRNN, GGRU, STGCN
+
+Results
+
+![image](https://github.com/user-attachments/assets/fb24567a-4ede-40bf-9631-acb96be79f40)
+![image](https://github.com/user-attachments/assets/38765a13-2d07-4096-91e3-233704332308)
+
+The impact of the self-adaptive adjacency matrix is examined through experiments conducted with Graph WaveNet, utilizing five different adjacency matrix configurations. Results indicate that even in the absence of prior graph structural knowledge, the adaptive-only model outperforms the forward-only model in terms of mean MAE. The forward-backward-adaptive model demonstrates the lowest scores across all evaluation metrics, suggesting that incorporating the self-adaptive matrix enhances model performance when graph structural information is provided. Further analysis of the learned matrix reveals variations in node influence, with nodes near intersections exerting more influence compared to those along single roads.
+
+![image](https://github.com/user-attachments/assets/8e8d64a5-054d-455b-8b7f-18ce3939e518)
+
+### GMAN: A Graph Multi-Attention Network for Traffic Prediction
+
+Introduction
+
+![image](https://github.com/user-attachments/assets/7adb004e-1317-4ec7-b7ab-153d52f3cd84)
+
+Complex spatio-temporal correlations
+
+* The correlations of traffic conditions of a particular road change over time. It is difficult to pick out relevant sensors' data that will be useful to predict a target sensor's traffic conditions in the long-term.
+* Non-linear temporal correlations. In cases of accidents, conditions may change suddenly. It is difficult to create an adaptive model to such non-linear temporal correlations.
+
+Sensitivity to error propagation
+
+* Small errors in each time step stack onto each other and can end up as big problems in a future step.
+
+To that end, Graph Multi-Attention Network (GMAN) is proposed to predict traffic conditions on a road network graph over some time steps ahead. GMAN can be applied to any numerical traffic data, but this paper concentrates on traffic volue and traffic speed.
+
+Related work
+
+Traffic prediction
+
+Popular historical approaches: LSTM, ARIMA, SVR, KNN, CNN, GCN.
+
+Deep learning on graphs
+
+Attention mechanism
+
+Graph multi-attention network
+
+![image](https://github.com/user-attachments/assets/7adb8f75-570c-41a4-b41e-c98af270b854)
+
+Spatio-temporal embedding
+
+Using node2vec, vertices are converted into vectors that preserve the graph structure information.
+
+For temporal embedding, each time step is encoded into a vector.
+
+The above are combined as a spatio-temporal embedding (2b).
+
+ST-Attention block
+
+Includes spatial attention, temporal attention and gated fusion.
+
+Spatial attention
+
+![image](https://github.com/user-attachments/assets/520523f5-6c48-4e81-be85-16d7f677bd77)
+
+A road's traffic condition highly dynamic, and affected by other roads with different weights. This mechanism is proposed to capture such complex properties. The key idea is to dynamically assign different weights to different vertices (e.g. sensors) at different time steps. At any time step, both the traffic conditions and the road network structure could affect the correlations between sensors (i.e. a congestion on one road can affect the traffic on nearby roads).
+
+![image](https://github.com/user-attachments/assets/be9b422e-84e9-47c8-bfd0-28fff4956753)
+
+When the number of vertices (sensors) gets large, the time and memory consumption is large. To tackle this issue group spatial attention is proposed which contains intra-group spatial attention and inter-group spatial attention.
+
+Temporal attention
+
+![image](https://github.com/user-attachments/assets/b28d5933-45d1-440a-b019-0d3d47f3ac9d)
+
+The traffic condition at time t is correlated with times before t as well, and this correlation can vary over time (it is non-linear). To measure the relationship between different time steps both traffic features and times are considered.
+
+Gated fusion
+
+The traffic condition of a road at a certain time step is correlated with both its previous values and other roads’ traffic conditions. As seen in picture 2c, spatial and temporal embeddings are fused together.
+
+Transformer attention
+
+To mitigate error propagation across different prediction time steps in a long-term forecast, a transformer attention layer between the encoder and decoder is introduced. This layer captures the direct relationships between each future time step and all historical time steps, transforming the encoded traffic features to produce future representations, which serve as the decoder's input.
+
+![image](https://github.com/user-attachments/assets/bdc0ff30-c84b-4c24-be87-923accf50806)
+![image](https://github.com/user-attachments/assets/6f636edf-4ef1-4a1b-86c2-c52d0bf7454e)
+
+Experiments
+
+Datasets
+
+![image](https://github.com/user-attachments/assets/6f2dbdbb-e2af-4ad4-b35f-919f74b56c30)
+
+Metrics
+
+MAE, RMSE, MAPE
+
+Hyperparametres
+
+Baselines
+
+ARIMA, SVR, FNN, FC-LSTM, STGCN, DCRNN, GraphWaveNet
+
+Results
+
+![image](https://github.com/user-attachments/assets/ffa9e35d-d97d-4958-b1a1-2784075394c0)
+
+Fault tolerance comparison
+
+![image](https://github.com/user-attachments/assets/46e0dd47-05f6-4af9-8d00-3604f9b0f737)
+
+The real-time values of traffic conditions may be missing partially, due to sensor malfunction, packet losses during data transmission, etc. To evaluate the fault-tolerance ability, a fraction of historical observations is randomly dropped to make 1 hour ahead predictions
+
+Effect of each component
+
+![image](https://github.com/user-attachments/assets/781e34c5-193a-49f3-a1de-4e0ec7e99e90)
+
+To investigate the effect of each component in GMAN, four variants are evaluated - by removing spatial attention, temporal attention, gated fusion, and transform attention from GMAN separately, which are named as GMAN-NS, GMAN-NT, GMAN-NG, and GMAN-NT, respectively.
+
+### DNN-Based Prediction Model for Spatio-Temporal Data
+
+Introduction
+
+Different from text and image data, ST data has two unique attributes: 1) spatial properties, which consists of a geographical hierarchy and distance, and 2) temporal properties, which consists of closeness, period and trend.
+
+Models
+
+Definitions
+
+Region
+
+* a city is partitioned into an M x N grid map based on the longitude and latitude where a grid represents a region
+
+Measurement
+
+* crowd flows are used. For a grid that likes at the m-th row and n-th column, two types of crowd flows at the k-th timestamp - in-flow and out-flow
+
+DeepST
+
+![image](https://github.com/user-attachments/assets/fecfdaf7-b25d-4dfb-a3bc-5868ae36a70a)
+
+All historical observations are fed into the spatio-temporal component. According to temporal properties, different timestamps are selected and concatenated together to model closeness, period and trend, respectively. The outputs of the three are further combines via early fusion, feeding them into a number of convolutional layers. In the global component, we first get metadata of the predicted time t, and transform it into a binary vector, which is then fed into a block that contains one (or a few) fully-connected layer(s). The outputs of the above components are merged via late fusion followed by tanh.
+
+Evaluation
+
+Datasets
+
+![image](https://github.com/user-attachments/assets/c247b156-5906-431d-a953-ee05596765e7)
+
+Models
+
+![image](https://github.com/user-attachments/assets/ce926ff7-84fc-4872-9369-3ca02564dd35)
+
+Evaluation metric
+
+RMSE
+
+Results
+
+![image](https://github.com/user-attachments/assets/64326325-6ad1-4786-bbed-81c215955d3f)
+
+More data is better
+
+![image](https://github.com/user-attachments/assets/92c4c761-0a8d-494a-bbb3-cb836d7e8426)
+
+That is all for today!
+
+See you tomorrow :)

From 04b557424f8e9fc21f50fd285509edd3ed7e15b1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:42:15 +0900
Subject: [PATCH 262/638] Update 2024-06-05-day-156.md

---
 _posts/2024-06-05-day-156.md | 160 ++++++++++++++++++++++++++++++++++-
 1 file changed, 158 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-05-day-156.md b/_posts/2024-06-05-day-156.md
index cadf548b6..73fc89658 100644
--- a/_posts/2024-06-05-day-156.md
+++ b/_posts/2024-06-05-day-156.md
@@ -1,5 +1,161 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 156) Final XCS224W - ML with Graphs homework
+categories: [gnn,applying-theory,reading-research]
 ---
+
+## Hello :) Today is Day 156!
+A quick summary of today:
+* finished and submitted the last assignment of the Stanford XCS224W: ML with graphs course
+* read a few more papers about spatial-temporal graph neural networks
+
+As with the previous ones, I am not allowed to share anything related to the homework. All I will share is that I got the full score
+
+![image](https://github.com/user-attachments/assets/df47e7c9-7549-40c2-8d58-3a06e7e63cf5)
+
+### As for the new papers I read today
+
+I found about [pytorch-geometric-temporal](https://pytorch-geometric-temporal.readthedocs.io/en/latest/modules/root.html). It is an extension of the pytorch-geometric library that implements popular spatial-temporal GNN papers' architectures. 
+
+![image](https://github.com/user-attachments/assets/b4d8308d-00ac-4bde-b8fd-739c7445a939)
+
+I feel like I do not want this blog to be filled with posts on the same topic 5 out of 7 days of the week (at least that is how I feel right now as I am doing literature review on spatial-temporal GNNs' application in intelligent transportation systems), so I will try to do some extra studying after my lab and share it here. I will still share what I do in the lab and what I study there, but it will be just a part of a blog post, rather than the whole thing.
+The reference graph is getting bigger and bigger, soon nodes might become to small to see.
+
+![image](https://github.com/user-attachments/assets/6f817047-5119-4909-80bb-605a827cac3e)
+
+Today I read the papers the implementation of which is in pytorch-geometric-temporal and I had not covered so far.
+* Adaptive Graph Convolutional Recurrent Network for Traffic Forecasting
+
+Introduction
+
+The paper proposes two mechanisms that build upon the popular GCN
+
+* A node adaptive parameter learning (NAPL) module, which learns node-specific patterns for each traffic series. NAPL factorises the parameters in traditional GCN and generated node-specific parameters from a weights pool and bias pool shared by all nodes according to the node embedding
+* A data adaptive graph generation (DAGG) module, which infers the node embedding (attributes) from data and generates the graph during training.
+* NAPL and DAGG are combined with recurrent networks and Adaptive Graph Convolutional Recurrent Network is proposed.
+
+Experiments
+
+Datasets
+
+PeMSD4, PeMSD8
+
+Baselines
+
+HA, VAR, GRU-ED, DSANet, DCRNN, STGCN, ASTGCN, STSGCN
+
+Metrics
+
+RMSE, MAE, MAPE
+
+Results
+
+![image](https://github.com/user-attachments/assets/3fbd611c-6af7-4480-939e-460630e5d57d)
+
+* Attention Based Spatial-Temporal Graph Convolutional Networks for Traffic Flow Forecasting
+
+Introduction
+
+ASTGCN consists of three independent components designed to model different temporal properties of traffic flows: recent, daily-periodic, and weekly-periodic dependencies. Each component has two main parts: 1) a spatial-temporal attention mechanism that effectively captures dynamic spatial-temporal correlations in traffic data, and 2) a spatial-temporal convolution that uses graph convolutions to capture spatial patterns and standard convolutions to describe temporal features. The outputs of these three components are then weighted and fused to produce the final prediction results.
+
+Observations close to each other in space and time are dynamically correlated and to solve traffic problems, effectively extracting spatial-temporal correlations of traffic flows is essential.
+
+![image](https://github.com/user-attachments/assets/5245bccf-a346-44a5-ab29-005d4dae2ed7)
+
+In the figure, the darker the colour line is, the stronger the connection and influence on each other. Also depending on time, a location's influence varies as well.
+CNN and GCN models still fail to capture the spatial-temporal features that come from dynamic traffic data, so the proposed method ASTGCN aims to collectively predict traffic flow at every location on the traffic network.
+
+Attention based spatial-temporal GCN
+
+![image](https://github.com/user-attachments/assets/4fe911de-8b69-4404-ae04-a6c6cd3ec118)
+
+The structure consists of three independent components with the same structure, which are designed to model recent, daily-periodic, and weekly-periodic dependencies of the historical data.
+
+Suppose the current time is t0, and the size of the predicting window is T_p.
+
+![image](https://github.com/user-attachments/assets/d24cafd2-6eed-48b0-8aea-2a3642ff8183)
+
+Three time series segments of length T_h, T_d, and T_w along the time axis are intercepted and used as the input of the recent, daily-period, weekly-period component respectively.
+
+The three components have the same network structure and each consist of several spatial-temporal blocks and a fully-connected layer.
+
+Spatial-Temporal Attention
+Spatial attention
+In the spatial dimension, the traffic conditions of different locations have influence among each other and the mutual influence is highly dynamic.
+
+Temporal attention
+In the temporal dimension, there exist correlations between the traffic conditions in different time slices, and the correlations are also varying under different situations.
+
+Spatial-Temporal Convolution
+
+![image](https://github.com/user-attachments/assets/d463ead2-8a11-4e94-b123-79f9904a70af)
+
+The spatial-temporal attention module let the network automatically pay relatively more attention on valuable information. The input adjusted by the attention mechanism is fed into the spatial-temporal convolution module. The spatial-temporal convolution module proposed here consists of a graph convolution in the spatial dimension, capturing spatial dependencies from neighbourhood and a convolution along the temporal dimension, exploiting temporal dependencies from nearby times.
+
+Graph convolution in spatial dimension
+Spectral graph theory extends the convolution operation from grid-based data to graph-structured data. In this study, the traffic network is represented as a graph, where each node's features are signals on the graph. To leverage the network's topology, graph convolutions based on spectral graph theory are applied at each time slice, processing these signals and exploiting spatial correlations. This spectral method transforms the graph into an algebraic form to analyse its topological attributes, such as connectivity.
+
+Convolution in temporal dimension
+After the graph convolution operations having captured neighbouring information for each node on the graph in the spatial dimension, a standard convolution layer in the temporal dimension is further stacked to update the signal of a node by merging the information at the neighbouring time slice.
+
+Multi-Component Fusion
+
+![image](https://github.com/user-attachments/assets/0b08aedd-3ca4-4372-b6fa-1f9c5d8e15db)
+
+It can be observed that the traffic flows at some areas have obvious peak periods in the morning, so the outputs of the daily-period and weekly-period components are more crucial. However, there are no distinct traffic cycle patterns in some other places, thus the daily-period and weeklyperiod components may be helpless. Consequently, when the outputs of different components are fused, the impacting weights of the three components for each node are different, and they should be learned from the historical data.
+
+Experiments
+
+Datasets
+PeMSD4, PeMSD8
+
+Baselines
+HA, ARIMA, VAR, LSTM, GRU, STGCN, GLU-STGCN, GeoMAN
+
+Metrics
+MAE, RMSE
+
+Results
+
+![image](https://github.com/user-attachments/assets/2b86e6cd-2dd7-4b5d-8c90-b6787af2ee25)
+
+* DIFFUSION CONVOLUTIONAL RECURRENT NEURAL NETWORK: DATA-DRIVEN TRAFFIC FORECASTING
+
+Introduction
+
+![image](https://github.com/user-attachments/assets/ab8a2f63-f7f2-47d6-b256-284df20e75f9)
+
+This paper proposes a model that uses the pair-wise spatial correlations between traffic sensors using a directed graph whose nodes are sensors and edge weights denote proximity between the sensor pairs measured by the road network distance. The dynamics of the traffic flow is viewed as a diffusion process and propose a diffusion convolution operation that captures spatial dependency. The full model architecture - Diffusion Convolutional Recurrent Neural Network integrates diffusion convolution, seq2seq architecture, and scheduled sampling.
+
+Methodology
+
+Spatial dependency modelling
+
+Spatial dependency is modelled using a diffusion process which can capture the innate stochasticity of a traffic's dynamics. This is done by a random walk on the graph. Lemma 2.1 from Teng et al. (2016) is referenced:
+The stationary distribution of the diffusion process can be represented as a weighted combination of infinite random walks on the graph, and be calculated in closed form
+
+Temporal dynamics modelling
+
+GRU is used to capture temporal characteristics.
+
+For multi-step forecasting, seq2seq is employed, where both the encoder and decoder are RNNs with DCGRU. During training, historical time series is fed into the encoder and its final state is used to initialise the decoder. The decoder generates predictions given previous ground truth observations. Entire network:
+
+![image](https://github.com/user-attachments/assets/6b891a00-e77a-4389-b112-31ef7e7e0967)
+
+Experiments
+
+Datasets
+METR-LA, PEMS-BAY
+
+Baselines
+HA, ARIMA, VAR, SVR, FNN, FC-LSTM
+
+Results
+
+![image](https://github.com/user-attachments/assets/88defe65-8b01-4674-8189-bc9bc72d50b2)
+
+
+That is all for today!
+
+See you tomorrow :) 

From 3d6e9a17ba0f40029a1b095a02b5cfdcba74c13f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 15:47:13 +0900
Subject: [PATCH 263/638] Update 2024-06-06-day-157.md

---
 _posts/2024-06-06-day-157.md | 112 ++++++++++++++++++++++++++++++++++-
 1 file changed, 110 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-06-day-157.md b/_posts/2024-06-06-day-157.md
index cadf548b6..6fd47850e 100644
--- a/_posts/2024-06-06-day-157.md
+++ b/_posts/2024-06-06-day-157.md
@@ -1,5 +1,113 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 157) GNN design choices and starting an MLOps book on manning.com
+categories: [reading-research,mlops]
 ---
+
+## Hello :) Today is Day 157!
+A quick summary of today:
+* read about design choices for GNNs
+* started reading a book about MLOps on manning.com
+* registered for Korea summer workshop on causal inference
+
+### Design choices for Graph Neural Networks [[arxiv](https://arxiv.org/pdf/2011.08843)]
+
+I saw this paper from the last short lecture from XCS224W: ML with graphs, it looks into how different design choices affect a GNN model's performance. It caugt my eye as some of the findings as to what is useful could be tested/applied in my work/research at the lab. 
+
+Below are some of the interesting findings:
+
+![image](https://github.com/user-attachments/assets/ebd74894-e1c2-4535-8338-2e0bfa1bb18b)
+
+For example, having batch norm is always better more than not having it; no dropout seems to perform better too, prelu activation seems the best, and using sum as an aggregation function outperforms mean and max. 
+
+### Design a Machine Learning System (From Scratch) [[book](https://www.manning.com/books/design-a-machine-learning-system-design-from-scratch)]
+
+The book will teach me how to:
+
+* Build an ML Platform
+* Build and Deploy ML Pipelines
+* Extend the ML Platform using various tools depending on use cases
+* Implement different kinds of ML services using the ML life cycle as a mental model
+* Deploy ML services that are reliable and scalable
+
+A quick summary of chapter 1: Getting Started with MLOps and ML Engineering
+
+An ML project begins with the experimental phase. This phase involves continuous iterations and adjustments across various steps, such as model training and evaluation, to refine models based on complex data and performance metrics. Building an orchestrated pipeline automates these steps, reducing errors and streamlining the process before full automation in later stages.
+
+![image](https://github.com/user-attachments/assets/a3defe1f-286f-4fae-8025-55357a8f761b)
+
+Next, is the Dev/Staging/Production phase marks the shift from model experimentation to deployment in real-world settings, emphasizing scalability, robustness, and real-time performance. This phase involves full automation of the pipeline, with continuous integration and performance monitoring to ensure smooth deployment and maintenance of ML models.
+
+![image](https://github.com/user-attachments/assets/683c1ae0-e2e3-4ca3-ac38-8a071ad634ae)
+
+The skills needed for MLOps include software engineering, data science, DevOps, data management, model training, deployment, monitoring, debugging, and automation to ensure reproducibility and compliance.
+An ML platform enables practitioners to develop and deploy ML services by integrating various tools essential for the ML life cycle. Typically, it comprises a collection of loosely related software that evolves as team maturity and use case complexity increase. This book guides you through building an ML platform using Kubeflow, covering components like pipeline orchestration, feature stores, and model registries.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgVYhtFqoNjeDnXopi060KLE0SA2NS_QgmxsmIx4xZQhVKfTx2HB67Fttt3yD1RzONfnQEA2MhfRKjU4RzgovX5YhaHx12gg7WQxUyak2f6KpFxF6s6VZRThHrtXVpt4KbayRj19n_cuo5hr5KH_qka7Ges7e3XIoD3XTHzBRcBBRsrsmBylZUam8FA_Hr5)
+
+### A Short Summary of Chapter 2: What is MLOps?
+
+First step - Data Collection
+
+- Data relevance to problem domain
+- Size of the dataset with respect to problem complexity
+- Quality of the dataset. Prevention of harmful biases and unintentional leakage of samples
+- Distribution of data and features is representative of the deployment environment
+- Sufficient diversity in the data collection process that defines the problem domain well
+- Lineage and detailed tracking of raw data, intermediate versions, and annotated datasets
+
+Second step - EDA (Exploratory Data Analysis)
+
+- What does my data look like? What is its schema? Can the schema change? How would I guard against invalid values? Does the data require cleaning?
+- How is my data distributed? Are all targets equally represented, or should there be additional class balancing?
+- Does my data have robust features for the task I have in mind? Are the features expensive to compute?
+- Does the input data vary cyclically? Does it exhibit correlations to external factors that are not modeled?
+- Are there any outliers in the dataset? What must be done to outlier values in production?
+
+Third step - Modelling and Training
+
+- Model and data versioning
+- Experiment tracking
+- Model training pipelines
+- Hyperparameter search
+
+Fourth step - Model Evaluation
+
+- Choosing the appropriate metrics
+- Examine and analyse misclassified samples, error patterns
+
+Fifth step - Deployment
+
+- To a specific environment, or API endpoint
+- Staging vs production
+
+Sixth step - Monitoring
+
+- Track anomalies in data and model performance
+- Data/performance/error monitoring
+- Need reliable notification/alert system
+
+Seventh step - Maintenance, Updates, and Review
+
+- Implement bug fixes to address issues and shortcomings in the model in production
+- Collect data to train the model on specific edge cases and improve performance
+- Mitigate data drift by informing the data collection component and retrain a new model
+
+Importance of Robust MLOps
+
+It is crucial because it ensures the seamless integration and collaboration of various specialized disciplines, leading to consistent and reliable model performance that drives business value. It addresses complex challenges such as data management, scalability, and compliance, reducing technical debt and enhancing the maintainability and scalability of ML projects. A robust framework supports continuous improvement and mitigates risks associated with data privacy, biases, and system failures.
+
+### Korea Summer Workshop on Causal Inference 2024
+[Here](https://www.linkedin.com/search/results/content/?fromMember=%5B%22ACoAACbOiGoBFDlwPKwVKOt9m1Q4sk-O76IV8Bs%22%5D&heroEntityKey=urn%3Ali%3Afsd_profile%3AACoAACbOiGoBFDlwPKwVKOt9m1Q4sk-O76IV8Bs&keywords=Jinsoo%20Shin&sid=!Zr&sortBy=%22date_posted%22&update=urn%3Ali%3Afs_updateV2%3A(urn%3Ali%3Aactivity%3A7204326325493374976%2CBLENDED_SEARCH_FEED%2CEMPTY%2CDEFAULT%2Cfalse)) is the post (in Korean):
+
+It spans over 4 days - June 13, 20, 27, 28, and some of the covered topics are:
+- Causal inference in policy making and various practical applications
+- Best practices for online controlled experiments and experiments platform
+- Data-driven decision-making
+- Data science & causal ML/AI in the generative AI era
+
+On the day of the events, I will post more detailed info about each day's sessions because there are a total of 16 for now.
+
+That is all for today!
+
+See you tomorrow :)

From 7a91447b81a78b31c1d2c5a509738760ff53e29e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 16:14:31 +0900
Subject: [PATCH 264/638] Update 2024-06-07-day-158.md

---
 _posts/2024-06-07-day-158.md | 160 ++++++++++++++++++++++++++++++++++-
 1 file changed, 158 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-07-day-158.md b/_posts/2024-06-07-day-158.md
index cadf548b6..9c5db01a9 100644
--- a/_posts/2024-06-07-day-158.md
+++ b/_posts/2024-06-07-day-158.md
@@ -1,5 +1,161 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 158) 50 minutes of audio in the Scottish dataset + exploring Mixture Density Networks in GNNs
+categories: [glaswegian,gnn,reading-research]
 ---
+
+## Hello :) Today is Day 158!
+A quick summary of today:
+* created a better dataset preprocessing 'pipeline' for new audio files
+* read a bit about Mixture density networks and their application
+
+### Firstly, about the Scottish dataset
+
+The latest dataset has ~50 minutes worth of Glaswegian (Scottish) accent clips. Amazing ^^ [huggingface link](https://huggingface.co/datasets/divakaivan/glaswegian_audio)
+
+I also finetuned microsoft's SpeechT5 on this latest data, but I am still getting a bit robotic outputs. I need to play more around with the trainer setup. 
+
+As for the 'pipeline' ~ 
+
+It starts with renaming the audio files (so that we have some kind of tracking), I rename them by adding the preprocess date.
+
+![image](https://github.com/user-attachments/assets/f5694a9e-30d6-4739-89a9-a875792561bf)
+
+Second is a bit longer, so I will just paste the execute function:
+
+`create_audio_metadata_csv(transcriptions_csv, filenames_df, audio_files_path, output_csv)`
+
+It takes a transcription csv that has 1 column with transcriptions, a 2nd csv with file names, path to the audio files, and where to output the csv which contains file_name, transcription, length_seconds, sampling_rate
+
+Final is concatenating the old metadata csv with the new one (and also add some default info like gender, class, accent info)
+
+![image](https://github.com/user-attachments/assets/57e0f614-6fd2-4653-939b-731fad8444fb)
+
+Definitely not the final version, but this took only a few minutes from downloading the files uploaded to google drive by my Scottish partner, to updating the dataset on huggingface. 
+
+### As for the papers I read today at the lab 
+
+First, Estimating city-wide hourly bicycle flow using a hybrid LSTM MDN
+
+I read this over the top for now, because I was interested in how they applied mixture density networks on an LSTM model
+
+Simple LSTM
+
+![image](https://github.com/user-attachments/assets/473d0829-c80d-4f89-ab13-7676c06b83a0)
+
+LSTM with MDN
+
+![image](https://github.com/user-attachments/assets/78a4a739-4e93-46fd-97d2-8b128faf51df)
+
+Results
+
+![image](https://github.com/user-attachments/assets/d5ed2b52-0191-4c96-ba8a-bd68c3790058)
+
+Second, Sparse trip demand prediction for shared E-scooter using spatio-temporal graph neural networks
+
+Introduction
+
+The shared E-scooter service, integrating GPS, IoT, and API technologies, offers a convenient, flexible, and eco-friendly alternative to private vehicles for short trips, promoting public transportation usage and reducing carbon emissions. Its rapid growth in the U.S. necessitates real-time demand forecasting for efficient urban planning, fleet management, and maintenance to balance supply and demand effectively.
+
+E-scooter demand prediction challenges
+
+* spatial-temporal trip demands with extreme sparsity and imbalance
+* difficult in predicting complexity and heterogeneity
+* lack of a more systematic integration of the correlations between various factors and the expression of urban complexity
+
+Contributions
+
+* recluster spatial data and implement fusion loss functions with a loss penalty to mitigate trip demand accuracy deterioration due to demand sparsity and imbalance
+* to simulate the relationship between urban complexity and shared e-scooter trip demands, different aspects are incorporated into the graph representation - i.e. built environments, weather conditions, periodic features
+* building upon DCRNN a model that outperforms other benchmark models is proposed
+
+Related work
+
+Multivariate statistical analysis and regression for shared E-scooter usage correlation
+
+Data-driven approaches analyzed correlations between ridership and various factors, including built environments, weather conditions, and periodic features, across different cities. Key findings include population density and education levels positively impacting E-scooter trips, city center proximity and high street density increasing ridership, and varying land use types influencing usage differently. Additionally, weather conditions significantly reduce trip frequency, and trip patterns vary between weekdays and holidays. Despite these insights, real-time accurate predictions for local subareas are still lacking, with deep learning models proposed as a potential solution.
+
+Deep learning methods for shared E-scooter trip demand prediction
+
+Traffic demand prediction has gained significant attention with the use of deep learning models, showing promising results. Key advancements include integrating spatial and temporal dependencies through combined models like RNNs and CNNs, with recent approaches leveraging graph convolutional networks for complex traffic networks (TGCN). For shared mobility, including E-scooters, models like CNN-LSTM, multi-graph convolutional, and gated graph convolutional networks have been developed to predict demand, considering dynamic origin-destination relationships and various influencing factors like built environments and weather conditions.
+
+Methodology
+
+Framework of trip demand prediction
+
+![image](https://github.com/user-attachments/assets/9b1ddce8-b567-4924-949e-94772c58d6b6)
+
+1. Data collection - data include OD shared E-scooter trip data, population density, spatial relationships, POI, land use, and weather conditions in Louisville
+2. Spatial data reclustering - mapping data to geographical divisions
+3. Graph representation - the above are converted into comprehensive graph representations with graphs, nodes, and edges to capture the spatial-temporal dependencies for trip demand prediction
+4. Model training - a model specifically for sparse trip demand prediction is proposed, and training is adjusted
+5. Pick-up demand prediction - after fine-tuning the model, an optimal model is obtained that can predict the pick-up demand in the next hour in each region
+
+Graph representation and built environment factors
+
+* Universal features - weather and periodic features; apply time-series information, including year, month, week, and hour, as features and transform them into cyclical representations via cosine and sine functions based on their periodic behaviours
+* Node features - extract the hourly pick-up demand as the first dimension feature, which is specified with negative integers to distinguish it from the drop-off demand as positive integers; by contrast, the hourly trip drop-off demand is the second dimension feature with positive integers. The sum of hourly trip pick-up and drop-off demands is the third dimension feature, which would be zero if rented and returned E-scooters share the same numbers. Other features representing urban built environments include spatial distance to the city centre, street density, intersection density, and population density for each region
+* Edge features - the three spatial relationship between the origins and the destinations are distance, POI similarity, and difference in land-use entropy. POI similarity is calculated using Pearson's correlation coefficient to estimate functional similarity between two regions. Land-use entropy ranging from 0 to 1 is used to estimate the equilibrium between all available land uses for each region and calculate the differences between origins and destinations.
+
+Model architecture
+
+The introduced model is a modified version of DCRNN.
+
+![image](https://github.com/user-attachments/assets/117c685f-e6bf-49f8-a38b-1f7cc8691d34)
+
+GRU captures the temporal dependencies, while a diffusion convolution layer captures spatial dependencies.
+Data description
+
+E-scooter data from Kentucky, USA. This research focuses on next hour pick-up demands as prediction targets.
+
+* Shared e-scooter OD trip dataset
+
+![image](https://github.com/user-attachments/assets/6cf90d17-d0d3-4642-8896-e82175dfa644)
+
+* population dataset
+
+![image](https://github.com/user-attachments/assets/74269dc8-5c2f-4451-942d-05e7660a6162)
+
+* street network and POI dataset
+
+![image](https://github.com/user-attachments/assets/f907f303-58ae-4096-ac64-edfc36a48f5c)
+
+* land-use dataset
+
+![image](https://github.com/user-attachments/assets/e0bd3f06-6f49-49af-9ea7-808807d2a065)
+
+Strategies for the imbalanced data
+
+* Spatial data reclustering: to integrate diverse learning features from different data sets measured across various standard divisions, data must be reaggregated and remapped into uniform geographical divisions. The study proposes reclustering prediction divisions from geographical census blocks to regions that better reflect the relationship between shared E-scooter trips and built environment factors. Using a hierarchical clustering method based on pick-up demand, the resulting reclustered regions, depicted in Fig. 5, highlight areas of higher demand, with low-demand regions being excluded from model training
+
+![image](https://github.com/user-attachments/assets/183c9079-1f47-491d-8bbf-a6d33605c4b3)
+
+* Fusion loss function and evaluation metrics: The proposed fusion loss function combines mean absolute error (MAE) and multi-classification loss with a penalty to address sparse and imbalanced data. It categorizes trip demand into different levels to improve performance and uses Jenks natural breaks algorithm to define six classes, later simplified into three levels for practical use. Evaluation metrics include MAE, RMSE, F1 score, and AUC, with the focal loss emphasizing minority samples and a loss penalty preventing the model from predicting zero demand excessively.
+
+![image](https://github.com/user-attachments/assets/4c5dc686-6ac0-4a60-9146-30ba5b6f6109)
+
+Experiments
+
+Baselines
+
+GCRN (with GRU and with LSTM), TGCN, A3T-GCN, 3 variants of the proposed model
+
+Average prediction performance over different periods
+
+![image](https://github.com/user-attachments/assets/ca096061-ea0d-427c-a876-7d8ad874653e)
+
+Ablation study for the additional learning features
+
+![image](https://github.com/user-attachments/assets/9c4e01c1-fe4e-481e-a05e-b7a36ca4c3a3)
+
+Contribution of fusion loss
+
+![image](https://github.com/user-attachments/assets/20153a26-9194-4978-b350-d78251d37710)
+
+Overall
+
+![image](https://github.com/user-attachments/assets/7eeb17e6-dd4e-4b2a-9173-c9390ac56856)
+
+That is all for today!
+
+See you tomorrow :)

From f13f6aa69b1d266bfe2235bd5818f4c783346d57 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 16:21:44 +0900
Subject: [PATCH 265/638] Update 2024-06-08-day-159.md

---
 _posts/2024-06-08-day-159.md | 172 ++++++++++++++++++++++++++++++++++-
 1 file changed, 170 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-08-day-159.md b/_posts/2024-06-08-day-159.md
index cadf548b6..199bf19f0 100644
--- a/_posts/2024-06-08-day-159.md
+++ b/_posts/2024-06-08-day-159.md
@@ -1,5 +1,173 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 159) Learning and using prefect for MLOps orchestration
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 159!
+A quick summary of today:
+* did Module 3 of the MLOps zoom camp from the [2023](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/cohorts/2023/03-orchestration) cohort that uses Prefect
+
+Before everything else, I finally got the notebook expert title on [Kaggle](https://www.kaggle.com/divakaivan12/)! :party:
+
+![image](https://github.com/user-attachments/assets/bbb323c7-be3d-4301-8ac5-cd37a4a08e33)
+
+### As for prefect ~
+
+Github repo from today's study is here.
+
+A common MLOps workflow
+
+![image](https://github.com/user-attachments/assets/8cc69196-27bc-48ec-9a49-bf66efb9af3f)
+
+Where we take data from a database into pandas, save it (checkpoint), reload with parquet, then maybe use sklearn for feature engineering and or running models, mlflow is there to track experiments, and finally the model is served. 
+
+However, we might have failure points at any of the arrows (connections) between steps. 
+
+Prefect comes in when we give an engineer the following tasks:
+
+* could you just set up this pipeline to train this model?
+* could you set up logging?
+* could you do it every day?
+* could you make it retry if it fails?
+* could you send me a message when it succeeds?
+* could you visualise the dependencies?
+* could you add caching?
+* could you add collaborators to run ad hoc - who don't code?
+
+The first few could be done quite easily, but the mid/later ones become hard without some kind of extra help like Prefect.
+
+Some terminology:
+
+![image](https://github.com/user-attachments/assets/57754778-c579-4a33-b93f-c71c8f1577ef)
+![image](https://github.com/user-attachments/assets/ba40220d-0828-49de-85ef-78ad01acca39)
+
+Next, running a simple flow and seeing it in the Prefect UI
+
+After some setup ~ and prefect server start ~
+
+![image](https://github.com/user-attachments/assets/844533ba-67e1-46a3-9494-c4c0559e6cec)
+
+We ran this cat facts flow. In the UI I can see:
+
+![image](https://github.com/user-attachments/assets/0599f671-34bf-43f8-88bc-1a0c98a6b0b7)
+![image](https://github.com/user-attachments/assets/185c5a27-71f1-42a3-a955-7db67fddfbd2)
+
+And at the bottom in the logs: 'A cat’s nose pad is ridged with a unique pattern, just like the fingerprint of a human.'
+
+Next, I executed a flow that utilises subflows.
+
+![image](https://github.com/user-attachments/assets/c1730a0f-4a3e-4fec-9802-278eb5406ffc)
+
+And in the UI, 3 flows are creating in total. If we visit the parent one, called animal_facts we can see two two subflows:
+
+![image](https://github.com/user-attachments/assets/c0ce3d7f-596b-4766-9248-e938e569a453)
+
+Next ~ creating a workflow
+
+From Module 1, I created a jupyter notebook for [taxi prediction](https://github.com/divakaivan/MLOps-camp/blob/main/Module%201/duration-prediction.ipynb), but here, to make it a bit more 'production-ready', the code is a bit more structured as follows:
+
+Create task - read data
+
+![image](https://github.com/user-attachments/assets/c7174950-5494-4ae9-b414-93b1e2be4d1e)
+
+Create task - add features
+
+![image](https://github.com/user-attachments/assets/b72891d1-0fb5-44f3-b71f-c89611b1b302)
+
+Create task - train model (a bit long, so a truncated version)
+
+![image](https://github.com/user-attachments/assets/1fe1d201-6971-47ce-a005-444ea3ce63f3)
+
+And then, the main flow that executes all the tasks:
+
+![image](https://github.com/user-attachments/assets/695fbb17-d255-48ed-b357-152c24d3ca87)
+
+After we execute this flow, in the prefect UI we can see a new flow run with this visualisation of the tasks:
+
+![image](https://github.com/user-attachments/assets/60e54614-6b83-4929-ba8e-dc5ea9fd88bb)
+
+There are logs too:
+
+![image](https://github.com/user-attachments/assets/a173ee2e-c156-4acd-908b-6a94b0a2be4d)
+
+And... some error on the 'deploying your workflows' part because of a new version of prefect. :/ 
+(I am writing today's post as I study ~)
+
+Thankfully, [youtube](https://youtu.be/D5DhwVNHWeU) came to the rescue!
+
+The commands are a bit different, but I got a deployment working.
+
+![image](https://github.com/user-attachments/assets/9a7406a0-58d0-47b5-b506-4a8ff486053e)
+![image](https://github.com/user-attachments/assets/23753a63-586b-425e-988a-ea75528b4631)
+
+(first one failed because of wrong data file name)
+
+![image](https://github.com/user-attachments/assets/35ab5d8d-2e8a-4174-9db3-c845baf7a602)
+
+This was just a local deployment, but there are other options to deploy to GCP, aws, etc. 
+
+In the next bit, I setup AWS - and S3 storage with taxi data and a user with S3 permissions in IAM.
+
+Executed code:
+
+![image](https://github.com/user-attachments/assets/1e1062f6-f20c-475d-a6d0-8857869ad567)
+
+And the result on the prefect server:
+
+![image](https://github.com/user-attachments/assets/14fd5bbc-c7a6-426e-957e-9ba19b4db5b2)
+![image](https://github.com/user-attachments/assets/1937d90b-1ccc-443f-991f-9d9cba249e1d)
+
+These blocks can be modified through the UI (also we can add new blocks through the UI directly too, rather than code)
+
+We can use the uploaded S3 data using aws_prefect
+
+![image](https://github.com/user-attachments/assets/d7cb668f-41e0-4d35-943f-6bd5d7459769)
+
+and running the whole flow again:
+
+![image](https://github.com/user-attachments/assets/fc39ea2b-b560-48d9-b78a-7334b5ebd424)
+
+We get the normal results and a trained model
+
+![image](https://github.com/user-attachments/assets/ec074800-42e8-4910-a01e-fc7375fc2336)
+![image](https://github.com/user-attachments/assets/0ee79352-447a-4b10-93ee-f2d1a5ce4c17)
+
+We can also create artifacts that show any info we want (using prefect's create_markdown_artifact)
+
+![image](https://github.com/user-attachments/assets/d4dab112-3da7-48b6-8975-6472461fc36e)
+
+We can see the output under artifacts in the flow run
+
+![image](https://github.com/user-attachments/assets/c3a01413-beb7-46bd-bcaa-d2e898897898)
+
+Final section - connecting to prefect cloud
+
+The cloud webapp has pretty much the same UI as the local one, and once I logged in, and deployed a flow, I could see:
+
+![image](https://github.com/user-attachments/assets/41b966f2-d320-454c-b363-47a6918e3f0f)
+![image](https://github.com/user-attachments/assets/fb54b730-a27b-4e39-95b2-83bc9a15d9ac)
+![image](https://github.com/user-attachments/assets/f42b84f9-9f9b-4917-872d-a59293d4156d)
+
+In the cloud, there are also automations with different trigger options
+
+![image](https://github.com/user-attachments/assets/2a8239a3-048e-498b-9a24-cd9a81c4d501)
+
+After a trigger, we need to select an action - there are too many, so in the pic are just a few
+
+![image](https://github.com/user-attachments/assets/36e73196-42d5-4eda-93bf-0a098706e031)
+
+There are so many options for each one, I did a basic email one:
+
+![image](https://github.com/user-attachments/assets/3275f79a-707a-4cf3-a924-d484f9ae3f71)
+
+Finally, give it a name, and its all done.
+
+That is the end of this Module 3 ~
+At least for now. I like prefect a lot, and might end up using it for the final project of the MLOps zoomcamp, but will see. I will give mage.ai another go. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 5a5fa2834507abb014c3bb7e56a808bbfe190d98 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 16:30:40 +0900
Subject: [PATCH 266/638] Update 2024-06-09-day-160.md

---
 _posts/2024-06-09-day-160.md | 131 ++++++++++++++++++++++++++++++++++-
 1 file changed, 129 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-09-day-160.md b/_posts/2024-06-09-day-160.md
index cadf548b6..31fe309ff 100644
--- a/_posts/2024-06-09-day-160.md
+++ b/_posts/2024-06-09-day-160.md
@@ -1,5 +1,132 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 160) Simple data engineering pipeline with Prefect, and... MLOps with mage.ai (tons of problems)
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 160!
+A quick summary of today:
+* simple data engineering pipeline with prefect
+* tons of trouble learning about orchestration with mage.ai
+
+After yesterday's journey with prefect the youtube algorithm recommended me [another tutorial for prefect](https://youtu.be/FxRZ9zo6GmI) - this time for creating data pipelines with prefect. So I decided to give it a go. 
+
+What is data engineering?
+
+![image](https://github.com/user-attachments/assets/94d0dc42-06d7-4f7e-9e27-0e47b2770513)
+
+* data scientists can do data engineering, but in specific cases where the two jobs cannot or are not needed to be separate
+* data engineers build databases, they build lots of data pipelines and manage infrastructure (also care about cost, security)
+
+What are data pipelines?
+
+* ETL(ELT)/batch pipelines that move data from A to B
+  * databases, APIs, files
+* streaming pipelines - as data comes in, we consume that data and send it wherever it needs to go 
+  * message queues, polled data
+
+The main github repo used is [here](https://github.com/CodeSeoul/intro-to-prefect).
+
+After some basic setup, when we run 'pipeline' in the terminal which runs the main.py file:
+
+![image](https://github.com/user-attachments/assets/6a5b9994-f13e-4a45-97d8-1e687053ced1)
+
+In prefect we get a flow run
+
+![image](https://github.com/user-attachments/assets/af2dff44-70d6-4927-87bd-82e33c855bfb)
+
+and logged outputs from the petstore url
+
+![image](https://github.com/user-attachments/assets/6b52d79f-5fe0-4478-8ca7-be52b10e8948)
+
+Now for a flow with a bit more tasks~
+
+1st task: retrieve from API
+
+![image](https://github.com/user-attachments/assets/9b4f8c53-c77c-496b-9251-09e7500c0106)
+
+2nd task: clean data
+
+![image](https://github.com/user-attachments/assets/32b3fe36-2633-4c1b-89cf-25ddc331f1b3)
+
+3rd task: insert to postgres db
+
+![image](https://github.com/user-attachments/assets/a9da4e5f-07ab-47db-b866-f2a3f245cbb4)
+
+final flow:
+
+![image](https://github.com/user-attachments/assets/24f4d5ff-64d5-4225-be5a-2d4d8971398a)
+
+Success!
+
+![image](https://github.com/user-attachments/assets/6d4bc67c-906e-4bf7-b7e4-b66ef2d1ae9a)
+
+We can also see the data loaded into postgres:
+
+![image](https://github.com/user-attachments/assets/233e2660-eda0-4269-8564-9bde68645a43)
+
+Beege (the teacher) showed also how to do simple task tests and avoid a common prefect error when executing prefect task functions outside a prefect flow
+
+![image](https://github.com/user-attachments/assets/a9511af6-f209-43b6-bcb1-3c5d22cca370)
+
+Prefect is amaing, but the MLOps zoomcamp course works with Mage.ai this year, and after I gave up on it due to errors that I could not fix last time - I decided to give it another go today. And... omg so many errors and problems - I almost gave up on it several times. The videos are recorded 2 weeks ago, and yet there is so much different on my UI (and other students' UI) that it is mind-boggling.
+
+The content was not much, but the constant errors... made me invest upwards of 12 hours of bug fixing for a total of 30 minutes of video content. 
+
+### Orchestration with Mage.ai from MLOps zoomcamp [module 3](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/03-orchestration)
+
+The content of the intro to orchestration is:
+
+![image](https://github.com/user-attachments/assets/d56f3cfc-2641-409a-9785-282f114e050e)
+
+I have to do 3.5 tomorrow (hopefully). 
+I just want to say a huge thank you to the QnA bot on the mlops zoomcamp slack channel that at least pointed me into the right direction to solve my issues. Just quickly - I am bit sad that there are so many issues (almost every video) because it will drive people away from this amazing course, and below I will just provide my successes, and if you are really curious about my problems - they are all on the datatalks slack channel haha. 
+I started from the beginning
+
+3.1 Data preparation
+
+![image](https://github.com/user-attachments/assets/f5600cea-7178-436e-8331-5827d9e29e5b)
+
+I created the above pipeline that reads NY taxi data, does a bit of preprocessing, and then outputs X, X_train, X_val, y, y_train, y_val, dv. By the way, each block is a piece of code.
+
+3.2 Training
+
+First, create a pipeline to train a linear regression and lasso model. It takes data from the above 3.1 pipeline, loads models, does hparam search, and finally trains the two models.
+
+![image](https://github.com/user-attachments/assets/c02497da-9d37-4379-863f-21482267a367)
+
+Secondly, I had to create a pipeline for an xgboost model
+
+![image](https://github.com/user-attachments/assets/a6decd1b-0ac9-4579-87d0-b13a1affb7ea)
+
+The last (pink) bit is connected to creating visualisations. 
+
+3.3 Observability
+
+![image](https://github.com/user-attachments/assets/b63c3a7e-bcce-41c9-a76b-952b7fae88d2)
+
+We can create any kind of bar/flow/line/custom charts, and the above 3 are some SHAP values from the xgboost model. 
+
+3.4 Triggering
+
+Here I created an automatic trigger to train the xgboost model when new data is detected.
+
+Also, created this predict pipeline
+
+![image](https://github.com/user-attachments/assets/c9b089ac-1da9-4791-8c39-3876acdbbed5)
+
+There is a nice way to setup a basic interface for inference
+
+![image](https://github.com/user-attachments/assets/535f95df-5cc2-4e15-8cc7-2bf6a0db9a4b)
+
+We can also setup an API to do inference through that as well:
+
+![image](https://github.com/user-attachments/assets/604a9c8b-39cf-4368-9bbb-d69e46d34710)
+
+That was all for today. I have very strong feelings towards Mage.ai but for now I will roll with it because of the nice course. Otherwise I will try to use prefect for some side project (have to think about that a bit). 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From f3302c0c005da877d1f0282ee537d4757aaa548a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 16:35:54 +0900
Subject: [PATCH 267/638] Update 2024-06-10-day-161.md

---
 _posts/2024-06-10-day-161.md | 153 ++++++++++++++++++++++++++++++++++-
 1 file changed, 151 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-10-day-161.md b/_posts/2024-06-10-day-161.md
index cadf548b6..b955e1899 100644
--- a/_posts/2024-06-10-day-161.md
+++ b/_posts/2024-06-10-day-161.md
@@ -1,5 +1,154 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 161) Learning about GANs' use in generating OD demand matrix
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 161!
+A quick summary of today:
+* explored how Generative Adversarial Networks can be used for OD demand matrix generation
+* found that the original graph attention network has limitations and read the paper for GATv2
+
+### First paper - A GAN framework-based dynamic multi-graph convolutional network for origin–destination-based ride-hailing demand prediction
+
+Introduction
+
+OD-based prediction focuses on estimating travel demand between specific origin and destination regions, which is useful for understanding inter-regional travel patterns. This approach requires consideration of complex spatio-temporal correlations, making it more challenging than region-level prediction. Although some progress has been made in OD-level prediction, several key issues remain:
+
+* Practical OD Demand Forecasting: Previous studies often only consider relationships between OD pairs, rather than between OD regions. Accurate OD demand matrices and inter-regional relationships need to be developed.
+* Dynamic and Directed Topology Modelling: Current models use static and undirected graphs, which fail to capture the time-varying and directed nature of OD demand. Dynamic and directed OD graphs are needed for better spatial dependence modelling.
+* Addressing Matrix Sparsity: OD demand matrices often contain more zeros than non-zeros, reducing prediction accuracy. A suitable model structure is required to address this sparsity.
+
+To tackle these issues, the authors propose the Dynamic Multi-Graph Convolutional Network with Generative Adversarial Network (DMGC-GAN). This model captures directed and time-changing correlations in dynamic OD graphs. Key components include:
+
+* Dynamic OD Graphs: Represent irregular regions as nodes with weighted matrices for different inter-regional relationships.
+* Temporal Multi-Graph Convolutional Network (TMGCN) Layer: Integrates various spatial-temporal information from different dynamic OD graphs, such as static neighbourhood relationships, dynamic mutual attraction, and mobility association modes.
+* GAN Structure: Helps overcome the high sparsity of OD demand matrices.
+
+Related work
+
+Graph convolutional network
+
+To adapt CNNs for urban demand forecasting, cities are often partitioned into rectangular grids, which do not accurately reflect the urban road network. CNNs are also limited in capturing semantic correlations between geographically distant regions with similar functions. Hexagon-based CNNs (H-CNN) improve precision but still fail to account for actual road layouts. Since passengers' demand data is non-Euclidean, traditional CNNs are inadequate for ride-hailing demand forecasting.
+
+Graph Convolutional Networks (GCNs) have been introduced in traffic fields to address these limitations, such as traffic flow and speed forecasting. The Temporal Graph Convolutional Network (TGCN) models roads as graph nodes to capture spatial and temporal dependencies. The Spatial-Temporal Multi-Graph Convolutional Network (ST-MGCN) captures dependencies like geographical distance, functional similarity, and traffic connectivity, using grouped lower and higher layer GCNs for generalised feature extraction. The Multi-Community Spatial-Temporal Graph Convolutional Network (MC_STGCN) incorporates GRU for temporal relationships and various graph structures to model spatial dependencies. The Multiple Spatio-Temporal Information Fusion Networks (MSTIF-Net) combines GCN and Variational Auto-Encoders (VAE) for dynamic spatio-temporal characteristics. Additionally, the Temporal Attention Graph Convolutional Network (TAGCN) describes spatial and temporal relationships with a temporal attention mechanism.
+
+OD-level demand forecasting
+
+While region-based demand forecasting is well-studied, OD-based demand prediction is less explored. OD-based prediction is valuable for understanding travel demand and inter-regional relationships. Existing models use static and undirected OD graphs based on geographic proximity or passenger demand, but these fail to capture dynamic and directed dependencies over time.
+
+Recent efforts, like LSTM and CNN-based encoder-decoder structures, still struggle with dynamic graphs. The Dynamic Node-Edge Attention Network (DNEAT) addresses evolving road network topologies in dynamic OD graphs.
+
+Motivated by these limitations, the authors propose a novel neural network to efficiently capture spatio-temporal correlations in OD demand data for better OD-based prediction.
+
+Methodology
+
+OD graph, OD demand intensity (split into hourly trends, daily cycle and weekly cycle)
+
+Problem
+
+Given a dynamic topology and the sequence of OD demand intensity at time t, the OD-based demand prediction problem is to determine the OD demand intensity and t+1.
+
+Framework
+
+![image](https://github.com/user-attachments/assets/d7626c62-9b06-41e0-af81-91086910208a)
+
+TMGCN cell
+
+![image](https://github.com/user-attachments/assets/46eec84f-0481-4a84-ad0b-085448f68f50)
+
+Training algorithm
+
+![image](https://github.com/user-attachments/assets/4931af9d-06a6-4ccc-8866-21239be1f831)
+
+Experiments
+
+Dataset
+
+Provided by NY taxi and limousine commission
+
+![image](https://github.com/user-attachments/assets/28abd7ed-f6c5-4836-b750-72fde63545a9)
+
+Example demand distributions between regions 4-68 and 4-74
+
+![image](https://github.com/user-attachments/assets/577ce47c-f271-4fe7-ac49-6bb652e2541f)
+
+Evaluation metrics
+
+RMSE, MAE, MAPE, R^2
+
+Baselines
+
+HA, MLP, LSTM, GRU, ConvLSTM, GCN, TGCN, TMGCN
+
+Results
+
+![image](https://github.com/user-attachments/assets/cba1502b-07d5-4bf2-9416-4c722feb3f05)
+
+Example of OD demand prediction
+
+![image](https://github.com/user-attachments/assets/2a9c2169-a3d3-4080-940e-1ab062987f3f)
+
+### Second paper - Ride-hailing origin-destination demand prediction with spatiotemporal information fusion
+
+Introduction
+
+Traditional methods for OD demand prediction often rely on static models and linear approximations, which fail to account for dynamic and complex traffic conditions. Recent approaches employ machine learning techniques, such as clustering and neural networks, but they still face limitations in handling high-dimensional, sparse data and incorporating spatial-temporal dependencies effectively.
+
+Deep learning, particularly generative adversarial networks (GANs), offers advanced capabilities for capturing complex spatial-temporal patterns in traffic data. Various GAN models, including conditional GANs (CGANs) and Wasserstein GANs (WGANs), have shown promise in traffic prediction tasks.
+
+This paper proposes a novel approach using a CGAN with a Wasserstein divergence objective (CWGAN-div) for predicting ride-sourcing OD demand. This model integrates the strengths of CGAN and WGAN-div to ensure stable training and accurate prediction by capturing both internal and external spatial-temporal correlations.
+
+Methodology
+
+![image](https://github.com/user-attachments/assets/262f58a2-968f-4f27-9411-991ab88d9c1f)
+![image](https://github.com/user-attachments/assets/979855cb-fe06-4a3a-9cad-5fe04f8977a7)
+
+Results
+
+![image](https://github.com/user-attachments/assets/6fae7d3b-ed7d-4165-89bc-d3045e0bc7b0)
+![image](https://github.com/user-attachments/assets/871de676-b092-4abe-ad30-4ba6cfbe6734)
+![image](https://github.com/user-attachments/assets/f7e8c075-ff5d-4db2-8e42-29e3c8b36c32)
+
+### Third paper - GATv2
+
+Introduction
+
+Graph Attention Networks (GATs) are a widely used (and SoTA) type of Graph Neural Networks (GNNs) known for their effectiveness in learning representations from graphs. However, we found that GATs employ a limited form of attention where the ranking of attention scores is fixed and not dependent on the query node. We call this restricted attention "static" and contrast it with a more flexible "dynamic" attention. Due to this limitation, GATs struggle with certain graph problems, as demonstrated in controlled experiments where they fail to fit the training data. To address this, we propose GATv2, a modified version that utilizes dynamic attention, making it more expressive than traditional GATs - in GATv2, every node can attend to any other node.
+
+![image](https://github.com/user-attachments/assets/82ba6134-8148-4d1e-81cb-4cfb7a46afad)
+
+In GAT, the standard averaging or max-pooling aggregation technique is standardised and every node updates its representation by attending to its neighbours. This way the node can compute a weighted average of its neighbours, and (softly) select which are the most relevant.
+The expressive power of graph attention mechanisms
+
+![image](https://github.com/user-attachments/assets/bf4d498c-56d4-41c4-81a5-be802b92523b)
+
+Theorems
+
+![image](https://github.com/user-attachments/assets/5e833950-b0b8-425c-8936-5e7d2fda2e78)
+
+Evaluation
+
+![image](https://github.com/user-attachments/assets/8642538c-b42c-448e-a10b-0401a9d56950)
+
+![image](https://github.com/user-attachments/assets/b48bf74f-2005-483b-b510-4aa1f0ab8e62)
+
+Discussion
+
+Which graph attention mechanism should I use?
+
+'We believe that the more complex the interactions between nodes are – the more benefit a GNN can take from theoretically stronger graph attention mechanisms such as GATv2'.
+
+The author of GAT, said on twitter that GAT was designed to work in the 'easy-to-overfit' datasets of that time.
+
+### On another note ~
+
+Today I also re-read my notes in obsidian to get an idea again of what I have read and note down any potential ideas for research. I read lots of papers and I have an idea of what direction I want to go to, but still I am keeping my options open as I want to read a few more papers and explore my current ideas.
+
+The 'reference' graph does not make much sense now, but it is nice to see it grow.
+
+![image](https://github.com/user-attachments/assets/be657a0c-ad85-4d7b-8737-cb3794bbc581)
+
+That is all for today!
+
+See you tomorrow :)

From b4acf1544702c2a994cdfc3d0a9de17232269d93 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 16:38:58 +0900
Subject: [PATCH 268/638] Update 2024-06-11-day-162.md

---
 _posts/2024-06-11-day-162.md | 80 +++++++++++++++++++++++++++++++++++-
 1 file changed, 78 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-11-day-162.md b/_posts/2024-06-11-day-162.md
index cadf548b6..3363c1f09 100644
--- a/_posts/2024-06-11-day-162.md
+++ b/_posts/2024-06-11-day-162.md
@@ -1,5 +1,81 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 162) Deploying a mage.ai instance to aws
+categories: [applying-knowledge,mlops,reading-research]
 ---
+
+## Hello :) Today is Day 162!
+A quick summary of today:
+* yes! I finally completed the 03-orchestration part from the [MLOps zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/03-orchestration)
+* downloaded the NYC taxi dataset and started learning how to turn it into a graph dataset
+
+### Firstly, regarding mage.ai
+
+I cannot express how glad I am, to finish *completely* module 3. Here is my [repo](https://github.com/divakaivan/mlops).
+
+So many errors and deviations from what is shown on the videos (which were recorded 2 weeks ago...) I am speechless. Nevertheless, I finally did it all.
+
+Today I figured out some of the errors related to AWS authentication, and managed to deploy an instance to AWS using github actions. I hope I can use it, or what is probably better - use what I learned today as a template for an actual production-ready deployment. Because right now some things like database info and passwords are on github. 
+
+![image](https://github.com/user-attachments/assets/de8af7fe-57e2-4127-9dc0-d9ac9531141d)
+
+I played around with deployments ~. Next will be module 4 - deployment. 
+
+### Secondly, about what I did in the lab
+
+Firstly, today I read a nice paper: PGCN: Progressive Graph Convolutional Networks for Spatial–Temporal Traffic Forecasting
+
+Introduction
+
+This study introduces the Progressive Graph Convolutional Networks (PGCN) model for spatial-temporal traffic forecasting. Key features include:
+
+Trend Similarities: Uses parameterized cosine similarity to define trend similarities between traffic nodes.
+Progressive Graph Convolution Module: Constructs dynamic graphs based on trend similarities, adapting to online input signals during both training and testing.
+Spatial and Temporal Feature Extraction: Captures spatial correlations through progressive graph convolution and extracts temporal features using dilated causal convolution with a gated activation unit.
+The model demonstrates state-of-the-art performance on seven real-world datasets, consistently performing within 1.5% of the best models, and remains competitive even without structural network information.
+
+Progressive graph convolutional networks
+
+This is the core idea of PGCN
+
+Progressive graph construction
+
+Static features like POI category or speed limit are not always efficient for modelling node signal correlations, which change over time (e.g., morning vs. afternoon peak hours). Instead, this study proposes learning rich semantics from online traffic data by measuring node similarities using parameterised cosine similarity. Unlike previous methods, which only updated graphs during training, this study introduces a progressive graph that adapts to traffic changes based on trend similarities during both training and testing phases.
+
+![image](https://github.com/user-attachments/assets/8e491f6e-6089-466b-9b23-a9995c71b8d7)
+
+Dilated causal convolution
+
+Causal convolutions extract temporal features by stacking 1-D convolution layers while ensuring that future information is not considered for prediction. Causal convolution requires many layers to increase the size of the receptive field. To overcome this, dilated causal convolution is used.
+
+![image](https://github.com/user-attachments/assets/dc447f55-fedc-4de2-99fb-213229d56045)
+
+Overall architecture of PGCN
+
+![image](https://github.com/user-attachments/assets/1f36f1d9-b3be-49b0-bff3-7e4e87eee9bf)
+
+Each spatialtemporal layer consists of dilated casual convolutions, a gated activation unit, a progressive graph convolution module, and a residual connection. While multiple spatial-temporal layers are staked to increase the size of the reception field in temporal feature extraction, the parameter for the progressive graph constructor is shared across the layers. Finally, the output layer consists of a skip connection from each layer to prevent information loss from the initial layers and two fully connected layers with ReLU activation.
+Experiments
+
+Datasets
+
+PeMS-Bay, METR-LA, Urban-core, Seattle-Loop, PEMS04, PEMS08, PEMS07
+
+Baselines
+
+HA, FC-LSTM, DCRNN, GraphWaveNet, TGC-GRU, DMSTGCN, STSGCN, AGCRN, Z-GCNETs
+
+Results
+
+(only results on 3 of the datasets shown in picture)
+
+![image](https://github.com/user-attachments/assets/426f7c98-9adb-468d-a078-e7c2fc9d3b0b)
+
+Then, I decided to try to start recreating some of the taxi prediction papers. But to begin with I needed to learn how to transform the NYC taxi dataset (one of the most popularly used) into a temporal graph dataset.
+There are of course ways, including using networkx and pytorch_geometric, but today I could not find one that explains it well. I found [this](https://www.youtube.com/watch?v=XPTwvvlHaUA&t=1s&pp=ygUhY3JlYXRpbmcgYSB0ZW1wb3JhbCBncmFwaCBkYXRhc2V0) youtube video which touches not only on transforming table to graph dataset but also to temporal dataset. Tomorrow I need to look more into this. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From cdc1700d04725687ff71dcdfcd373d8466d4d95e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 16:42:57 +0900
Subject: [PATCH 269/638] Update 2024-06-12-day-163.md

---
 _posts/2024-06-12-day-163.md | 104 ++++++++++++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-12-day-163.md b/_posts/2024-06-12-day-163.md
index cadf548b6..67a16c347 100644
--- a/_posts/2024-06-12-day-163.md
+++ b/_posts/2024-06-12-day-163.md
@@ -1,5 +1,105 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 163) Reading about OD demand matrix prediction models
+categories: [reading-research,applying-knowledge]
 ---
+
+## Hello :) Today is Day 163!
+A quick summary of today:
+* started concentrating more on OD demand matrix papers
+* trying to recreate baseline models for OD demand matrix prediction
+
+Firstly, the papers I read
+
+### Deep Multi-View Spatiotemporal Virtual Graph Neural Network for Significant Citywide Ride-hailing Demand Prediction
+
+Introduction
+
+In spatial-temporal deep learning, two main spatial data representation methods are used: image-based and graph-based. The image-based approach grids urban areas by latitude and longitude, with statistical data as pixel values for CNN models. This approach struggles with data sparsity at high granularity and loss of detail at low granularity. The graph-based approach, used for defined networks like roads, captures dynamics via GCN models but has limited structured data access and transferability.
+
+The paper proposes a method using high-granularity grids of urban areas, discarding sparse regions, and retaining significant demand signals to create virtual nodes. A Deep Multi-View Spatial-temporal Virtual Graph Neural Network (DMVST-VGNN) is introduced, combining short-term temporal, spatial, and long-term temporal dynamics. This method uses gated 1D convolution for short-term dynamics, Graph Attention Networks for spatial dependencies, and Transformers for long-term dependencies. Contributions include:
+
+* Modelling virtual graphs of significant ride-hailing demand without external data, addressing spatial sparsity.
+* A novel multi-view deep learning model integrating different temporal and spatial dynamics.
+
+Methodology
+
+![image](https://github.com/user-attachments/assets/79ed1538-54ba-426f-97b7-1887a9b273e5)
+
+Graph generation
+
+The method proposes a novel approach to citywide spatial-temporal prediction by converting image-based spatial representation into a graph-based one. This involves three steps: discarding sparse regions, aggregating similar regions, and constructing virtual graphs. Sparse regions with low demand are discarded, similar regions are aggregated based on Pearson similarity, and virtual graphs representing distance, correlation, and mobility between aggregated regions are constructed
+
+![image](https://github.com/user-attachments/assets/99f67252-9b7f-44e8-96b1-7058e7c7ab0c)
+
+Short-term temporal dynamics view
+
+The approach considers using 1D CNN models to capture short-term temporal dynamics efficiently, as they offer comparable performance with lower computational requirements compared to RNN structures. The short-term temporal dynamics view involves two 1-D CNN units followed by gated linear units (GLU) for non-linearity activation. Each virtual node is processed by a 1D CNN to explore k temporal neighbors of the input time series, with padding to maintain sequence length.
+
+![image](https://github.com/user-attachments/assets/fa8cb075-58fe-46a4-8c8a-c71a5bcf2212)
+
+Spatial dynamics view
+
+In most previous works, ChebNet and Diffusion-Net are two common GCN model to capture spatial dynamics on traffic network. To simplify computation and consider the spatial physical meaning, we utilize spatial GCN model in this case. GAT is a fruitful spatial GCN model to capture spatial dependences, which adaptively calculates contribution of neighbor regions by attention mechanism. In this case, we generalize the traditional 2-D GAT to 3-D GAT, which has an additional time step length dimension M.
+
+![image](https://github.com/user-attachments/assets/627f7027-c125-4bd8-be36-98968b3ad991)
+
+Long-term temporal dynamics view
+
+This is actually a transformer layer that is applied to each node individually, and is composed of self-attention layer, position encoding mechanism and feed-forward output layer.
+
+Experiments
+
+Datasets
+
+NYC Uber data, NYC taxi data
+
+Baselines
+
+ARIMA, Random Forest, ConvLSTM, ST-ResNet, DMVST-Net, DCRNN, GraphWaveNet, Multi-GCN, ST-MGCN
+
+Results
+
+![image](https://github.com/user-attachments/assets/ab31c6b5-4abf-4b2e-8161-ddcc240b5282)
+![image](https://github.com/user-attachments/assets/bd04275f-bc42-4444-836c-0f06a242dfdf)
+
+### Hexagon-Based Convolutional Neural Network for Supply-Demand Forecasting of Ride-Sourcing Services 
+
+Introduction
+
+The paper discusses the effectiveness of hexagons over squares in capturing spatial correlations and proposes hexagon-based convolutional neural networks (H-CNN) for spatio-temporal forecasting. It introduces three versions of H-CNN compatible with standard deep learning packages, mapping hexagons to squares/tensors to preserve topology information. These models, combined with local map-to-map prediction and hexagon-based ensemble mechanisms, outperform benchmark algorithms in predicting ride-sourcing service demand-supply gaps, leveraging real-world datasets. Key contributions include the design of H-CNNs and mapping functions to enhance predictive performance in hexagon-based systems.
+
+Advantages of Hexagons
+
+1) Hexagons offer a clear definition of nearest neighbors, simplifying connectivity characterization in hierarchical network topology. Movements between zones can be more accurately represented due to equidistant neighboring zones.
+
+2) Hexagons have a smaller edge-to-area ratio compared to squares, reducing bias from edge effects and better capturing inflow/outflow characteristics between adjacent zones, crucial in transportation/urban computing.
+
+3) Hexagons exhibit greater isotropy, providing more consistent grid distance to straight-line distance ratios than squares. This enhances the characterisation of complex spatial structures, beneficial for various applications including simulation and neural network modelling.
+
+H-CNN framework
+
+![image](https://github.com/user-attachments/assets/50f446c7-54ad-489a-8323-cc2f083380df)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgFRO2QzleVGG7AuCVW1B0G2BQcRoXzpHmCvxgjhBCpebbilU6Nl4cG3JEbKJ6ql8oNC54521yga5scs99WFGxLiApn76ZhrlQuydS-u92oSYqgdTp36UI8vKpLa2qNZtT4TJv87LiM1ZCjNf5_PR2H2si2FqGwGolHkXpAJwV7d0_owaA24-SO80PDDlk9)
+![](https://blogger.googleusercontent.com/img/a/AVvXsEihRg0j546xf3-0w27E39opALtLwJZFUocLBgGzoWRxISunNEnCJ66j8fhKvOHrzKWB5VFx73mK9LgsV0R1DXxydvEjYVJcTNaA3kM83jVuXsIF7yDe0sbJ8I7R6RlNtxk9-XDV2IqVAU4XLnvyXHBbThRxpAr0t1XSv7q8YE6FOGV5XCWd-l398alcEB6F)
+
+Experiments
+Dataset
+Provided by the Bigdata Research Lab of Didi
+
+Baselines
+Lasso, Random Forest, MLP, XGBoost
+
+Results
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjXGbZZBToYNIGkqwI3KL_Sw3XQzzjtLfXOE1Wd3Cvze9UTIlGd4fdlOgS-GJBek_qVtuuQyvrfzmvCMmCZ1jlhqVnA-gfxcjCUKV7_V4YdkmG3zIEbvuYgMVjbbVA1Zk4POMu2R53hWsMQnKfJhgLf14BkVoB-UNkppbbWKd4kkETkBGhTZUvEazQtrM3E)
+
+Secondly, about trying to recreate baseline models
+Two of the first models I saw a lot during the literature review in the past almost month are Historical Average (HA) and Auto-regressive (or ARIMA). Today I think I recreated them which was not hard to be fair, but even tho I might be using the same dataset as some papers getting exactly the same results is not really possible because I might not be doing some specific preprocessing step, and without the code a paper used, I have no way to confirm. So I guess going forward I need to figure out how such things are done, confirmed for sanity.
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 98f6cbc7ddf572c0281c0fcb24c1ff1bc9a5a7d8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:15:04 +0900
Subject: [PATCH 270/638] Update 2024-06-13-day-164.md

---
 _posts/2024-06-13-day-164.md | 47 ++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-13-day-164.md b/_posts/2024-06-13-day-164.md
index cadf548b6..aa54c8950 100644
--- a/_posts/2024-06-13-day-164.md
+++ b/_posts/2024-06-13-day-164.md
@@ -1,5 +1,48 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 164) Learning about model deployment (and deleting AWS services)
+categories: [applying-knowledge,mlops]
 ---
+
+## Hello :) Today is Day 164!
+A quick summary of today:
+* started [Module 4](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/04-deployment): deployment from the MLOps zoomcamp
+* re-creating a baseline OD demand matrix model - ConvLSTM
+
+### Frstly, about module 4: deployment
+
+Code from today is on my [github](https://github.com/divakaivan/MLOps-camp). The first part was about simple model packaging with pickle and serving the model through a flask api
+
+![image](https://github.com/user-attachments/assets/92a88777-c6aa-4bfe-99ed-ece111694f38)
+
+Second part was about using mlflow and saving the ran experiments to S3 on AWS
+The code is pretty much like the one in the pic above, but with the below added above it:
+
+```
+RUN_ID = ""
+
+AWS_S3_BUCKET = ""
+
+logged_model = f's3://{AWS_S3_BUCKET}/1/{RUN_ID}/artifacts/model'
+
+model = mlflow.pyfunc.load_model(logged_model)
+```
+
+And this is when I realised I am getting charged by AWS. It is only $9, but I cannot continue using it. I thought I am under some 'free tier' but nope... I started deleting all EC2, RDS, Load balancers, Users, everything I could see. But because I am not completely sure, I wanted to use some cloud-nuke tool - and faced multiple errors with it. So finally, I contacted AWS support and they let me know what else I need to delete/cancel so that I do not get charged more. For any AWS/cloud parts, I might skip for now, at least until I figure out how can I get free credits for GCP (or if there are for AWS). 
+
+The next bit is about streaming models and using AWS Kinesis and Lambda, but I will just watch along on this one haha.
+
+
+
+### Secondly, about what I read and did in the lab today
+
+Mostly, I was trying to recreate baseline models. Today I had my eye on [ConvLSTM](https://paperswithcode.com/method/convlstm)
+
+![image](https://github.com/user-attachments/assets/c675b28c-06db-40b6-9b88-04645098d934)
+
+I only took this picture - showing predicted and ground truth OD matrix (and the diff). It took some time to train because my google colab good GPU compute credits were exhausted :/ Tomorrow I might bring my laptop to the lab so I can use it to train and play around with hyper params. 
+
+
+That is all for today!
+
+See you tomorrow :)

From bfdb7571914a29ad08d3d8b71c48f0d786a24e8c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:17:02 +0900
Subject: [PATCH 271/638] Update 2024-06-14-day-165.md

---
 _posts/2024-06-14-day-165.md | 37 ++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-14-day-165.md b/_posts/2024-06-14-day-165.md
index cadf548b6..a343ad1c4 100644
--- a/_posts/2024-06-14-day-165.md
+++ b/_posts/2024-06-14-day-165.md
@@ -1,5 +1,38 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 165) Starting to use mlflow for my research's model tracking + homework 4 of the MLOps zoomcamp
+categories: [mlops,applying-knowledge,reading-research]
 ---
+
+## Hello :) Today is Day 165!
+A quick summary of today:
+* started using mlflow to track experiments for my research in the lab
+* completed MLOps zoomcamp [Module 4](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/04-deployment) and its [homework](https://github.com/divakaivan/MLOps-camp/tree/main/Module%204/homework)
+
+### Firstly, about experiment tracking with mlflow
+
+I got a feeling that I will run lots of models from now on with different parameters, and hey, I have been learning and using this cool library called mlflow in my studies - this would be a great opportunity to use it in a real scenario. 
+
+So I set it up and just started running models. The training is a bit slow because my GPU is not that strong, but I found ways to get free GPU hours. I also want to find a way to host a simple sqlite db to use for the experiment trackings/artifact store. 
+
+My initial thoughts on sharing code that I will use for my research - I will definitely do it. I do not like that many papers do not share it for one reason or another, but for now I will keep it closer to my chest while I am still replicating base models. 
+
+### Secondly, about finishing module 4 and its homework
+
+I think the main new thing I got was about batch vs streaming model deployment. The former involves processing data in large, pre-defined chunks at specific intervals. The model is applied to a batch of data all at once, and the results are produced collectively after processing the entire batch. The latter, involves processing data in real-time as it arrives. The model is applied to each data point (or small batch) as soon as it is available, and the results are produced immediately. There was a nice 1 hour video on streaming deployment using AWS, but I just watched that one for now after yesterday's fiasco with the incurred bill. 
+
+The homework was about converting a jupyter notebook to a script and making it executable by taking argparser params: something like python starter.py --year 2024 --month 3 which runs the taxi duration prediction model on March 2024 NYC taxi data. 
+
+Then setting up an environment with pipenv, and creating a docker image and Dockerfile
+
+![image](https://github.com/user-attachments/assets/021ef76f-8dfa-4ec6-a750-44e161c80ae9)
+
+Which then allowed me to run the script through docker-run. 
+ 
+Tomorrow, I am going up to Seoul for a '[pseudocon](https://event-us.kr/pseudolab/event/85206?utm_source=mail&utm_medium=e_remind&utm_campaign=event_remind&utm_content=eventname_btn)'. The poster is in Korean but the even tldr is ... AI. Tomorrow I will share about the sessions/tutorials.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From ee1a7533e13944174de84e7cb7d3ea92be3d0674 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:19:16 +0900
Subject: [PATCH 272/638] Update 2024-06-15-day-166.md

---
 _posts/2024-06-15-day-166.md | 42 ++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-15-day-166.md b/_posts/2024-06-15-day-166.md
index cadf548b6..5de8ea11e 100644
--- a/_posts/2024-06-15-day-166.md
+++ b/_posts/2024-06-15-day-166.md
@@ -1,5 +1,43 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 166) Buying a new book + pseudocon in Seoul + using Gradio for a quick demo app
+categories: [applying-knowledge,stats]
 ---
+
+## Hello :) Today is Day 166!
+A quick summary of today:
+* attendend the 8th pseudocon by pseudolabs in Seoul
+* went to a big book store and bought a cool little book 
+* created a space on huggingface for fine-tuned whisper on our Glaswegian dataset
+
+### Firstly, about the pseudocon
+
+![image](https://github.com/user-attachments/assets/d942f326-3f0e-4d28-a76f-2085138191af)
+
+It was decently big - many people. And was organised in the same Microsoft office area as the Microsoft Azure hackathon I attended a few weeks ago. 
+Some interesting bits from the sessions which covered a wide range - open source, genAI, AWS, stable diffusion and games. 
+
+* Found out that [firebase](https://firebase.google.com/) has free db hosting (again?). I used it back in 2018 when I was initially learning about webapp dev so I might give 'an old friend' a try again
+* Found a nice community of AWS learners - [startAWS](https://startaws.com/)
+* There was a session on using AI to generate characters for games. It is a touchy topic among gamers and even more among artists, but it is something that as presented today 'can massively decrease cost'
+* Also in the final session I got some extra resources for building RAGs (in Korean)
+
+### Secondly, about my new book
+
+![image](https://github.com/user-attachments/assets/a2d4f097-bee5-431b-854e-8f1290e661f2)
+
+Next to the microsoft office is a big book store where I know they have imported English books. So I had a look at the probability/stats section for something interesting. And this one was just 10,000 won (~10$) which was very surprising for an imported book. I read a bit on my ride back home, and it includes topics like: multivariate analysis, matrix algebra, displaying multivariate data, tests of significance with multivariate data, measuring and testing multivariate distances, PCA, factor analysis, discriminant function analysis, cluster analysis, canonical correlation analysis, multidim scaling, and ordination. Not new topics, and not topics that I have not read about in other books. But I like having books in my personal library, and importing foreign books in Korea is very expensive for me. So I will happily read this one. 
+
+### Thirdly, a demo webapp for the Glaswegian dataset
+
+Today in the free holes of time I got, I wanted to create a small demo app so that my Scottish partner can use it to test different audios so that we can continue adding audio to our dataset. Initially, I wanted with streamlit, but then I realised that huggingface spaces exist. 
+
+![image](https://github.com/user-attachments/assets/6c9c1bbf-fbe9-45bd-a57e-edde4404b7cd)
+
+With the above few lines of code, the webapp is hosted in just a few seconds. Amazing ^^
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From 7c2a388c6802a4aa2715448769fbe2746059cca0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:22:31 +0900
Subject: [PATCH 273/638] Update 2024-06-16-day-167.md

---
 _posts/2024-06-16-day-167.md | 51 ++++++++++++++++++++++++++++++++++--
 1 file changed, 49 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-16-day-167.md b/_posts/2024-06-16-day-167.md
index cadf548b6..f6d117365 100644
--- a/_posts/2024-06-16-day-167.md
+++ b/_posts/2024-06-16-day-167.md
@@ -1,5 +1,52 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 167) Learning about model monitoring
+categories: [applying-knowledge,mlops]
 ---
+
+## Hello :) Today is Day 167!
+A quick summary of today:
+* started [Module 5](https://github.com/DataTalksClub/mlops-zoomcamp/tree/main/05-monitoring): Monitoring of the MLOps zoomcamp
+* read a bit of Multivariate statistical methods: a primer (which I bought yesterday)
+
+### Firstly, about ML model monitoring
+
+![image](https://github.com/user-attachments/assets/bcce1bff-0eed-435f-a2e3-69e6999f78e6)
+
+ML monitoring involves looking at things like service health, model performance, data quality and integrity, data and concept drift, performance by segment, model bias/fairness, outliers, explainability. 
+
+![image](https://github.com/user-attachments/assets/386cac4c-15bc-49e8-bb14-42e956274b06)
+
+[Here](https://github.com/divakaivan/MLOps-camp/tree/main/Module%205) are the materials over which I studied today.
+Using evidently's python library we can great nice looking reports and dashboards like the below. 
+
+In this, 2 created reports can be seen. 
+
+![image](https://github.com/user-attachments/assets/ba2309ad-202b-40cf-86d1-ce953c14ca3e)
+
+Below is one of the default suggested dataset summary tables
+
+![image](https://github.com/user-attachments/assets/0dd36f9a-eac5-41ea-9a7f-6da0bc5f3026)
+
+Along with variable info
+
+![image](https://github.com/user-attachments/assets/051d73ce-c868-47b5-9a13-9d991b9dbac3)
+![image](https://github.com/user-attachments/assets/fb5004cf-3a15-4dc7-b9cc-59839f61ccc5)
+![image](https://github.com/user-attachments/assets/c170cc97-dfeb-42fc-ab37-2c2385fa30b2)
+
+And this is an example of a simple dashboard we can use to monitor specific items from our reports
+
+![image](https://github.com/user-attachments/assets/cedaf9ce-1ceb-44e3-96bc-d80342da158c)
+
+
+### Secondly, some things from the book
+
+* examples of multivariate data
+* matrix algebra (vectors, matrices, eigenvalues/vectors)
+* visualising multivariate data (draftsman's plot - scatter plot matrix, lines that show profile of variables)
+* tests of significance with multivariate data (Hotelling's T-squared test, Box's M-test, Levene's test, Van Valen's test, Wilk's lambda statistic, there were many more that appeared throughout the 20th century)
+
+
+That is all for today!
+
+See you tomorrow :)

From 41b8c0f1f371236abbe651288199ed831cff24bc Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:26:02 +0900
Subject: [PATCH 274/638] Update 2024-06-17-day-168.md

---
 _posts/2024-06-17-day-168.md | 75 +++++++++++++++++++++++++++++++++++-
 1 file changed, 73 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-17-day-168.md b/_posts/2024-06-17-day-168.md
index cadf548b6..e5e2f0f14 100644
--- a/_posts/2024-06-17-day-168.md
+++ b/_posts/2024-06-17-day-168.md
@@ -1,5 +1,76 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 168) First day of Internship Experience UK - Technology by Bright network
+categories: [applying-knowledge]
 ---
+
+## Hello :) Today is Day 168!
+A quick summary of today:
+* attended day 1 (of 4) of [Internship Experience UK](https://youtu.be/qwna3sCOopU): Technology by Bright Network
+* did MLOps zoomcamp's [homework 5](https://github.com/DataTalksClub/mlops-zoomcamp/blob/main/cohorts/2024/05-monitoring/homework.md) on model monitoring
+* discovering LLM zoomcamp
+
+![image](https://github.com/user-attachments/assets/62c8a83a-ebe7-469a-b86c-8c355906ba08)
+
+For completeness: here is the full schedule, and there are various sessions by top employers in the UK on building up our careers. Below are some notes from today's sessions
+
+![image](https://github.com/user-attachments/assets/725ec838-e8c2-4256-96e4-c0344a9c1196)
+
+Session on leadership skills with Deloitte
+
+* creative problem-solving
+  * identify challenges
+  * generate solutions
+  * making mistakes
+  * skills in practice
+* empowering your team and embracing responsibility
+  * encourage diverse perspectives
+  * build trust
+  * accountability and reliability
+* there was a task where we had to think about being a leader in a 2-week deadline merger&acquisition case - how to organise, manage communication, strategy and deadlines
+* using the Eisenhower matrix for task management
+
+![image](https://github.com/user-attachments/assets/92b4c768-03a5-42b4-bd0b-d071c6505a83)
+
+Session on transferable skills by ICAEW
+
+* teamwork
+* people skills
+* decision making & problem solving
+* professionalism
+* commercial awareness
+* IT skills
+
+Session on building your personal brand with PwC
+
+Your personal brand is how you promote yourself. It is about how you present yourself, what are your values - what do you stand for. 
+
+'Your brand is what others say about you when you're not in the room.' - Jeff Bezos
+
+Personal brand mistakes
+
+* careful with online presence
+* communication skills and knowing your environment and how to speak in the workplace
+* integrity
+* professionalism
+
+Tomorrow is when the Tech-specific sessions start and also the day ends at ~2am for me, so it will be a long day. 
+
+Finally
+
+Right now when I was closing tabs, I saw on youtube a livestream by DataTalksClub. Their [LLM zoomcamp](https://github.com/DataTalksClub/llm-zoomcamp/tree/main) is starting today - and it is the first time they are running it. So I am thinking of joining because at the end (just like their other courses) I will have a nice project that I can add to my portfolio. 
+
+It covers:
+
+1. Introduction to LLMs and RAG
+2. Open-source LLMs and self-hosting LLMs
+3. Vector databases and retrieval techniques
+4. Workshop: dlt
+5. LLM orchestration and ingestion pipelines
+6. Monitoring and Guardrails
+7. Tips and Tricks for advanced RAG systems
+8. Project
+
+That is all for today!
+
+See you tomorrow :)

From e356ff79591461c3850f1cb58055f1325fa09913 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:28:28 +0900
Subject: [PATCH 275/638] Update 2024-06-18-day-169.md

---
 _posts/2024-06-18-day-169.md | 68 ++++++++++++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-18-day-169.md b/_posts/2024-06-18-day-169.md
index cadf548b6..1fb3629d3 100644
--- a/_posts/2024-06-18-day-169.md
+++ b/_posts/2024-06-18-day-169.md
@@ -1,5 +1,69 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 169) Writing first version of introduction for my paper + day 2 of IEUK
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 169!
+A quick summary of today:
+* started writing introduction for my paper
+* attended day 2 of IEUK
+
+### Firstly, in regards to my paper
+
+I decided to give overleaf a go. It is a popular latex editor so I went with it. I will share with white covering most of the part I bit.
+
+![image](https://github.com/user-attachments/assets/b739ca07-fae8-4e98-b383-fc0a3c2dc0b3)
+
+I managed to write a whole page. Not sure if that is a lot, but I am quick to recall what I have read. Every time I wrote a university assignment or previous short research proposals, I end up changing the sections a lot. It is pretty normal (for me at least) that the initially written version is never the final version. Nevertheless, I also learned about bibtex and how its used for citations and referencing. At the moment I am thinking to give myself time until the end of June to write this intro + literature review + data plan + expected results, and then submit to my professor for review. And it would be good to do it at the end of June since it is exam season now for him. 
+
+(I am also a little paranoid that because I am using the free version my progress will be lost haha. So just to be extra safe I copy-pasted everything into a google drive docs file)
+
+I think a cool app to do in the future is a 'homemade' RAG app that takes all the read papers and then I can ask it questions. Because today I knew I had read some parts somewhere but it was not in my notes, so I ended up using 1 word ctrl+F searches over many papers. 
+
+### As for Internship Experience UK day 2
+
+I think the only highlight of the day for me is hearing from Lloyds Banking Group employees. It felt I was back there again. 
+
+What skills do we need to become a data scientist?
+
+* computer science
+  * python and sql
+  * cloud platforms
+  * model performance and security
+* maths/statistics
+  * statistical models
+  * forecasting
+  * optimisation
+  * ML
+  * DL
+* business/domain knowledge
+  * understanding business problems
+  * applying domain specific insight
+  * adding business value
+
+What types of data scientists are in the Bank?
+
+data scientist modeller
+* builds ML models which find patterns in data to make predictions
+* key skills: python/coding, stats, ML, business knowledge
+
+ML engineer
+* productionises ML models
+* key skills: python/coding, cloud platforms, software engineering, ML
+
+data science leader
+* uses DS experience and knowledge to lead teams and translate business problems into DS solutions
+* key skills: business knowledge, communication, ML, stats
+
+AI ethics expert
+* creates the tools, frameworks and guardrails which ensures DS and AI are developed responsibly
+* key skills: AI ethics, governance, data privacy, security, ML
+
+Finally, the IEUK team assigned us some mini-project where we play to role of a project manager and we have to deliver on some items. I will talk about it in tomorrow's post because it is 2am now. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 25d5b08b825849f88ee16180c0aa6b18448a3ffa Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:31:59 +0900
Subject: [PATCH 276/638] Update 2024-06-19-day-170.md

---
 _posts/2024-06-19-day-170.md | 68 ++++++++++++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-19-day-170.md b/_posts/2024-06-19-day-170.md
index cadf548b6..ec50cd905 100644
--- a/_posts/2024-06-19-day-170.md
+++ b/_posts/2024-06-19-day-170.md
@@ -1,5 +1,69 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 170) Uber data engineering project using GCP and Mage
+categories: [mlops,applying-knowledge,reading-research,cloud]
 ---
+
+## Hello :) Today is Day 170!
+A quick summary of today:
+* decided to give mage.ai another go and now use it alongside GCP
+
+Well... I obviously did not suffer enough with the countless problems I experienced when I was first learning about mage from DataTalksClub's MLOps zoomcamp, so I decided to do a cool-looking data engineering project [[youtube](https://youtu.be/WpQECq5Hx9g)]. The caveat is that I will use GCP. And I just hope I do not incur any major costs. I checked, and I have 21 days left on plenty of free credits. 
+
+Getting to the project
+
+It uses the infamous NYC taxi dataset. 
+
+![image](https://github.com/user-attachments/assets/f8433aaa-4dd1-46d5-934b-bf4026101ce0)
+
+Using lucid I learned a bit amout data deminsion modelling
+
+Then using python, some basic preprocessing was done on the raw data, to convert it into the top 8 tables. I actually put everything so far on my [github](https://github.com/divakaivan/taxi-data-eng-project), and plan on doing a nice readme documentation once everything is finished. Even though I started it after work today, I did not finish it because of *again* mage problems. 
+
+Before I get to the mage problems, a bit about GCP.
+
+I set up a cloud storage (similar to S3 in AWS) and uploaded the raw data.
+
+Then I set up a VM with the right access permissions. And this gave ma nice SSH-in browser to install python and run mage. There is a lot of IP addreses, so I am afraid to share pictures today. 
+
+Using that VM, I installed python, pip and mage, and started a project.
+
+Added a block to import data, and a transformer block that basically follows the jupyter notebook in my repo. However whenever I run the transformer block that ends up with the above 8 tables, the kernel restarts and seems like it never starts again. I restart mage through the terminal, same thing happens.
+
+Turns out mage bugs out when I was converting 8 datasets to dicts, and when I just retirn 8 dataframes as a tuple - it is all fine.
+
+Then I created a user and credentials so I can connect to big query and push data to it, 
+
+The final mage pipeline looks like this:
+
+![image](https://github.com/user-attachments/assets/74a0d803-8637-40ff-a885-c27bbb712bcf)
+
+And ~
+
+![image](https://github.com/user-attachments/assets/e01bcf4e-f824-4ced-a451-51ce6f11c0a3)
+
+yay~~~
+This took a bit of debugging as well due to uninstalled packages and putting access keys and secrets i n the right place and format. 
+
+FYI, this is the project stack:
+
+![image](https://github.com/user-attachments/assets/8772238a-c089-4f5c-8200-1685c6ce13ce)
+
+So for the final step ~ using looker for some kind of a visualisation dashboard.
+
+This is the final dashboard (unsurprisingly, looker is very similar to powerBI). Looks awesome. Can't wait to apply what I learned to data of my choosing.
+
+![image](https://github.com/user-attachments/assets/49c10829-96d6-49f1-a63a-da80286aeae0)
+
+Lastly, I deleted everything from GCP, but I know what I need next time. ^^
+
+Just a quick mention - today I kept reviewing my intro that I wrote yesterday, and added a table which I can share. 
+
+![image](https://github.com/user-attachments/assets/c3b0b006-f842-45ab-b208-6c2e82d3c72f)
+
+All of these papers' summaries/notes I have shared throughout the last few weeks, and I was taught that tables like that (and even more detailed) are a good idea in a paper.
+
+
+That is all for today!
+
+See you tomorrow :)

From a853537696c47ae603e80e9d7cdf4b05476ef9ec Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:37:12 +0900
Subject: [PATCH 277/638] Update 2024-06-20-day-171.md

---
 _posts/2024-06-20-day-171.md | 79 +++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-20-day-171.md b/_posts/2024-06-20-day-171.md
index cadf548b6..062087f51 100644
--- a/_posts/2024-06-20-day-171.md
+++ b/_posts/2024-06-20-day-171.md
@@ -1,5 +1,80 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 171) Data engineering zoomcamp by DataTalksClub
+categories: [data-eng,applying-knowledge]
 ---
+
+## Hello :) Today is Day 171!
+A quick summary of today:
+* started [Data engineering zoomcamp](https://github.com/DataTalksClub/data-engineering-zoomcamp) by DataTalksClub
+* completed IEUK by Bright Network
+
+### Firstly, about the data engineering zoomcamp
+
+I know they deliver good content, and ends up with a solo project that will be a great addition to my portfolio. 
+
+This is the camp overview:
+
+![image](https://github.com/user-attachments/assets/3bdf17dd-0f06-480a-96cc-c3d82fbe7bc1)
+
+Tons of new things to learn, some old friends (in mage.ai), kafka, dlt, spark, dbt. It was a no brainer decision. So I guess at the moment on different days of the week (I need to set up a schedule) I am taking their data engineering camp, their MLOps camp, and LLM camp. 
+
+As for covered material for today~
+
+Learned about docker, and using postgres on docker. 
+
+Created this dockerfile
+
+![image](https://github.com/user-attachments/assets/a29a0485-795b-4989-8519-df5c3aefee84)
+
+And then below are some of the ran comands and steps covered.
+
+Start the postgres server on docker
+
+![image](https://github.com/user-attachments/assets/c14d9d9b-48ab-4b64-8202-e123a9458a26)
+
+And access it through the terminal
+
+`pgcli -h localhost -p 5432 -u root -d ny_taxi`
+
+Then, using python, I uploaded data and I could query it from the terminal where I ran pgclie
+
+![image](https://github.com/user-attachments/assets/cf0f2b85-2787-4caa-8928-f07c9375ea94)
+
+For better user experience, I started a pgadmin server in docker
+
+![image](https://github.com/user-attachments/assets/1ed8758f-6740-4de0-b802-dafe70c205e0)
+
+And I could access it on port 8080, however, I could not add a connection to the created table. This is because the two docker instances (postgres and pgadmin) were not connected. To connect them, I used:
+
+`docker network create pg-network`
+
+And then I ran the postgres and pgadmin docker containers in 1 network
+
+![image](https://github.com/user-attachments/assets/1e03e52c-50c3-40c6-a1e1-621ed925ce09)
+
+And I could make a connection, view and query the data in pgadmin
+
+![image](https://github.com/user-attachments/assets/1198db9a-b8ed-488e-882b-01d1f3c2e2ad)
+
+However, the above two commands required me to run them in 2 separate terminals. To make this a bit more convenient, I used docker-compose.
+
+We first set up a docker-compose.yaml file
+
+![image](https://github.com/user-attachments/assets/29ab7d06-ac4b-4cb7-ab51-e9245706195f)
+
+And *now* I understand what is going on inside this file, and what docker-compose does for us - it combines containers. Awesome!
+
+There are a few videos left from module 1, but everything from today, incuding my command docs are in Module 1 of my [repo](https://github.com/divakaivan/data-eng-camp).
+
+### Secondly, for IEUK by Bright Network
+
+The event ended with the submission of some kind of a work sample where we act as a product manager. 
+
+I can share a bit of mine. It is the product roadmap that I developed. Overall it was a good exercise because those are new things for me ~
+
+![image](https://github.com/user-attachments/assets/f6c34ff7-8b46-4d13-9736-dc7884b6d889)
+
+That is all for today!
+
+See you tomorrow :)

From e14a1951f8ce256eddf4f6473d546d7dc8d1398a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:41:23 +0900
Subject: [PATCH 278/638] Update 2024-06-21-day-172.md

---
 _posts/2024-06-21-day-172.md | 55 ++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-21-day-172.md b/_posts/2024-06-21-day-172.md
index cadf548b6..0645308b9 100644
--- a/_posts/2024-06-21-day-172.md
+++ b/_posts/2024-06-21-day-172.md
@@ -1,5 +1,56 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 172) Learning about terraform + adding more data to the Glaswegian audio dataset
+categories: [data-eng,applying-knowledge,glaswegian]
 ---
+
+## Hello :) Today is Day 172!
+A quick summary of today:
+* preprocessed and loaded more audio into the Glaswegian audio dataset on [huggingface](https://huggingface.co/datasets/divakaivan/glaswegian_audio)
+* learned about Terraform from Data Eng Zoomcamp
+
+### As for the Glaswegian dataset
+
+![image](https://github.com/user-attachments/assets/1a5022a1-df3c-4a59-a1af-05870a3af066)
+
+Preprocessing took a bit longer because now I was cutting multiple >5min audio clips and matching to the transcription that my Scottish collaborator had written down. I actually did only 4 out of the 6 so I have 2 more to do from Limmy - a famous Scottish comedian.
+The total time at the moment is 63 minutes.
+
+I started fine-tuning whisper-small again, but turns out the colab subscription I got (the cheapest one) includes a limited amount of computer units. So right now I am finetuning it on the free limited amount. It says it takes about 4 hours... but hopefully it finishes before the free TPU hours run out. 
+
+### As for Terraform
+
+Continuing yesterday's videos. The final part of Module 1 from the DataTalksClub's data engineering camp is an intro to terraform. Fortunately, I have met terraform before - at the Microsoft Azure hackathon, but then I had no idea what I was running, I was just running it. So today it got (a little but) clearer.  
+
+First, I created a service account in GCP and give it access
+
+![image](https://github.com/user-attachments/assets/901cd482-4c3a-4c92-a6e2-ad2caf906eae)
+
+Create a manage key
+
+![image](https://github.com/user-attachments/assets/12ce6a9a-fca9-418a-bd43-b1470bb93ac2)
+
+Create a main.tf file and use google provider to set up terraform (using terraform init)
+
+![image](https://github.com/user-attachments/assets/87a4b32d-0032-48af-8eb3-a4836984aa3f)
+
+After initialising terraform with my service account.
+
+Next is creating a cloud storage bucket resource (adding the below to main.tf) (and then executing terraform plan)
+
+First I needed to set my GOOGLE_APPLICATION_CREDENTIALS env var and then it worked.
+
+![image](https://github.com/user-attachments/assets/b8920c65-e01f-4548-9617-8e6816684bae)
+![image](https://github.com/user-attachments/assets/a418cc31-2df4-47e8-8784-218b29af8638)
+
+Next, to create the bucket, I executed terraform apply, and then I see the created storage in GCP
+
+![image](https://github.com/user-attachments/assets/0e53917f-40e9-4c85-acaa-3f3507891d7a)
+
+We can destroy the created bucket with terraform destroy.
+The main.tf file is on my [repo](https://github.com/divakaivan/data-eng-camp/tree/main/Module%201/terrademo).
+
+
+That is all for today!
+
+See you tomorrow :)

From e54ce397b7e7307d6bde9222c91f161b1a9cbe9b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:41:53 +0900
Subject: [PATCH 279/638] Update 2024-06-21-day-172.md

---
 _posts/2024-06-21-day-172.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-06-21-day-172.md b/_posts/2024-06-21-day-172.md
index 0645308b9..868987587 100644
--- a/_posts/2024-06-21-day-172.md
+++ b/_posts/2024-06-21-day-172.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 172) Learning about terraform + adding more data to the Glaswegian audio dataset
-categories: [data-eng,applying-knowledge,glaswegian]
+categories: [data-eng,cloud,applying-knowledge,glaswegian]
 ---
 
 ## Hello :) Today is Day 172!

From 2f712773a9936feab7cf418804b8bb43b3e0e867 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:42:13 +0900
Subject: [PATCH 280/638] Update 2024-06-19-day-170.md

---
 _posts/2024-06-19-day-170.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-06-19-day-170.md b/_posts/2024-06-19-day-170.md
index ec50cd905..28d48d99e 100644
--- a/_posts/2024-06-19-day-170.md
+++ b/_posts/2024-06-19-day-170.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 170) Uber data engineering project using GCP and Mage
-categories: [mlops,applying-knowledge,reading-research,cloud]
+categories: [data-eng,applying-knowledge,reading-research,cloud]
 ---
 
 ## Hello :) Today is Day 170!

From a8d1b47df23e4f9782ec2f6e0c214ffa01ce11d1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:47:29 +0900
Subject: [PATCH 281/638] Update 2024-06-22-day-173.md

---
 _posts/2024-06-22-day-173.md | 167 ++++++++++++++++++++++++++++++++++-
 1 file changed, 165 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-22-day-173.md b/_posts/2024-06-22-day-173.md
index cadf548b6..998cf11ae 100644
--- a/_posts/2024-06-22-day-173.md
+++ b/_posts/2024-06-22-day-173.md
@@ -1,5 +1,168 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 173) Terraform, GCP, virtual machines, data pipelines
+categories: [data-eng,cloud,applying-knowledge]
 ---
+
+## Hello :) Today is Day 173!
+A quick summary of today:
+* learned more about terraform and how to set up a GCP VM and connect to it locally
+* used mage for some data engineering pipelines with GCP
+
+### Last videos from Module 1: terraform variables, GCP set up
+
+Turns out there is a bit more of terraform from the data eng zoomcamp, and today I covered it.
+
+After learning how to connect to gcp using terraform and create a storage bucket, the first thing today was creating a bigquery dataset
+
+![image](https://github.com/user-attachments/assets/4d0d500d-5d9a-4e4b-85f0-04e6f716fa9e)
+
+Adding the above to main.tf which now looks like:
+
+![image](https://github.com/user-attachments/assets/48f8f05b-0e76-4947-b63d-a6b27cc8ca9f)
+
+terraform apply, creates a demo_dataset as well
+
+![image](https://github.com/user-attachments/assets/ccb04ba7-2e42-4e56-a8db-00bc7dc0b927)
+
+Then I learned about variables in terraform
+
+Create a variables.tf file and put a variable like:
+
+![image](https://github.com/user-attachments/assets/685a7f1e-0952-4f8d-8a87-9decdf3ab9ff)
+
+and in main.tf we can directly use the created variables like:
+
+![image](https://github.com/user-attachments/assets/186c76eb-7c3e-43d1-b7bc-d08dbba34d0b)
+
+Great intro to terraform - being able to define infrastructure code, create resources, and destroy resources.
+
+The next part was an instruction on setting up GCP (cloud VM + SSH access)
+
+First was creating an ssh key locally
+
+![image](https://github.com/user-attachments/assets/fe82c565-756f-4707-9e20-426dcb5813a1)
+
+And add it to the metadata in GCP's compute engine (hiding the username just in case)
+
+![image](https://github.com/user-attachments/assets/b8fd77c4-50ac-4003-9d6d-8b2bcf74778f)
+
+Then create a VM, and connect to it locally using that ssh key, using `ssh -i ~/.ssh/gcp username@gcp_vm_external_ip`
+
+![image](https://github.com/user-attachments/assets/90f569c1-a451-4e67-a9e1-5f0afa575ebb)
+![image](https://github.com/user-attachments/assets/87a10fd4-0beb-437c-99a9-3cbcc0ea12af)
+
+For a quick connection to the VM, I set up a config which includes Host, HostName, User and IdentityFile, so now I can just run ssh `Host` and I am connected to the VM through my terminal. Nice.
+
+Also set up vs code to connect to the created ssd
+
+![image](https://github.com/user-attachments/assets/76b80425-a50c-4bac-9b79-c1cdc8597c25)
+
+Then, I installed anaconda. 
+
+![image](https://github.com/user-attachments/assets/1ccb1f56-34b5-48a6-92c8-5413279201b1)
+
+And docker
+
+![image](https://github.com/user-attachments/assets/e756126f-b6b8-49ea-a4a8-fe059f888c0f)
+
+Then docker-compose
+
+![image](https://github.com/user-attachments/assets/8c0f113b-db47-42c9-b45d-5ce5c1e4f9f1)
+
+And make it executable from anywhere by adding the below to .bashrc
+`export PATH="${HOME}/bin:${PATH}"`
+
+And now we have it 
+
+![image](https://github.com/user-attachments/assets/23d2d024-c9bf-43b1-9421-a740f7e028db)
+
+Then installed pgcli with conda
+
+![image](https://github.com/user-attachments/assets/a47938bc-503e-42f1-90ba-3cb8202ab2eb)
+
+(random note - I am using a vm from my local terminal like this for the 1st time and its kind of cool)
+
+And just like before(2 days ago, first part of the data eng zoomcamp), I can run docker-compose up -d and then pgcli to connect to mt db
+
+![image](https://github.com/user-attachments/assets/689ab26c-3456-4874-a691-3ac16e8b1a46)
+![image](https://github.com/user-attachments/assets/0eb1c2f6-1b08-4b88-bc6e-51380d626e8d)
+
+In VS code that is connected to the VM, we can forward the port to the db
+
+![image](https://github.com/user-attachments/assets/bc3ae7c0-59cf-433b-8355-e6917156373f)
+
+And now when I run pgcli from my own PC's terminal, I can connect to it too.
+
+![image](https://github.com/user-attachments/assets/4b99d843-c4f7-43ea-b24e-1e032d61dd87)
+
+By adding port 8080 as well in VS code, I now can access pgadmin too from my browser (even tho it is all running on that GCP VM)
+
+![image](https://github.com/user-attachments/assets/76cbfd68-4337-4654-b6c1-a6ca84f1f723)
+
+Same for jupyter - added port 8888 in vs code, then I can run jupyter notebook in the VM, and access in my browser. 
+
+Next, I installed terraform for linux
+
+![image](https://github.com/user-attachments/assets/5bc1eea7-446d-4a12-8a28-0b8fa46bfab2)
+
+When I was setting up terraform, I created a my-creds.json with the credentials from GCP. Now using sftp I transferred the json file from my local to the VM (sftp - another tool I am using for the 1st time)
+
+![image](https://github.com/user-attachments/assets/2b158fb0-7717-4a0a-ba54-da8049fc2c7d)
+
+And then I could run the same terraform apply and destroy to create and destroy resources.
+
+And if I want to stop and restart the instance I can do it through the terminal (`sudo shutdown now`) or the GCP console. And when I start it again in order for the quick ssh connection command to work, I need to edit the config file's HostName that I created earlier. 
+I found that if I restart the VM and want to use terraform, I need to set my credentials and gcloud auth using (and also just saving the commands for later):
+
+`export GOOGLE_APPLICATION_CREDENTIALS=~/.gc/my-creds.json`
+`gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS`
+
+### Next onto [Module 2](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/02-workflow-orchestration#222---%EF%B8%8F-intro-to-mage): workflow orchestration
+
+My good old friend mage.ai. Let's hope for at least less errors than when I covered it in the MLOps zoomcamp. 
+
+The first bit was to establish a connection with the postgres database which is ran alongside mage in docker-compose.yml
+
+![image](https://github.com/user-attachments/assets/af92e004-10a8-4bbc-8a27-25df1f67e5a7)
+
+Creating a block in a new pipeline to test the connection:
+
+![image](https://github.com/user-attachments/assets/8cf45bc1-5fe6-4e2e-b5ef-b8fc69949172)
+
+So far so good.
+
+Next is writing a simple ETL pipeline - loading data from an API to postgres, where I just load taxi data in the first block using data type checking, do a little bit of preprocessing in the 2nd block and then make a connection to my db and load the data there in the 3rd block
+
+![image](https://github.com/user-attachments/assets/37fcd9a8-e416-4dc0-9e9f-f24224ee69a4)
+
+Next is connecting my gcp service account to mage (using the creds.json file), and a connection is made. 
+
+Loading data to google cloud storage is very easy - just adding a google cloud storage (GCS) data exporter block, and putting my info down and its done
+
+![image](https://github.com/user-attachments/assets/2ca381e8-49e4-4bbb-819d-8b655207c698)
+
+Can be seen in gcs
+
+![image](https://github.com/user-attachments/assets/d9f2cd8d-01bd-4ea5-b5f0-742070fe760e)
+
+However, with larger files we should not load data into a single parquet file. We should partition it.
+
+I learned how to use pyarrow (as it abstracts chunking logic) for that as in the below block:
+
+![image](https://github.com/user-attachments/assets/2f3d928b-37e2-430a-98d1-5b48621c4b42)
+
+And the partitioned data is in gcs now. Awesome
+
+![image](https://github.com/user-attachments/assets/cf10fec2-2c90-4080-8bdd-938c7395d736)
+
+And then load it into BigQuery
+
+![image](https://github.com/user-attachments/assets/ba0df419-911d-4d90-a780-c4b0c658ed7f)
+
+My experience using mage in this course compared to the MLOps zoomcamp is completely different haha. Now, the teacher Matt Palmer did a great job ^^
+
+
+That is all for today!
+
+See you tomorrow :)

From 0dde996b164822a5c2520607c14ab1e95cef53bd Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 19:55:29 +0900
Subject: [PATCH 282/638] Update 2024-06-23-day-174.md

---
 _posts/2024-06-23-day-174.md | 165 ++++++++++++++++++++++++++++++++++-
 1 file changed, 163 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-23-day-174.md b/_posts/2024-06-23-day-174.md
index cadf548b6..606d616e5 100644
--- a/_posts/2024-06-23-day-174.md
+++ b/_posts/2024-06-23-day-174.md
@@ -1,5 +1,166 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 174) Starting LLM zoomcamp + Learning about Data Warehouses + BigQuery
+categories: [nlp,data-eng,cloud,applying-knowledge]
 ---
+
+## Hello :) Today is Day 174!
+A quick summary of today:
+* did [Module 1](https://github.com/DataTalksClub/llm-zoomcamp/tree/main/01-intro) and its [homework](https://github.com/DataTalksClub/llm-zoomcamp/blob/main/cohorts/2024/01-intro/homework.md) from the DataTalksClub's LLM camp
+* covered [Module 2](https://github.com/DataTalksClub/llm-zoomcamp/tree/main/02-open-source) using free models and creating a simple FAQ bot
+* covered [Module 3](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/03-data-warehouse) from the data engineering camp on data warehouses
+
+### LLM camp module 1: basic RAG app
+
+Maybe a big thing today is that I finally gave in and loaded $10 into my OpenAI account. I am officially in the belly of the beast haha. I finally decided to use the almighty GPT API. That being said, I will use the cheapest one gpt-3.5-turbo and be conservative with my prompts whenever I can. 
+
+![image](https://github.com/user-attachments/assets/747d858b-b932-462e-81db-448a635a8311)
+
+As for the LLM zoomcamp material from Module 1 which serves as an intro to RAG ~
+
+![image](https://github.com/user-attachments/assets/ac334a0d-2dd5-4395-993d-4a1d7a151d6b)
+
+This is a nice simple graphic done by the instructor and it explains the basics pretty well. I have already had plenty of experience with RAG apps but still I wanted to give this camp a go as I know I will learn something new. And even today I did - mainly GPT api and the Elasticsearch library for a search database. 
+Here is a bit of the material from today. Everything is on this [repo](https://github.com/divakaivan/llm-camp/tree/main/Module%201). 
+
+The built RAG app is based on DataTalksClub's Q&A documents where they have gathered tons and tons of questions so that students can reference and ask the doc itself, rather than the same questions appearing in Slack all the time. 
+
+First, load documents (and you can see an example)
+```python
+{'text': "The purpose of this document is to capture frequently asked technical questions\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\nSubscribe to course public Google Calendar (it works from Desktop only).\nRegister before the course starts using this link.\nJoin the course Telegram channel with announcements.\nDon’t forget to register in DataTalks.Club's Slack and join the channel.",
+
+ 'section': 'General course-related questions',
+
+ 'question': 'Course - When will the course start?',
+
+ 'course': 'data-engineering-zoomcamp'}
+```
+Next, create a search index
+
+We can run elastic search using docker. 
+
+And to create an index we need the following format
+
+![image](https://github.com/user-attachments/assets/84ca98c3-38e0-44ad-810a-dee129ff0992)
+
+Then adding documents to the index using (the actual doc count is ~985 so the for loop is fine in this case)
+
+```python 
+for doc in tqdm(documents):
+    es_client.index(index=index_name, document=doc)
+```
+
+Next, querying elastic search
+
+We need the below format
+
+![image](https://github.com/user-attachments/assets/d4198fcc-cdda-4519-9785-6a27c8199e2d)
+
+Finally, sending the context + query to GPT
+
+![image](https://github.com/user-attachments/assets/174d993c-45be-4d6a-9046-1efa58b2b539)
+
+Query: 'I just discovered the course. Can I still join it?' (Answer should be yes)
+
+Returned result: 'Based on the information provided, yes, you can still join the course even after the start date. Registration is not necessary, and you are still eligible to submit homework. Just be aware of deadlines for final projects and work at your own pace.'
+Nothing *that* new, but nice to discover a new library in Elasticsearch and start using the infamous GPT API.
+
+As for the [homework](https://github.com/DataTalksClub/llm-zoomcamp/blob/main/cohorts/2024/01-intro/homework.md), it was quite similar to the above. [Here](https://github.com/divakaivan/llm-camp/blob/main/Module%201/Homework%201.ipynb) is my submission.
+
+Also, all of this made me think for when I finish this 365 day learning journey. I should create an app like that, that lets people chat and more easily learn about my journey. 
+
+### LLM camp module 2: open-source LLMs
+
+Here the instructor introduced us to free alternatives to OpenAI, like huggingface and ollama.
+
+I learned how to setup ollama locally using docker. 
+
+Firstly, a docker compose file was needed to run both elastic search and ollama
+
+![image](https://github.com/user-attachments/assets/a134f5fd-71ae-4692-989a-a98226a3cf99)
+
+Then, we used phi3, and because we need it locally downloaded, I went into docker and downloaded it
+
+![image](https://github.com/user-attachments/assets/626492f1-e055-4173-b6fd-5caffcde5915)
+
+And then, I just modified the final notebook from Module 1 just so it uses ollama, and the output is surprisingly good. 
+
+Query: 'I just disovered the course. Can I still join it?'
+Result: ' Yes, you can still join the course even after the start date. We will keep all materials available for you to follow at your own pace after it finishes. You are also eligible to submit homeworks and prepare for future cohorts if needed.\n\nHowever, please note that there will be deadlines for submitting final projects. It is recommended not to leave everything for the last minute.'
+
+[Here](https://github.com/divakaivan/llm-camp/blob/main/Module%202/rag-intro-with-ollama.ipynb) is the full code.
+
+The final bit of content of module 2 is creating a basic streamlit [FAQ bot](https://github.com/divakaivan/llm-camp/blob/main/Module%202/faq.py) using ollama and elastic search. 
+
+![image](https://github.com/user-attachments/assets/7e3c1f88-5735-4f9c-8e2a-9329e010818d)
+
+Very nice so far. Can't wait for the next topics which will come in the next 2 weeks~
+
+### Data engineering camp Module 3: data warehouses
+
+Firstly, I learned about OLAP vs OLTP
+
+![image](https://github.com/user-attachments/assets/f771ee13-4362-45de-97ef-32fbb9447c3e)
+![image](https://github.com/user-attachments/assets/7a522b49-8e1e-414a-b041-82a40fe46bed)
+
+What is a data warehouse?
+
+it is an OLAP solution, that is used for reporting and data analysis. They might have many data sources that all report to a staging area which is then written to a data warehouse. A warehouse can consist of metadata, raw data and summary data. Then it can be transformed into various data marts, and then users can access this data. 
+
+![image](https://github.com/user-attachments/assets/e80830ce-22c8-487c-ac25-57a2991c06dc)
+
+What is BigQuery?
+
+* a serverless data warehouse
+  * there are no servers to manage or database software to install
+* it includes scalable and highly available software and infrastructure
+* there are built-in features like ML, geospatial analysis, BI
+* it maximises flexibility by separating the compute engine that analyses our data from our storage
+
+Then a bit about partitioning in BigQuery
+
+![image](https://github.com/user-attachments/assets/29d4d2f0-f942-4439-b8f9-a1319a055ea1)
+
+And clustering
+
+![image](https://github.com/user-attachments/assets/a54827ec-fac1-4fc9-b5ba-fb9960f41b50)
+![image](https://github.com/user-attachments/assets/dd2d07b9-1092-4dc9-a4a5-d63a0cecd9ed)
+
+Choosing clustering over partitioning
+
+* partitioning results in a small amount of data per partition (apprx < 1GB)
+* partitioning results in a large number of partitions beyond the limits on partitioned tables
+* partitioning results in our mutation operations modifying the majority of partitions in the table frequently
+
+### Automatic Reclustering
+
+As data is added to a clustered table, the newly inserted data can be written to blocks that contain key ranges that overlap with the key ranges in previously written blocks. These overlapping keys weaken the sort property of the table. To maintain the performance characteristics of a clustered table, BigQuery performs automatic reclustering in the background to restore the sort property of the table.
+
+For partitioned tables, clustering is maintained for data within the scope of each partition.
+
+Best Practices in BigQuery
+
+Cost Reduction
+
+- Avoid `SELECT *`
+- Price your queries before running them
+- Use clustered or partitioned tables
+- Use streaming inserts with caution
+- Materialize query results in stages
+
+Query Performance
+
+- Filter on partitioned columns
+- Denormalize data
+- Use nested or repeated columns
+- Use external data sources appropriately
+- Reduce data before using a `JOIN`
+- Do not treat `WITH` clauses as prepared statements
+- Avoid oversharding tables
+- Avoid JavaScript user-defined functions
+- Use approximate aggregation functions
+- Order Last: For query operations, to maximize performance, optimize join patterns. As a best practice, place the table with the largest number of rows first, followed by the table with the fewest rows, and then place the remaining tables by decreasing size.
+
+That is all for today!
+
+See you tomorrow :)

From d759ed827c67fd20fe2823752e39a536e5b056fc Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:07:08 +0900
Subject: [PATCH 283/638] Update 2024-06-24-day-175.md

---
 _posts/2024-06-24-day-175.md | 234 ++++++++++++++++++++++++++++++++++-
 1 file changed, 232 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-24-day-175.md b/_posts/2024-06-24-day-175.md
index cadf548b6..161161694 100644
--- a/_posts/2024-06-24-day-175.md
+++ b/_posts/2024-06-24-day-175.md
@@ -1,5 +1,235 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 175) Learning about and using dbt cloud
+categories: [data-eng,cloud,applying-knowledge]
 ---
+
+## Hello :) Today is Day 175!
+A quick summary of today:
+* basic SQL in BigQuery
+* continued [Module 4](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/04-analytics-engineering): analytics engineering and dbt from the data eng camp and learned about dbt
+
+There were some bits from Module 3 that I did not finish. About using BigQuery to create a partitioned and clustered dataset and see the impact it has on processed data.
+
+Using the infamous nyc taxi dataset, I executed some simple queries on creating external datasets in BigQuery and could see the effect of partitioning and clustering
+
+First create a non-partitioned and partitioned table in BigQuery
+
+![image](https://github.com/user-attachments/assets/22f2f4a9-4b3b-4eee-92aa-6d87eb61cf69)
+
+The dataset is by dates so we are using one of the datetime columns to partition by it. Below we can see a significant drop in processed data when doing a simple WHERE between dates on the non-partitioned vs partitioned tables.
+
+I faced an issue here because when creating the partitioned data, in BigQuery there is a table details section where it says 0 partitions. And I was really confused because the data I was partitioning is 10GB and I saw that for other people it says there are partitions there. But for me it says 0. I kept rerunning and changing/uploading more data.
+
+![image](https://github.com/user-attachments/assets/58f428e7-d861-4849-b8a0-20629b0a43ed)
+
+But it turns out that even though it says 0 partitions in the Number of partitions part, my dataset was indeed partitioned and the results were clear (as in the pic)
+
+![image](https://github.com/user-attachments/assets/4f4b4e4e-ea53-424f-b176-008cc519aaca)
+
+And finally a table that is both partitoned and clustered, and there is a decreased in processed MB of data in the two same queries as well.
+
+![image](https://github.com/user-attachments/assets/28d215a0-f780-478d-963d-d44d236e81e7)
+
+### Now onto Module 4
+
+First I learned about ETL vs ELT
+
+![image](https://github.com/user-attachments/assets/66773554-e3f8-4964-8de1-7aad450f1ca2)
+
+Extract-Transform-Load:
+* slightly more stable and compliant data analysis
+* higher storage and compute costs
+
+Extract-Load-Transform
+* faster and more flexible data analysis
+* lower cost and lower maintenance
+
+Then - Kimball's dimensional modelling
+
+Objective
+
+* deliver data understandable to the business users
+* deliver fast query performance
+
+Approach
+* prioritise user understandability and query performance over non redundant data (3NF)
+
+Other approaches:
+* Bill Inmon
+* Data vault
+
+Elements of Dimensional Modelling
+
+Facts tables
+* measurements, metrics or facts
+* corresponds to a business process
+* 'verbs'
+
+Dimensions tables
+* corresponds to a business entity
+* provides context to a business process
+* 'nouns'
+
+Architecture of Dimensional Modelling
+
+Stage area
+* contains raw data
+* not meant to be exposed to everyone
+
+Processing area
+* from raw data to data models
+* focuses on efficiency
+* ensuring standards
+
+Presentation area
+* final presentation of the data
+* exposure to business stakeholders
+
+**What is dbt?**
+
+![image](https://github.com/user-attachments/assets/408e3001-48bc-449d-8c32-b46770fe1202)
+
+It is a transformation workflow that allows anyone that knows SQL to deploy analytics code following software engineering best practices like modularity, portability, CI/CD, and documentation.
+
+Workflow we will use:
+
+![image](https://github.com/user-attachments/assets/b42e9ebd-ead2-4d20-8ddf-ec65c31fc30d)
+
+Creating a new dbt project
+
+This took some time to figure out creating right folders and access to my repo, but in the end:
+
+![image](https://github.com/user-attachments/assets/a41361f8-2a4b-410a-9502-a8b211e07ab6)
+
+When creating a new project there are sample dbt data models (with its lineage)
+
+![image](https://github.com/user-attachments/assets/2b4c24c9-a249-4c9f-8fb7-f85c254853dd)
+
+Anatomy of a dbt model
+
+![image](https://github.com/user-attachments/assets/5908db88-13af-4bd9-8bee-f559b4d3803c)
+
+Materialisations in dbt cloud
+
+* Ephermal - temporary and exist only for a duration of a single dbt run
+* View - virtual tables created by dbt that can be queried like regular tables
+* Table - physical representations of data that are created and stored in the database
+* Incremental - powerful feature of dbt that allow for efficient updates to existing tables, reducing the need for full data refreshes
+
+In dbt we will select FROM two types
+
+Sources
+* the data loaded to our datawarehouse that we use as sources for our models
+* configuration defined in the yml files in the models folder
+* used with the source macro that will resolve the name to the right schema, plus build the dependencies automatically
+source freshness can be defined and tested
+
+Seeds
+* CSV files stored in our repo under the seed folder
+* benefits of version controlling
+* equivalent to a copy command
+* recommended for data that does not change frequently (and is not huge)
+* runs with `dbt seed -s file_name`
+
+Ref
+* macro to reference the underlying tables and views that were building the data warehouse
+* run the same code in any environment, it will resolve the correct schema for us
+* dependencies are built automatically
+
+Now onto creating my 1st model
+
+In the models folder, create a new folder and in it a .yml file
+
+![image](https://github.com/user-attachments/assets/166efaf6-6be9-4ed8-b454-967ccb3d2b9b)
+
+The names in this yml file come from my BigQuery dataset
+
+![image](https://github.com/user-attachments/assets/b7021926-469c-4f77-83d8-1dcaf3fa36c9)
+
+By clicking on 'Generate model' above the -name: green_tripdata, we get
+
+![image](https://github.com/user-attachments/assets/a8c0fce7-9294-4c4d-b4d0-01390c43acbc)
+
+To run this, we use `dbt build` and dbt executes the models in order of the folders. Then, the below is seen: (ignoring the failed test)
+
+![image](https://github.com/user-attachments/assets/41e6131c-d278-476b-845c-fa4536281749)
+
+Next, I learned about Macros
+* use control structures (i.e. if statements and for loops) in SQL
+* use env vars in our dbt project for production deployments
+* operate on the results of one query to generate another query
+* abstract snippets of SQL into reusable macros - these are analogous to functions in most programming languages
+
+Creating a macro (reminds me of excel VBA in high school with the macro syntax)
+
+![image](https://github.com/user-attachments/assets/e9ddcd8f-2986-4613-8191-7339405ba3cd)
+
+Then we can use it as a normal function in our models
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiMjQ7fRvGC8ca8dWx2FfIKJ4RnjWVJAzS5FRwJIsn-nzCOyzyc2p-KSINnc2jyyUYJP0weJQ4edmBFpXqDB9dl_Qk-gsf7MIXy6xwdhseasfZPq7Dg__gEMo5AuV8zd6_KZGUBZ8DcFfHQe180B3WqJUzLLZJZkkoeDGr5Ql0VI158QsG3vnw9pVhkBL_E)
+
+Then, I learned about packages
+* like libraries in other programming languages
+* standalone dbt projects, with models and macros that tackle a specific problem area
+* by adding a package to a project, the package's models and macros will become part of that project
+* imported in the packages.yml file and imported by running `dbt deps`
+
+We can add packages to packages.yml file which lives on the same level as our dbt_project.yml
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhu0gGvXpM6AseZN0TGhGEu4fWh7dlrb_HQ5Wy0oHJ4Gnzm8e15mb0X1Ai86bm8FOFp7u-xG9QZ6Afi__r_-PjW3GFH5I9WGvMawOne9ZVEmIrX8b6SjS9OVXtUZMcsbHbrTrG3iJwcqFA5Quvq6nHLZjLLZdqQLZHB4uFqwF53gTT7aZAilDFRzHvzYljx)
+
+Then run with `dbt deps`
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjOhwh-xo0ZWMb5kwAyedo9hnidbRIiiEWAK61ZC18h8EM9Um_mOf5RMQ6vQlyUdmg7h9XWkTrhk_o7oCXy5VTzJfAQBW5KL9A1bwsczO4ZV4Gsbkp7ajADvefPyM9ZQ45rJDTFJnCwPLjkPI4nSezH6oE-bR9tDQYRIs0Kg2ZWvJ4WdR74UqDa5uqiPhL1)
+
+Then we can use functions from these packages in our data models.
+
+Once we run models, we can also see their results in BigQuery as well:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg3JAG-9WEupOQ9vVChbksXoVSu5PcCnZ_OQaAEZsB7sy0H8kVPJHER9wdHWQYNMSCmj4QdG8mzbmbYoo0zeoqrDfc32lAyocw6hqbAVtTWFOjrRsvgzMAEqe1GJvhKMwNDFwx0BQf4x5HShyKiqlViDjEhgyGlWhHwKfuUZ5J7IbpVNv2EAzm4dnSiOuYt)
+
+Then, I learned about variables
+* useful for defining values that should be used across the project
+* with a macro, dbt allows us to provide data to models for compilation
+* to use a variable, we use the {{ var('...') }} function
+* variables can be defined in two ways:
+  * in the dbt_project.yml, or
+  * in the command line
+
+Example:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEje8YkZujzWgPsKv5WrmEm6xvOch8c1xY-5TAeOy0zxkx0guSlLxkm7t5aVkOJch4NsGEc8HqP-m_7mlrPGIHt5ZyQ_fbMClRsvJ_8MYZxOE5wtqIivY4zo7ET3yLWakPEdcKHtDtEI869PuSgyAxPjKytff6_wUgGm9IsPces6hQbmqs742uhuHhgMiGAJ)
+
+When we do dbt build this_model and we provide the var is_test_run, there will be a limit 100 added. 
+
+Next, after adding a model for the yellow_tripdata, the project's data lineage looks like:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhsARm9rnaJrmm2FUNK36tGkbWxML65KZDZycFo2udoMbc3Emndo0kQMe2C_Pa2pXOkLwGM5_5uWmssOre9sK_4ZhCeKgtP4Lf1ELkkjpdtXHESS-cXADkklvXzf0j0bSBxkhhfXtuydkdyuCZObgsbcXwKGy2d6A727il0YkiwqtybxpdlbBe9NLkTbKZe)
+
+After adding a seed data, and creating a model for it as well, we can combine all and get something that looks a bit more modular
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjjUD83UOvNF3C7yotQZhkYmkxcDp5X9NMjwPxOc57xTO93xUrur6KKT4Ef9OCKTZx80twB5t8oXluFC5Bvw694ilkjZejBTMvIEHlspFBLge6n-Mfe3ZmpY3KDWmp7017E_8mo8PxInqap7gPo9iA3xQdLz0LhJomzBQoxwMOMF1xBQFydsqa9vyLNU8e8)
+
+Here is an example error that shows some details
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhORLFCV5Qpww_WYaBHU0HDWmdaZxOLdL_sWmBCITnhhRl6b0JQ7FifYKHp3A4erF6729w_9B0nDWC4z29FDzMyfGR8g6hMUXB6Ykm48O7a51qKkdVtiOURiBBbUlCCNXrrrh_CSd82pviTdB2SPTE5wnRLyJiu7jizr7KWat1pyeVCFrCUv4Gk7Dgz8_FA)
+
+After fixing ~ Nice
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiKGKdyiEQ5a6h0NceFGyr9_vdGOsUWmVF6Uo7m-PPhuVIgnYn2ncWJS6CkOB8UOcMLE1rYXhY19jgZD-tHim75xk4nK9Ccz7gu77ut_Hyh4Y4DRee06SGox4GFjDVR8MZspTuYPxB5awr9vAg3wDwK2EkNFxq1NaMmSJeCnj11VPGh_vpCl9b8EtsUCvHk)
+
+I am finishing up now ~ and this is how the folder system looks
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjDv7-fCSKZHzIxxgAB_Wfk24ZKl8X0kmrzMMYcVcctj8VfxZ5L1jXgmUGyzU8QW8dJSAfpA-W07fqE58sW8ulIlUnMIdcDio5eZ2leHQqfjq5roUwYBUjuf4KBWDncOvCPXNM_OAlxhRn8abEqf9rcanF_6rr6iO4uT0x1KL50TARyvYDlwrXpYbFG_ptn)
+
+and I can push directly to my github, which is awesome. 
+Everything is on my [repo](https://github.com/divakaivan/data-eng-camp/tree/main/Module_4).
+
+This is not all for Module 4, tomorrow hopefully I will cover the rest ^^
+
+
+
+That is all for today!
+
+See you tomorrow :)

From f740f7cf0b696ae4a0f3d53ea6998d2115d3d03c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:17:48 +0900
Subject: [PATCH 284/638] Update 2024-06-25-day-176.md

---
 _posts/2024-06-25-day-176.md | 181 ++++++++++++++++++++++++++++++++++-
 1 file changed, 179 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-25-day-176.md b/_posts/2024-06-25-day-176.md
index cadf548b6..6106d281b 100644
--- a/_posts/2024-06-25-day-176.md
+++ b/_posts/2024-06-25-day-176.md
@@ -1,5 +1,182 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 176) Testing, Documentation, Deployment with dbt and visualisations with Looker
+categories: [cloud,data-eng,applying-knowledge]
 ---
+
+## Hello :) Today is Day 176!
+A quick summary of today:
+* finished [Module 4](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/04-analytics-engineering): analysis engineering and using dbt from the data engineering zoomcamp
+
+A preview of what I created in the end
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg-EklHUAERxDrqjoIea4GZHIjuzO15JEHxT2-obQe62p3UJoN1n1A2q_Jb78-WqvRbscXU2SdJAEgGrc7aqEaHZb4b6O5tFTr4hHxf4bAeXRaT5_AVjxOqQ_lYaDpp6-kudeDP4bhzvSNuz6QEI-aVm3yl_rycN79absognjQl4Vz-kAeBARV2JIRf_PKt)
+
+Continuing from yesterday with dbt ~
+
+First I learned about testing and documenting dbt models
+
+We need to make sure the data we deliver to the end user is correct, and how do we make sure that we are not building our dbt models on top of incorrect data?
+
+dbt tests
+
+* assumptions that we make about our data
+* tests in dbt are essentially a select sql query
+* these assumptions get compliled to sql that returns the amount of failing records
+* tests are defined on a column in the .yml file
+* dbt provides basic tests to check if the column values are:
+  * unique, not null, accepted values, a foreign key to another table
+* we can create custom tests as queries
+
+Before writing tests, to ensure our data's schema is correct we can autogenerate it using a package
+
+First, include the package in packages.yml (and run dbt deps to install it)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEje0mDWA2F4gCMosBHYl3YKrwMRkWwtxTZkYZuBLRvAh6g3QwDRQYy7cA9CVPVwQfNy1pCBTYlO2z51LG5o2J_1xMFAwPjju5XDVc_PmoPdJnGHHUCiAvnHNiE3ZUEkMF5quEb7M3TIONl1QoEIU0BHhnbeuD93G7vfpcDU7TIRlfeoorHkxpZLXMI0W5Yb)
+
+Then, we can use a macro the auto-generate schemas for our data
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEisqg1LUYxJhgfT4FZCX0xf3w1ylV-E0cvXXyl0QD9VVzZB_4WgW4Kv_r1hPIJ3PaBaUcPeozYsYlDnhMzPNUQHnxKk6nMMONGzj863EvMNzikm-Q8AvygL9VLKsO8dFh47LlWK10jxLeNCIGyToqoSs4KHu_JMjRyMG6hTnS5eZViypcZfHBXH3jvPUYiU)
+
+And when we compile the above code, we get (image is truncated)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgkgoaUelmz6xSuBOy3Dzg8azztMAlpMnqKnHQZRU3MTkirOwzrXZ0HTwBbyMWb2DZVzknEn4Dz-PQZ2ubSwmsdjfWmR-vgrk4BrLI_48ZIgPgnbFk1IgbNmIaX-LacajYAveS01bQay0N7Vnk60FuRr0BPpxpGtVZG4F0MCOW0UuvUfQ-DC9U5f_aDmqYm)
+
+Which we add to the schema.yml which keeps info about our data.
+These schema.yml files are not needed to run the models, but are recommended because it serves as documentation.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjZqifB36qoXIr-TTQIhNES2g1nkoUcEOSv7vL2rFQEn4cjjto1HHilWvg1CeKskN1lwlD_XHnwCxAjn0t4-Q2Vw8A3KhYzSs6OfYHAeo9U4AECabc32p1ol5R3-V76CgSCte8Hr9JGaoT33P71PBH0GD7eLl-Nd1ff6EPIAGUq-TcMVb9UBIwtkfGpsgsI)
+
+Now, adding tests is fairly straighforward
+
+For example, we can add tests to check that the column tripid is unique and not null
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgXHl6C8gZa0DkfZ0gXk7-8xIOrjAe4d1hdkncer0y1CTvbz5u34-DOgDq2YHKqG01BbEKmTCfm3Gp5N-PihJ0Az2iS7Aw8sBeNJHlskNHlE_nJHE3ptFbQWjsqQtupFW_jkBAkmYFsOtFtbiMf73oBqbYoadqZGgkKkuePp6_qlMZaqR-b6FffLx9j1U96)
+
+In addition to severity, we can add thresholds as well. 
+Another test to check that pickup_locationid and dropoff_locationid are within the existing locationids in a reference table
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgL2yGNYl8y1ggZG-D2cwYTPh_pM4e97A5ptnOikmiEpHhjqCY15Gg-elhJX_-23xzZirwZ_X4LUTfiwwDr9SNhfAFG0r3dU3NKsSeVxRHDA8WB8XLCjlNWrQjbPCoFlnb1RB9cvrrQOUovkz2C-s5gepb9Tzbj77drdNYfT1svb4rIgf0tqlmugAcXkFb_)
+
+Also added a test to check that payment_type is within certain values
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj5juKrIlf_-ySyGNaq0gJtLsbby2hqsRHr86fDnESCgmiuutaLBDLQSzYdJmqRzj1jJgxjeB0ftrVTUyZ9BcAfYMKdAPYAaXJOn1V7q9FAEWH9ZsIiLZXbzhspjy8v67pnjZ_CipVH_XdPykJ-uPWWwkFALuiz0_PM6sajU8u7iA9Hw92nRwISSdYszfwP)
+
+We use a variable here, which is defined in the dbt_project.yml and is a list of [1,2,3,4,5,6]. It is just an example of how we can use variables, because the same check needs to be done for green_tripdata (the above tests are for columns in the yellow_tripdata table). 
+
+And when we run dbt build now
+
+### Next, learning about documentation
+
+If we check the pics from above, there is a description: `""` field, which is part of dbt documentation.
+
+dbt provides a way to generate documentation for our dbt project and render it as a website. The documentation includes:
+
+- Information about the project
+- Model code
+- Model dependencies
+- Sources
+- Auto-generated DAG from the `ref` and `source` macros
+- Descriptions (from `.yml` file) and tests
+- Information about our data warehouse (`information_schema`)
+- Column names and data types
+- Table stats like size and rows
+
+dbt docs can also be hosted in dbt Cloud.
+
+Once we are happy with the descriptions in our schema.yml files
+
+for example:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg2q9CogcsW6ON6BwJbokX1ryLOJL72hB5Orn3gSsEWDLUZmZmAFtN6nso4mO5K18rt47TngyHNPRr1YIaKpYWGkV3vzLdvVybcPFmTqba8OpQwEiwn3gaAgw_uIH5ouFDcEW4fZl3LK-fniVmijSKYEkenHv9uNpYZD_cC57AjiPdC2CCPxcm9IliTd7o-)
+
+We can run `dbt docs generate`
+
+And we get a docs page
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg56MG2P7NCazuq5-ASGj_huGSPjjzRcveY3UPeg5ENrCkg_2OlSnPWukRyrxX4Gz3UE-RGsG6_xZ3pOxSZ9oYETgvZCq5y-8Jlkp_bnssx2lfkQlTppvam6UqRce95Lyxm8J8LyQlwUgsT16lyJ46yT251S-Cux32ywzqg33SZOEbh9dEPD268qmDzbJxB)
+
+The structure of folders is the same
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhk18NR1BWmJKP5PMh_y2tXp4MKGkgjLzMdklsxQmj57hNhK56s2yjn92PUtpd0As3WTOt5nl1L_frEFsFRiDp5YjC0rts7Zd6_YWMHf4ajfQwaEvL9ZV6PoHV4sutJJmXDnfRF-cbS71txA3GrjLdmCvJ3mTAoTrn-e0hXRcL3hhL1YCfKBcQSiUG4dbsr)
+
+And for example in dim_zones we can see
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhcPCfqAWAzbhLZyirUJtzcbBnltc7AivxH4F9F9Uxzd7EOgKxFI3JkB64FIuioaIGO-Nv_rcTD1y_m3xbenK97bymuR1L2VvBGeJf43Ni-rTbli3_yVpFmP9XY3UH7bedYZ14Pcj-ih_wa93mt-Ci00iEvO0DvayCiQhNG7MrrLcPkYr9zgs9Z3tOJEJzv)
+
+There is a database tab where we can look at documentation related to data in BigQuery
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiTHUIFgRdslVTs8mvQ6OJ8w0hnqRGUQVl3rYF7sbDr8TebmSMhfOHso1JOJr2pYsbZ84_zE1y60aMxRsx3Izdzk2YwMfeFWBdzot3Q3U5x_4II6RVy3M_zdhyaJB6lmZqqiKgOf3hS-uxHZW7aaV2gkrRx8lke4S06bar3fwQdDK_YTGHmXfb5MZaVEPNo)
+
+Also in the bottom right corner there is a button that shows us the lineage of our project
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEioEFWANFc2M00nPvZM8N32UEDeHWw8VsF4xY9niU-s6ieOIISLRHJsFa5lQSSJt6Y-xGtAvj-3rV9UkdZ78iVwlC2at2MDVRn-mrq7YqdOW10qNmqjDVCfHfZ5GpZa0YzNKEUtN-L1RvTjuf0P4ezd6IHnHLaqF6paJD-qz2Sm4j69idOQvCAvtJLLEdeJ)
+
+### Next, learning about deployment using dbt cloud
+
+To deploy we need to create a PROD environment
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgVIuMvERJHSi-YxdGaKn2-aTBqScUA0a0slCjPXPwmC28lWrLxD7pnQxmtWLjEhJKXqlPwp08eRlUEUiZCL-1Ymaiz1CeTaKCejZly9HeNdwLJT5qxkGCMDpz1MooBRhc0B13f4mlvZ901YJ8BEUdxT6gTi3FpHtbKUgPhUP92ramzLaHaEHYP9xDmQYM2)
+
+In there, we can create jobs
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEggtQg0rkauraPxRPhTjwqGf-VEMtzNEraQriVSbiVm3WWgMC1i6V-8wfDWdJiE9zHCp-TzZxRuOziaAYV68ZYnqg7wqKUB9GLfO20TtSxSJJov38sPMEZFegoP1UlIzXEPrlsj6nTqGoX3dxE_3aZ0a9982dbk71FuRl0QdZ0c59vD84QqmCx6Sg8Y36x9)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjS3jDvtqjBRlyOT-5MM8ziI2S_9_zND9j4G0HfrZhV_Vh3GvyL2-nMr-1kQseXGyr_I3ttqCKooMfiisF3vozZ4iJvq7QRPhwrWPM6IzeNyesbTcssejXy3sFBI9cZCvXz7BbzGaV5etWEHlELhi9yKFMg9vZh9ZLH--Ic3roIUeybvSgi4Q7bp8ihLnPr)
+
+Where we have a Job settings section
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgiHzBNYUMz-6nAHv6O1Wq6oaCmYomvfNT_ftGfrvfoNNX8fef0S6zqJfIuWC_C9fzC3zHUJPJqaT0dAD_1ibtfPFV3aEUE6Wv9E6gPlXDdtkr1a-mMp8mJes8f1AXVaFuLMMNcC55ojDmxAzdSfK6MFiDeB5YP17HVM-aY8F2CqlLPXxvvXs_SjrRrfPfX)
+
+Execution settings section
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhqDlhrzkErqnJTWI-A5Mk8VVmw_yORKVUB6OJOMuyPOLRR7yYbL3ng2vV3taSotQtmdceGambU9avu4l2py42thEOzimuVA7ozjVF3p_m_jv6oK0BNBNf305yaDCdQp7rDIpL4TuU_ay6TLldowAqifJlG7YtsEeA55fkuEY3Y2J0rxowiAuPktUNRhzQv)
+
+Triggers
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhJZD2jr4LX846-e79F9w25L-VWvWc4nz4y0ow-gTD2OlDaOfHN3woDGU2Unwq7xU8N6LXx2TKnPllVXAgdvEwimksFicA0AYHFVAmqTMJxtkP6Z7YBR44ymukr__H30vm_4XOU7RNf3-lhKQJ25qMCbsBAUrUA7I1nvL9AsJ1sQ-Gqv93cjafWHgWHJ8bM)
+
+And advanced settings section
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgZy-zJaSmBxWBUkJiFglWsprGwM5UYL9ABYV4h1hmr0imfXzP8nbWs4jawZiQD_hyr-ITNLM4qkCc50K82ShWPB9BtZ4Wwqd3rBZbCFe71s0Xm8goZGKn6ns6l_mhfED_7JUtX4OoaFRNyvP8syEsltUD3ICIZEIOi7cfuqBKJvncBbKB_AduzE7pA_Rh9)
+
+Once created, we see: (there is a next run because I set it to run automatically every friday midday UTC)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhM6SKmDvRvELdAW-ldL_74kVoFt6L_CW597w9WNOKhzz3uxnAg2MyT8qKK49kCoYlF3NK57ElminwXpNnA1uon-v73QDkeSFi5j666leSSbnrLx8etSvAFnGvtCfhOTrWenmp4yzEsdBgdiOCxaeYslRJOp5tq_1lbMNKTAwOaispZ-ARIVPe02d8yM0hm)
+
+And we can trigger a run via API as well.
+I pressed Run now and I see a new run:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgTuwQEr83WgDOxt4N7QsDqkyLuy56BPtK4N7VAYONq9D37XNsIyed3RK3A3IxmVI7_B1_mnQASBuDkNrcqkFRMM1an81vqqwMPdI-DrayDrDRxKTzDoDn_utCWNZb96zqidQxb5FTss_ux2BJQenwfXdQXfWtIN1rhabI7VKzVMiCLv3uyR3TChXqS1fjN)
+
+We can see details by clicking on it (the warning is about a deprecated functionality)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjmfhK7a2TQgJWxewnnjNKE-tBT6StlCAcTgkYvHctwmU1aIF_s4YkcfhPsWTFN7-jU0Ts2yCGGnFPZs_7U7V1vIGvRJjnQpcCgqSiO99Hwo79lp1c8692sWpzqr2Zr_83C9u3VeW1XxRLtEd54VKfoijusi9AQpiyrJHcKhzjxkJjTy0OdRNDl57yCiSlp)
+
+This also generates a hosted documentation.
+
+Going back to creating jobs ~ we can also create CI jobs
+There is a git trigger on pull request
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhfwSXIkZTzhRxCl6B8MKilAY881e7fj2Lj6a6_nuHPskmnZjozNJkGrNxVg9FUDT3jHpcQ7c_PUpEQPXKTsVJ6X_m2uATBR52wz3_Ua8OjLtJdbrdcGtDNMU6YGEp8L7ybWbzIQ-x_ck2CUKHNih5U92R5rJkH5yvZhAqTccD8bnlMkstoJLKxxU8agR2m)
+
+There is an option to list commands where we can run whatever we want. And also a field to select with which env to compare the run
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg1VgRJ1yCR_XxJpoSNAIwwn6BllaXc72_qIJiX9_HBrrIT2YlLXxClR6sp0r8VyYrTVQJaoyoTOAnR4dyI2bkI7yOU1l_q_A9pMjlzcaa6vUEfqW4mFtTA52uMzF-9ya8ho9xsfrudX55bEnVqvUAIcRjAVYhYfKjWEx6F7FKnww512Wd7TG6ArapfQ4mu)
+
+### A final task - visualising data with Google Data Studio
+
+Create a new report, make a connection to BigQuery and we can see the fields of the selected table
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg9gBjrnmwvE9b6nU3zLowOjbfV-DVY6oTb98BK8Dp3pWQOJeAhbb2bPi0810wDB1JBHrPLWjhV48xRj85b3QAMSzzns_do61BK-2jboatik2HIK4S0XAAroI3u06odJm5RQ5W0ZbyVAyveaKQ0UaSf3b8Wx0iXM7pzb3KrTPhvxl6kkgvYISbUC3qaQIl1)
+
+After some playing around on Looker ~
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhsq1dI6CF7xj5HX7XKX2KU9Ls3RwSLkEfUglg1M98NabPAne1cUZIkVEULE6NV7v2P-m1HHVnMxnE1GDF2mcnLlPJ2kJuOG0vaO70dFZlIrx2cZC-4Ve1jPu0z5HFfHusYe3qN59aY3cJ7QNX75N3v6i2ruW-vdHJSvsh1DgE-W58E14u9HtNceCXjm_Yo)
+
+All the code from today as in dbt + a pdf of the visualisation is on my [repo](https://github.com/divakaivan/data-eng-camp/tree/main/Module_4).
+
+
+That is all for today!
+
+See you tomorrow :)

From 30cd5256d823290bd6961cb3b812407cbf3c7f5a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:22:53 +0900
Subject: [PATCH 285/638] Update 2024-06-26-day-177.md

---
 _posts/2024-06-26-day-177.md | 117 ++++++++++++++++++++++++++++++++++-
 1 file changed, 115 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-26-day-177.md b/_posts/2024-06-26-day-177.md
index cadf548b6..6b4f9e321 100644
--- a/_posts/2024-06-26-day-177.md
+++ b/_posts/2024-06-26-day-177.md
@@ -1,5 +1,118 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 177) Spark for batch processing
+categories: [data-eng,applying-knowlege]
 ---
+
+## Hello :) Today is Day 177!
+A quick summary of today:
+* started Module 5: batch processing from the data eng zoomcamp
+  * Spark
+  * Spark operations
+  * Connecting PySpark to GCS
+
+What is batch processing of data?
+
+* method of executing data processing tasks on a large volume of collected data all at once, rather than in real-time. It is often done at scheduled times (i.e. hourly, daily, weekly, x times per hour or minutes) or when sufficient data is accumulated
+
+Technologies used for batch processing:
+* python scripts
+* SQL
+* Spark
+* flink
+
+Advantages of batch jobs:
+* easy to manage
+* retry
+* scale
+
+Disadvantages:
+* delay in getting fresh data
+
+Spark is the most popular batch processing tool, and its variation in PySpark is popular. I have used PySpark during my placement at Lloyds Banking Group back in 2019-2020 so there was not much new info~
+The most important bit is that spark works with clusters and in order to utilise spark's power in handling large datasets, we need to partition our data. 
+
+Then, I got a refresher on spark cluster (image is from google)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhLTHYD2G99xONlBb8aXH4roaE5qoI2OXI94KduHsJVOdpZU58PZ56VBYWq-C2qnQtqxR_O2ZjFrJKbz177v8BdMWmAlqWFT4kUQ7xcR9ED322IshUuS4GhQyjOmeamSkTUU7d_I5S4DQUkK4K1JGBlwr81uG9CHR6c_LHD8ER5hFeyX4E7EfZjNhQgvhCZ)
+
+We are the guest where we connect to a SparkContext and execute functions. In the spark cluster there is a cluster master that manages the workload among workers. In turn, each worker node runs multiple executors (individual tasks) and store partitions of data.
+
+Spark groupBy (thank you GPT for the nice summary)
+Stage Creation:
+* Spark divides the computation into stages based on the shuffle boundaries.
+* For a group by operation, data often needs to be shuffled so that records with the same key end up on the same partition
+
+Shuffling:
+* Spark performs a shuffle operation to redistribute data across the cluster.
+* Each partition will send its records to the appropriate partition based on the hash of the group by key.
+* This step is expensive because it involves disk I/O and network I/O
+
+Aggregation:
+* After the shuffle, each node performs a local aggregation on its partition of the data.
+* Spark uses combiners to perform partial aggregation before shuffling to reduce the amount of data transferred
+
+Result Collection:
+* The results from each node are combined to form the final output
+
+Logical Plan:
+
+Spark join
+
+Spark creates a logical plan based on the join operation specified. This plan is then optimized.
+
+Physical Plan:
+
+* The logical plan is converted into a physical plan that specifies how the join should be executed (e.g., broadcast join, sort-merge join, etc.)
+
+Execution Strategy:
+
+* Broadcast Join: If one of the DataFrames is small enough, it can be broadcasted to all worker nodes to avoid shuffling
+* Sort-Merge Join: Both DataFrames are sorted by the join key and then merged. This requires shuffling both DataFrames
+* Shuffle Hash Join: DataFrames are shuffled based on the join key, and a hash table is built for the join
+
+Shuffling:
+* Depending on the join type, Spark may shuffle the data across nodes to ensure that matching keys from both DataFrames are on the same partition
+
+Joining:
+* The actual join operation is performed on the partitions, and results are combined
+
+Then using a download bash script, I downloaded some taxi data as .csv.gz and then using spark, uploaded it to GCS. 
+
+Bash script to download the data (ran it for green and yellow taxi for both 2020 and 2021):
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiBgqmPxQn45hMiDpdeaheiab7z2dP5_paGpfsW-235gz2V1Xmf3BSNvE5OgyNVG4x39ZvDic6CHtF-azWYn4EnjSsuTfaUxh8Z20b61fuknYvwqJGhaHgd8sNKJHzfDfYB8qycyma5eQHVxWG5yZYdQbSazG9TKcit90Mvr7wpBTrSmgOjx1yi36sWwe7O)
+
+Code to transform to parquet
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiDw8vsI-UQTYlie0RdtEuIoN0o6BhHKlznwl8Zz4wqvduUuAglGWZptJZdFt1YXjybNe-vb1-yCyuk2CzSMdHZUS2D_M1okXZZCSwfTZWvQj54ITldblde9k8ZuEqNhAb6rqMB6upQZNQ0aUhO0QfQkwn6815w5Nw75kvoHZqDV73Mlq7njO1eijRw26w5)
+
+I installed the gcloud cli and uploaded the data through it ~ here is the data in GCS (in the pq folder)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhbse8Q4R3TFmvNKMURwa4o1WwidGlYrfn6WRew4VoyieD6ci7kwaz-LNJCYNcVlvzY8hhtbWh1kjuwnXI3Ua_s5EhsxdIStRnqKgohvW8rAMFPEEcwpok62ypiI4DE9ndmWu3QZzJQYcwmCNovOW0W0U3s6S6aeRKU4XUWmBu7iAOX2xuPxzTNenEJpJpN)
+
+Next, to load data from gcs using pyspark, first I needed to install an extra google cloud util for hadoop so that it can read gs://mage-zoomcamp-ii/.... And after some debugging on spark not being able to connect to an IP ~
+
+Creating a context
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEj37DpzEbYGixYY_0VVlHp4UeXspMc-rzkPeB1xhJg6UBbuF7ceh-r0Jne1tbEInuFmw8xLUp-MkNTeRoDtbrSeTN1jY1ui9AAUhzDDQ6KoOp1JTDZYB2lemErhfNUIiZiC6FSX3YG3atIdPsuUIpedVYtjb0PAbWZpVHIUQb8QE2O_UY5awClUhvG3hyf1)
+
+And connecting and getting the data ~
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEit_9l8XUl3NnAuIKk2eEaoC3Wa46HOkx32LVrPeNJMBucgiF-NRQJZWNUlKIrYQ3s1p-HYkRlr6oN1Ztoo1Dhh6DGDumvt1Ny83c6ej1X-odbl0Dmb2fh24cPBY1bH9_4fGHl_Q3Qpi0KuC76AX2TPcIswgnxkGhErnlP3pN1rQpzPATc2a8crwwVWswPS)
+
+The spark debugging took me a lot of time ~ it had to do with the default ip (127.0.0.1) I had set in spark-defaults.conf for spark to connect to, and because I was trying to use GCP, it could not connect so I removed the IP, and now it is just (pic) and it works. 
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhQjTsNyXXOS1UD0h0WpMh1AEdpIIvKmSD46axT4CcczE1vjT1euFcDt8p-EQ5bq7wvkv5s8tqvmpB4ZLWj47_Idb1t--6sP3ZJCCIqgSCzXqYAL30V2kJjHpJmVyD5rDh8nwOQr0geG7nzIqOCWOrXigQiB3sr503TPRAld0mPsoQeqkJXt4Z6F7wthMq2)
+
+This is not all from this module, there is a bit left, but I will leave that for tomorrow. I uploaded the code I used today to my [repo](https://github.com/divakaivan/data-eng-camp/tree/main/Module%205). 
+
+
+
+Also, a small update from the lab ~ I wrote the first version of my (at the moment titled) 'Previous works' section. ^^
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From 8a057e720cdef81e6054a9d999c03e91813b9519 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:26:47 +0900
Subject: [PATCH 286/638] Update 2024-06-27-day-178.md

---
 _posts/2024-06-27-day-178.md | 71 +++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-27-day-178.md b/_posts/2024-06-27-day-178.md
index cadf548b6..b90741c78 100644
--- a/_posts/2024-06-27-day-178.md
+++ b/_posts/2024-06-27-day-178.md
@@ -1,5 +1,72 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 178) Starting 'Lending club data engineering project'
+categories: [applying-knowledge,cloud,data-eng]
 ---
+
+## Hello :) Today is Day 178!
+A quick summary of today:
+* started my own data engineering project
+  * GCP as cloud storage solution
+  * terraform for infrastructure management
+  * mage for orchestration
+  * dbt for modelling
+  * maybe more to come
+
+The last bit of Module 4 was about creating a spark cluster in GCP - using Dataproc
+
+We create a cluster, which creates a VM for it. 
+
+On that cluster we can add PySpark (or other types like Spark) jobs where we can submit python scripts (that use PySpark) that we wrote. And we can run them on the cloud
+
+![image](https://github.com/user-attachments/assets/7178246d-2267-46ab-ba29-f4e81cbb8dd0)
+
+And because this cluster is created on GCP, it can directly read data from GCS and also write data to it. 
+
+
+### Now ... onto the Lending club data pipeline project
+
+It is still early stages but, I had an idea in mind how to combine the different tools I learned from the DataTalksClub data engineering camp. I am taking inspiration from the data engineering zoomcamp where full documentation is encouraged so I will try to create nice graphs, visualisations and explanations when the project is near completion. Also I decided to use some structured commit messages - [feature], [pretty] for renamings and similar, [delete] for files, maybe more will be needed.  
+
+What data to use?
+
+I have had the lending club dataset in my mind for a while. But I wanted to get a smaller version so it is not so heavy on my machine for dev, but also to keep on GCS. So I ended up with [this](https://www.kaggle.com/datasets/gabrielsantello/lending-club-loan-preprocessed-dataset) smaller version from Kaggle.
+
+First, I started off with terraform and creating resources on GCP
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgb-xYCmakjlxSVgvt-rU46eov4D_eJB947kwQn1iAy_atq2lnFS4D1jQsEw7dR-H7_AtHENLVIIGMBmuTPAJxQNqMfiml7FSIcnIshDi3jRR375j3zEaC51Dxj9lLyJMpFTJzeASJkkzKC_pztwnZ8uZuYubmRA0Envb42X1WZm_dMOd1VqEhaTgUKdAvM)
+
+I have only two terraform files - main.tf and variables.tf - [here](https://github.com/divakaivan/lending-club-data-pipeline/tree/main/terraform) in my repo. As I am writing this before finishing for today, I realise that creating these resources is a bit manual - running terraform init/plan/apply - so I wonder how can I automate them. 
+
+Next, I initialised my good ol' pal mage.ai
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgGqXFr_6Ywo8ewDWVEhqs2Z_S6T9CUBEiGVvv5YUFzd0h0WS1FJqlS7p84twToGUCeKqvBWt_5JvLwIeLATPyBL7tqPL7O3Ce3bJIX2fk5Tdd8EI-fliV50j_FqvP5lXdChS2jaZrKj30-RQzfIQlj_9wLmMjOJvBzMXpsVgtvalWzxHqF5BXekpt8S2RK)
+
+The above pipeline (called 'data_raw_to_gcs') 
+1. first loads data using Kaggle's API. I had to set up my kaggle credentials for this too. 
+2. next it checks basic data types (str and float). I also wrote some basic tests like making sure we get the same columns. To be honest, maybe these two 'data loader' blocks can become one.
+3. the 3rd block does some basic transformation - like converting date columns to the right format
+4. the final block, connects to GCS and uploads the final raw file version as a parquet file
+
+All the block's code can be found [here](https://github.com/divakaivan/lending-club-data-pipeline/tree/main/mage/mage-orchestration). I spent a bit of time here, configuring connections, and thinking how to design this. I was thinking about the structure of my project. Is it better
+1. to do this splitting into dimension tables in dbt - so that most of the data transformations are kept in  place (dbt) + data tests, documentating, or
+2. should I split the big dataset into dimension tables in mage which are then loaded to GCS. And in dbt, I do data modelling - adding tests, documentation, creating fact table.
+
+I feel like the 1st one is better because all the transformations I am planning to do can be done using sql, and if I do them in python, dbt kind of loses a bit of its value. So I decided to go with 1 just so that I can get a bit more practice with data modelling in dbt, but also chatGPT seems to agree that by doing 1, I am separating the ELT process better.
+
+Then I created data in BigQuery using the uploaded file from GCS
+
+I did this manually, and I am realising it as I started to write this blog. The bit I did manually was bring data from GCS to BigQuery and then partitioning and clustering it. A mage pipeline might be a better fit for this task. 
+
+Then, I went to set up dbt cloud
+
+I managed to connect my repo to dbt cloud and initialise the project. Also added packages that I will use (dbt utils), and just basic cleaning operations. Code is [here](https://github.com/divakaivan/lending-club-data-pipeline/tree/main/dbt). Before I do anything, I need to figure out my data modelling architecture - fact and dimenstion tables. I learned about those concepts on Day 170, so I will refer to then when making my plan. 
+
+
+For tomorrow I will think about the data modelling structure, and automating more of the manual bits I did today.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From da5f3e62ef673d83d8d7633f3e70e68c9bf889a6 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:30:22 +0900
Subject: [PATCH 287/638] Update 2024-06-28-day-179.md

---
 _posts/2024-06-28-day-179.md | 61 ++++++++++++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-28-day-179.md b/_posts/2024-06-28-day-179.md
index cadf548b6..237a22942 100644
--- a/_posts/2024-06-28-day-179.md
+++ b/_posts/2024-06-28-day-179.md
@@ -1,5 +1,62 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 179) Using Docker, Makefile, and starting Data modelling for my Lending club project
+categories: [cloud,data-eng,applying-knowledge]
 ---
+
+## Hello :) Today is Day 179!
+A quick summary of today:
+* continued working on my Lending club data engineering project
+
+Today I added a few cool features, and I learned plenty
+
+1. Introduced docker to the project
+
+Here is the Dockerfile I created
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg9vQJ-whsilhhySXpxg_1seyyq0HG4ohP_NnZ8uPDaRPZTocZrsV81FHDiLep9K0bWPAy0QpkZwmY_B4z_U449FeJgqVCSR9YU8tNlL5wgUk9LcYNGM0Z65sHEFohyzdy8-Odcq0n1QLC9A1-Lrk2dxj4VneHNDMQIH3iXLlftDTn76sFYTWNL2sI_Sqh9)
+
+Today I learned more about the docker folder structure and where to copy what, and where things live. At first I was not sure where things go, in which directory should I point my env vars, and where should I copy files. But then I also found that in Docker desktop I can view the files in a running image, so that is how I figured out what and where.
+
+The bash script referenced is here:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiIh4C3cY6eXZf9BjqqWalEGGnOjMryzkN8ANqrVhwFhgi7i2rOKD8hlfQMMnmxlyKIjVOIIKyv5A43IzTVrCWOgwFJPcCeRk_n46M8Omj5xx02i88nhSPcINeS3FD9sqJDWUShdYOAw02A3W9Hf7GMqP53_wL9f9Kdth6JE0L9jPT-2Kcu-GWx4c4-In1J)
+
+And my docker-compose.yml (before I added the volumes, the code I was writing in mage was not persisting, so now I know what happens without volumes)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhe-jREHj9Kwbu-6BUEbzJdlEE1lQsU5TEGRzMEt8Rh18qiGeSDxJeYiEWkC_AyGfliO4S1FBRFlUIstPcJMzjUkA57sykdub6msVgldRj0Ai10b3yrshYGDWVgxq4Gl9G22GPjGG7NC8lCLqTO-dXJc7KQMg_-r2XpfAf2ULJ2tREkH4oZzN17I31f143b)
+
+2. Added a Makefile
+
+I also made a Makefile (using this for the first time). I saw that adding a Makefile is good in the data eng zoomcamp project advices. And is good for reproducability.
+
+These are the options that can be executed
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEijZQiLCvCspN48PSIj1ve9hh-4X42Qn7jX2M0NdhUxTSdMoNzk6yzlTVmG3mKgrzlEW9skQsJuE7PWqRNCZuHYySX-4WDtilfvxysasEe5y-NBh3cZj6XgID7803C3QZdtCUssbeazRZjoRmNoli59Ho0WKtflWFQc3dMB2e3aqnzK5IK-khooMxoM0Y2n)
+
+(this make 'interface' is looks so nice)
+
+3. Began thinking and designing my data dimension modelling strategy
+
+I created the below using lucidchart
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgmWw5aBJLTQQwyGyJekNLkVQ69l9lHhrBDPzMGPqxgq9HlTPEJVFWQc3DLaTeb3lsQ_N6Fb3eem4gJaqLeOJRzTNXj5OaVX5mD6ykxYfmk8GvwssxjwxjIudtIRfwqLMLZPEGU5Dcfe2OrC7W0B9LdwwbVD8tCqA-_oB5QpNMGGQm39aCRkIrL-LBvsME0)
+
+I do not have natural unique identifiers in my data, so I had to go with surrogate keys. Initially I had a loan dimension table as well, but I felt that I could not create a surrogate key using unique enough columns.  
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEjRVQE63hHEXMcpAe068I5HCC2Vd5cUVE3Gpqf94f_tZFCoQ4fERfNKG-07skYMBxXWft0oz6shg5Z0kn0iAtAGcAtfW1H-xzIhZNRkW3MOLGH5qLH1EZM0H3NCNoCS_ZRV7vUMXE6WlR42lw0Nakn8DVOUdc6yFeepCulKpUqa0vAY3kkRe4J1qqQcv2XX)
+
+At the moment the data lineage looks like the above. 
+I am adding data documentation:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEixULEbt0S7eMNDzA2zLRRa1GXYoAtFaMN0PXBnoRPSNKE5m0iC1H-bGYwedIHYORhihgIPKacPayfVX3jSsgL4-9qD_yT5igvkRwPOcqsbiynyfLqHYY2LmKQgG15NLWtIdXyMD-bhrJj8kEhqG90xYprEXjBVT5yHtu7Md5wLTkvUdNuPH94M0LcJFFu_)
+
+I have descriptions for other columns too. 
+
+I will look to add some tests as well. 
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From 92442fa89110ff4ebb5a1616502dcd6299a67c70 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:35:33 +0900
Subject: [PATCH 288/638] Update 2024-06-29-day-180.md

---
 _posts/2024-06-29-day-180.md | 78 +++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-29-day-180.md b/_posts/2024-06-29-day-180.md
index cadf548b6..ef03bc18e 100644
--- a/_posts/2024-06-29-day-180.md
+++ b/_posts/2024-06-29-day-180.md
@@ -1,5 +1,79 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 180) From Kaggle to BigQuery dimension tables - an end2end pipeline
+categories: [applying-knowledge,cloud,data-eng]
 ---
+
+## Hello :) Today is Day 180!
+A quick summary of today:
+* finished data modelling in dbt
+* set up PROD in dbt
+* set up automatic dbt job runs in mage
+* created an end to end pipeline
+
+All code from today is on [my github repo](https://github.com/divakaivan/lending-club-data-pipeline).
+
+1. Settling down on a data model in dbt
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhFzC9iFIYqNstDJ2IDqyh7_QL91djC7NQPcfcSXyecjNJvfsB7AHBHci8kiE_YEmKhkolJL_osPV7Q2gz--Cig99lpnKPON9J3DACAqzV6CLfghbfzJmjWoJ56d-pUH6gx3wb5nkRxAPo2jBjSOCUyaQYL8gDqSlgwYgzF7R6KA-fx9QT2DdJhdZKkIux8)
+
+I went over a few different today, but I ended up with the above one. Because all my data is coming from 1 source I felt like, in order to avoid redundancy - I just decided to have dim_loans as the main table, and then have dim_borrower which includes just info about the borrower and dim_date just about the loan issue date.
+I also added data description fields, and some tests.
+
+The below pics are taken from the dbt generated documentation:
+* dim_borrower
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhpUusra68HSPhtHBuFF88sojLAB4_mH7BcBuLfl1IDBfp8SVUPMsaI6tuJ5vwaymMdjU5odekeKx9AS1bH1WsPJOruNevacWe4-SA9EVcTiesdBxRrP2LbCBWniBxdaeCQzury3UEboKq1SGC-AhRS3miHSeu2x9ss8w1N_sG0SNNYRNnNYxRBSAhlTYoD)
+
+* dim_date
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEhdJfe5gIKUc5_rYLZ-0AyvzZ-N5qsN8mjUwHUX8q7FfnSMhFToQjokbU5FtMZ_EkDtoGKO5y6nvHIuT0ko2BZwk_IuWQ5lPMNBVo_82L47Q48sRxTIhDIzFhA7aySY2O8-UbmTigR98eDfFhOvpnxE_I0pb0gGhk5x3JzuLW8vHTISFGPwBhiqALLUo9uk)
+
+* dim_loans (image is truncated as there are many fields)
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiaRdspMRZPOSVx1lIKbrBp13Jfk4eNmg3z3Alg5bLvvKuVupYXZ2y9kXnnUyBOpRVI0aqtUONLBc0Ni9fTszK14EJ41ipB6jyiaNc8cNMr0duw3bXEiNE55xccPTynJRnpsCbI5VqiJxM9duIHrWQPHCrfpah6Wm-Z4cB42cg3HXR7c3dZTeAJ5GWHkPs_)
+
+2. Setting up a PROD environment for dbt
+
+After I finally settled on a data modelling architecture, I created a PROD env to run all the models in a job.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg5qsRUCrDXPlRaOBHAdUATYKA0rmTY5BclLKDhMBNea25hkEEnro21Noh7FsffaelCMdEamp2exa87lybKI7QQeiI4yGrcCMX2SkE1FhVe2nyPTs2r_V3MZwsGcvbA2Kabrx9BvUeqI7qTMf3G-tl6z3uNPiczC3-yhdbIl5J5KYd8kmLrGTY3suqxTxKi)
+
+Not seen in the pic, but there is an 'API trigger' button which shows how we can trigger job runs from some other pipeline orchestrator. This gave me the idea to use this in mage
+
+3. Automatic dbt build trigger in mage
+
+It is just a one block pipeline
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEg-Jnuq465_IZ1P8UZnk0LDcMMmAwqdV9QUzqk86eyBKQDah56iIPb9DMj3vthBc9dJy4dCokVQgPBQoKn6nP9yGkrOvadi0uduvNDPWh8gIYj6yf6IoEKLM6qhLGevf9CWBHp-qJoHvjFZr89qUEP6aI26af_MrBJz6uX7F3XPU4-RqjLq3NGT5QKsBNPi)
+
+But it took me some time to figure out an issue. The issue was that I kept getting error 401 Unauthorized when I used mage's dbt trigger integration. Turns out, the functionality only works when mage is used from their docker image. However, I am running mage, after installing it with pip. I checked whether my API link actually works using postman and then I just switched to using python's request library to send a trigger request to dbt. 
+
+4. An end to end pipeline in mage
+
+At this point, I had 3 pipelines in mage: 
+
+* data_raw_to_gcs - which takes data from the kaggle api, does small checks and loads it to GCS
+* gcs_to_bq - SQL queries to load data from GCS to BigQuery
+* run_dbt_job - the pipeline described above
+
+So now, I wanted to make an end to end one that executes all 3 of the above pipelines. And mage has a nice way of doing it too. 
+
+I created this end_to_end 1 block pipeline
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiGINMkXTe9mCLBCsZvv9M1jIAUbFP8EYMe_u6ncX7Zc4Vz85o9X1I4NG5lZG8_0uSksLPPpkAxKckpoUCCrxyxtZ-7Vb9_HMp55TDJXPIvEM2SDlOtTCKjKv1Woercs8KW6Iwru00LuGQhbWd7xLuzW1o7XegpYpi1RYntfiisEGY-Bsqwk5Rf24idACKc)
+
+And it executes all three above - from downloading Kaggle data to uploading the final dim_loans, dim_borrower, dim_date to BigQuery. 
+
+
+5. A small thing 
+
+I updated the Makefile help options and descriptions again. To make them a bit more user-friendly
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEiqUPNvEDN9lQJIRwapfviOswOPXPck1AxKJN7rKTMfDa7WyLEpsqKlGemEpTkksKi_Ak43Ta5SQgnmtSp-ix_jQOWyzqTaOWaXTrEBHVvrfofldx4GXXSM0TRjXHSxnDsv7-xZXItAoqdkTbLMNHgTxep-MO6DwZm1z4JVapb8iRLA4_AiScb7SUtc1kgL)
+
+Tomorrow, I will start working on a dashboard in Looker. ^^
+
+That is all for today!
+
+See you tomorrow :)

From 469618d53244a1548155d27024696c468e44dc5a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:38:34 +0900
Subject: [PATCH 289/638] Update 2024-06-30-day-181.md

---
 _posts/2024-06-30-day-181.md | 43 ++++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-06-30-day-181.md b/_posts/2024-06-30-day-181.md
index cadf548b6..04516f90c 100644
--- a/_posts/2024-06-30-day-181.md
+++ b/_posts/2024-06-30-day-181.md
@@ -1,5 +1,44 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 181) Lending club data engineering project - Done
+categories: [cloud,data-eng,applying-knowledge]
 ---
+
+## Hello :) Today is Day 181!
+A quick summary of today:
+* completed and documented my data engineering project
+
+Everything is on [my github repo](https://github.com/divakaivan/lending-club-data-pipeline), but below I will provide an overview.
+
+A diagram overview of the tech used:
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEh9AAHDbwRObk9crubw4VdFgTvDQHxRYNOaSWjVcLDi8_-krMMq4KHEixMuLZbgPR3ubxAGoiEM9SwSUT1jAmr-qTNvs22uIgh_2-7iDXvBKNZLL10GxtRDqO4JgGOsMLkQsj6AY43dlzYKccP9ISP6Iy9DS2VYS2Wn6r3jkcO6r_TnNOnDpRZvUO_rSMJu)
+
+* Raw Lending Club data from Kaggle
+* Mage is used to orchestrate an end to end process including:
+  * extract data using kaggle's API and load it to the Google Cloud Storage (used as a data lake)
+  * create tables in BigQuery (used as a data warehouse)
+  * run dbt transformation jobs
+* Terraform is used to manage and provision the infrastructure needed for the data pipeline on Google Cloud Platform
+* dbt is used to transform the data into dimension tables, add data tests, and create data documentation
+* Looker is used to create a visualisation dashboard
+
+For the dbt documentation, I was using dbt cloud IDE for the development, but to deploy a docs I needed to get its files, so the easiest way was to sync and run dbt locally. Setting up dbt to sync with local files was not hard, and this gave me easy access to a 'target' folder which contains files for the auto-generated documentation. Once I got this folder, I 'dropped' it into Netlify and boom - the [dbt docs](https://lending-club-project-dbt-docs.netlify.app/) is now available for all. 
+
+Today I completed the visualisation dashboard in Looker, which is the last official bit from the project.
+
+![](https://blogger.googleusercontent.com/img/a/AVvXsEgu6qEwSXiUbzWILWJTz8qEC427FQ5uQUe0seI1SlKx7o32wXPzqfIw0TRmIVRCqml8hrkz-N1z7Mi5WraOD6CcVOh5r4pp3I_syw3PRumyd_OLIKGYhffP2pN9RE8OtgOtrGaWRlRNaPW9pjQfNfO2tydHwi7UJ6mAQn38Oa-AXM_0IYX5pD2woJSqEb8h)
+
+[Here](https://lookerstudio.google.com/u/0/reporting/de05bd99-c678-4088-8abd-fa0974c0da0e) is a link to Looker. It was a learning curve... one thing I struggled with was histograms. They do not seem to be a native chart in looker so I had some bad looking bar charts for a bit. Then I saw that I can create bins of numerical variables and that is how the middle and bottom right charts became a reality. Overall, it was pretty smooth but Looker can be kind of slow sometimes if I make too many changes quickly. 
+
+Overall thoughts ~
+
+All of these technologies I learned about from DataTalksClub's Data Engineering zoomcamp. The value I got from this *free* course is astonishing. Thank you so much to the origanisers, I learned about data modelling, the cloud, infrastructure as code, data pipelines, orchestration, automation, batch, streaming, and of course the exact tools from the first picture. 
+
+I am currently doing their LLM zoomcamp and MLOps zoomcamp so I hope I have a project from there too in the near future.
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From 52443dc16149d917fe011908c4e60093139c9ae5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:45:13 +0900
Subject: [PATCH 290/638] Update 2024-07-01-day-182.md

---
 _posts/2024-07-01-day-182.md | 185 ++++++++++++++++++++++++++++++++++-
 1 file changed, 183 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-01-day-182.md b/_posts/2024-07-01-day-182.md
index cadf548b6..cc687bc7f 100644
--- a/_posts/2024-07-01-day-182.md
+++ b/_posts/2024-07-01-day-182.md
@@ -1,5 +1,186 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 182) Learning about feature selection in fraud detection and finding a classifier model with low recall
+categories: [traditional-machine-learning,applying-knowledge]
 ---
+
+## Hello :) Today is Day 182!
+A quick summary of today:
+* learning about IV, WoE, and finding a best model for an imbalanced insurance fraud imbalanced dataset
+
+The time has come to start thinking about the project for MLOps zoomcamp. 
+I was looking around for some interesting dataset related to PD (probability of default) or LGD (loss given default) or EAD (exposure at default), and I found [this](https://www.kaggle.com/code/chandrimad31/credit-risk-part-1-binning-woe-iv-pd-model#Splitting-into-Train-and-Test) notebook. Warning - it is fairly long. But inside I saw something that interested me - it talked about WoE and IV. It says that they are good estimators for evaluating features for fraud and similar classification tasks. [This](https://www.listendata.com/2015/03/weight-of-evidence-woe-and-information.html) website's definition was the most clear.
+
+**Weight of Evidence (WoE)**
+
+It is a technique used in credit scoring and predictive modeling to assess the predictive power of independent variables relative to a dependent variable. Originating from the credit risk world, WoE measures the separation between "good" and "bad" customers. Here, "bad" customers typically refer to those who have defaulted on a loan, while "good" customers are those who have repaid their loans.
+
+Key Concepts of WoE:
+
+* Calculation: WoE is calculated using the natural logarithm (ln) of the ratio of the proportion of non-events (e.g., non-defaults) to the proportion of events (e.g., defaults) within a group or bin of the independent variable.
+* Interpretation: A positive WoE indicates that the proportion of non-events is higher than events, suggesting a stronger likelihood of the event not occurring. Conversely, a negative WoE indicates a higher proportion of events compared to non-events.
+* Transformation: WoE transforms both continuous and categorical variables into continuous measures, making it easier to interpret their relationship with the target variable.
+
+Steps to Calculate WoE:
+
+* Bin Creation: For continuous variables, divide data into bins or groups based on the variable's distribution.
+* Event and Non-event Calculation: Count the number of events and non-events (e.g., defaults and non-defaults) within each bin.
+* Percentage Calculation: Calculate the percentage of events and non-events in each bin.
+
+WoE Calculation: Compute WoE using the formula:
+
+![image](https://github.com/user-attachments/assets/feaf418a-d367-4a75-8989-2cecd3c460f3)
+
+Usage: Use WoE values instead of raw input values in predictive models like logistic regression to enhance model interpretability and performance.
+
+Advantages of WoE:
+
+* Handles Missing Data: Allows separate treatment of missing values.
+* Treats Outliers: Bins extreme values into groups, reducing their impact.
+* Simplifies Model Interpretation: Provides a clear linear relationship between predictors and the target variable's log odds.
+* No Need for Dummy Variables: Directly transforms categorical variables into continuous ones.
+
+Considerations:
+
+* Monotonicity: WoE should exhibit a monotonic relationship with the target variable across bins.
+* Bin Size: Each bin should ideally contain at least 5% of the observations to ensure robustness.
+* Model Stability: Incorrect binning can lead to model instability, hence the importance of validating the monotonic relationship of WoE values across bins.
+
+Information Value
+
+It is a technique used in predictive modeling to assess the importance of independent variables in predicting a binary outcome, typically used in credit scoring and similar applications.
+
+Definition and Calculation:
+
+* Definition: IV measures the predictive power of a variable by quantifying the relationship between the variable's WOE (Weight of Evidence) and the target variable.
+* Formula: IV is calculated as the sum of the differences between the percentage of non-events and events in each category (or bin) of the variable, weighted by the WOE of that category:
+
+![image](https://github.com/user-attachments/assets/d57151e0-efdb-410a-9125-fe1a5a09e5fb)
+
+Interpretation of IV:
+
+IV values are used to rank variables based on their predictive power:
+* Less than 0.02: Not useful for prediction.
+* 0.02 to 0.1: Weak predictive power.
+* 0.1 to 0.3: Medium predictive power.
+* 0.3 to 0.5: Strong predictive power.
+* Greater than 0.5: Suspiciously strong predictive power, warrants careful examination.
+
+Usage and Considerations:
+
+* Variable Selection: IV helps in selecting important variables for inclusion in models, especially in binary logistic regression where it directly relates to the log odds of the target variable.
+* Limitations: IV is most effective for binary outcomes and logistic regression models. For other models like random forests or SVMs that can capture non-linear relationships well, IV might not always identify the most predictive variables.
+* Bin Size Impact: Increasing the number of bins or groups for a variable can increase its IV, but too many bins with few observations can lead to unreliable IV values.
+
+So how did I apply this new thing that I learned?
+
+I found a nice car insurance fraud dataset on [Kaggle](https://www.kaggle.com/datasets/shivamb/vehicle-claim-fraud-detection/data).
+
+Turns out I can do IV and WoE for both numerical and categorical values. All besides 2 in that dataset are categorical.
+
+Here are the features I used:
+
+```python
+numerical_features = ['DriverRating'] 
+
+categorical_features = [
+
+        'Month', 'WeekOfMonth', 'DayOfWeek', 
+
+        'Make', 'AccidentArea', 'DayOfWeekClaimed', 
+
+        'MonthClaimed', 'WeekOfMonthClaimed', 'Sex', 
+
+        'MaritalStatus', 'Fault', 'PolicyType', 
+
+        'VehicleCategory', 'VehiclePrice', 'Days_Policy_Accident', 
+
+        'Days_Policy_Claim', 'PastNumberOfClaims', 'AgeOfVehicle', 
+
+        'AgeOfPolicyHolder', 'PoliceReportFiled', 'WitnessPresent', 
+
+        'AgentType', 'NumberOfSuppliments', 'AddressChange_Claim', 
+
+        'NumberOfCars', 'Year', 'BasePolicy', 'Deductible'
+
+    ] 
+
+
+
+target = 'FraudFound_P'  
+```
+
+First, I used IV to filter out features under 0.02 and above 0.6 (not 0.5 because there was 1 feature with 0.55)
+
+This filter cut down the list to only:
+
+```python
+['NumberOfSuppliments',
+
+ 'AgeOfVehicle',
+
+ 'AgeOfPolicyHolder',
+
+ 'Month',
+
+ 'Deductible',
+
+ 'MonthClaimed',
+
+ 'Make',
+
+ 'AddressChange_Claim',
+
+ 'PastNumberOfClaims',
+
+ 'VehiclePrice',
+
+ 'VehicleCategory',
+
+ 'Fault']
+```
+
+Then I plotted WoE for each of those. 
+
+An example - from AgeOfVehicle
+
+![image](https://github.com/user-attachments/assets/283dc626-ec68-4596-ac99-b0fbe5d691bd)
+
+Here I decided to combine 3 years and 4 years into one, 5+6+7+more than 7 years into another one. 
+
+I repeated the same process by looking at the WoE line graph for each of the 'meaningful' features. For some I also considered their distribution. For example, in car Make (manufacturer) these are the count of each value
+
+![image](https://github.com/user-attachments/assets/de077c73-e1b1-4cfb-a5eb-7705b01c8059)
+
+Here I also decided to combine the low represented cars, only if their WoE was similar. Maybe there are other things to consider but for now the above is what I did. 
+Next, I had a dataset with 12 categorical features and I got dummies for all, and the dataset was ready for a model. 
+
+Just like in the notebook I referenced at the start that uses a different dataset (related to credit risk), I started with a simple logistic regression. 
+
+The resulting confussion matrix:
+
+![image](https://github.com/user-attachments/assets/03f5d9b6-20bb-41ae-9384-7bd3b9df87b5)
+
+Recall being 0.83 for fraud cases. I think recall is the number that is the most important because if it is 1 that means we can detect all insurance fraud cases. 
+
+To search for a better model, I looked into XGBoost
+
+Best model I got (recall 0.41 for fraud cases):
+
+![image](https://github.com/user-attachments/assets/a3ca47e5-627e-46d5-b8c8-a12e15575754)
+
+Next, I tried CatBoost (recall for fraud cases is 0.21):
+
+![image](https://github.com/user-attachments/assets/0c032f05-b7b0-43b9-bc7f-8a69edfd1d3a)
+
+Next, an ensamble with RFClassifier and XGBoost (recall for fraud cases is 0.39):
+
+![image](https://github.com/user-attachments/assets/57825cd5-3857-4739-bad0-d372f15e1a41)
+
+Next, I decided to look on Kaggle and see the notebooks that use this dataset. Turns out that BalancedRandomForestClassifier does the best job (according to the top notebook). I applied it using my features, and recall for fraud cases is 0.90. However as seen from the confusion matrix, the other indicators are hurt. So, the task next (tomorrow) is to find a way to get the False Positives (top right) lower, while holding the True negatives down. 
+
+![image](https://github.com/user-attachments/assets/a97543b5-f085-41c8-8713-3abbcbd1d0a3)
+
+That is all for today!
+
+See you tomorrow :)

From 001e8c80efc83a5642fdc24792949478225eda1c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:49:23 +0900
Subject: [PATCH 291/638] Update 2024-07-02-day-183.md

---
 _posts/2024-07-02-day-183.md | 54 ++++++++++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-02-day-183.md b/_posts/2024-07-02-day-183.md
index cadf548b6..c8a800bc1 100644
--- a/_posts/2024-07-02-day-183.md
+++ b/_posts/2024-07-02-day-183.md
@@ -1,5 +1,55 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 183) Failing to install Kubeflow, and setting up mlflow on GCP
+categories: [mlops,cloud]
 ---
+
+## Hello :) Today is Day 183!
+
+Today is actually past the half-way point in this blog journey. 182 days to go!
+
+A quick summary of today:
+* setting up mlflow for experiment tracking in GCP
+
+Today I started looking at tools I can use for my MLOps zoomcamp project. I used Mage for the data engineering project so now for an orchestrator I want to try another tool.
+
+My first idea (which I had on my mind for a while) is Kubeflow. Tldr - it is so hard to set up locally and I could not find a guide that clearly explains how :( 
+
+![image](https://github.com/user-attachments/assets/b6566749-0473-4976-8c06-74758f242aca)
+
+I had [this](https://www.youtube.com/watch?v=6wWdNg0GMV4&t=1814s&pp=ygUIa3ViZWZsb3c%3D) 1hr video in my 'to watch' for a bit and it goes over building an ML pipeline in kubeflow however it does not go over how to install from kubeflow from 0 to what is needed in the video. Then I started looking in kubeflow's documentation - I tried to set it up on GCP but I could not and actually was a bit adamant to use it because it explicitly said that the free credits on GCP do not cover kubeflow hosting costs. So ~ I started looking into local deployment. There were 3 different posts (1 of which from IBM) that were from 2023 and all set it up differently from nothing to kubeflow, but ... none of them work :( I was losing hope as I was investing so much time and I could not, not even use it, just install and set up kubeflow...
+
+At some points I got to the part where I have a kubeflow cluster running and can go to localhost:8080 but I could not run a pipeline due to some error. 
+
+I decided to step back a bit, and set up something more basic - mlflow. 
+I did it locally, but for this project I wanted to use GCP. 
+
+I created a Compute Engine 
+
+![image](https://github.com/user-attachments/assets/af09d443-7e11-4605-8b61-ed5b9274d478)
+
+Created a storage bucket for mlflow artifacts
+
+![image](https://github.com/user-attachments/assets/416a2409-8f1f-4a1a-842f-d6dd2f93ea5b)
+
+Using SSH I went into the VM and installed mlflow and set up a sqlite db. I decided to use sqlite because:
+
+1. I will not have that many experiments to keep
+2. creating an SQL db on GCP seems expensive to keep active
+
+Then I tried running a simple experiment locally, but I was getting a weird error from mlflow: Anonymous credentials cannot be refreshed
+
+it seemed related to my GCP credentials so I thought that I needed to add my creds.json file to the VM so that it can access them - did that but it was not the issue.
+
+Turns out, in the jupyter notebook I needed to explicitly point to the creds.json file 
+`os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'my-creds.json'`
+and with that my issue was fixed and now mlflow hosted on GCP worked. 
+
+Though I failed to do anything with kubeflow today, I will not give up. I used Mage for orchestration in the data eng project, so now I will try to use another tool like Prefect (if not Kuberflow). But still, nothing is certain, so I will not be surprised if I write in tomorrow's post that I am using Mage for this project too. Mage is pretty nice ^^
+
+I uploaded the parts of my project to [this GitHub repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline).
+
+
+That is all for today!
+
+See you tomorrow :)

From 7148c0023dcbd465eb0e95c631f60e1ba0028f5c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:53:46 +0900
Subject: [PATCH 292/638] Update 2024-07-03-day-184.md

---
 _posts/2024-07-03-day-184.md | 49 ++++++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-03-day-184.md b/_posts/2024-07-03-day-184.md
index cadf548b6..e6351e24d 100644
--- a/_posts/2024-07-03-day-184.md
+++ b/_posts/2024-07-03-day-184.md
@@ -1,5 +1,50 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 184) Mlflow experiment tracking and trying out metaflow
+categories: [mlops,applying-knowledge,cloud]
 ---
+
+## Hello :) Today is Day 184!
+A quick summary of today:
+* got various experiments ran and saved in my GCP mlflow server
+* set up and tried out metaflow for MLOps orchestration
+
+Today I had to re-create my GCP VM because of a python version issue. So after starting a VM with python 3.10 I could use mlflow's most current logging functionality. 
+
+I ran a few experiements and the currently saved are:
+
+![image](https://github.com/user-attachments/assets/348dd018-390a-435f-bba7-df9cc1710faf)
+
+I will select a model solely based on recall - being able to detect the most fraud cases out of all fraud cases. Based on that, so far (after finding optimal params) the Balanced RF Classifier is the best -> recall being 0.92. 
+The most recent code and files are in [my repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline). 
+
+Today I did some refactoring to make things look nicer -> added a utils folder with some helper functions that I used a lot throughout notebooks. The current are log_model_performance and get_best_params which log a model run to mlflow, and then load the params of a model from mlflow. 
+
+Another thing - I asked on reddit's MLOps page what is a good tool to tryout because so far I know only Mage as an orchestrator, and what is a good, popularly used tool, for beginners and for an MLOps project. Metaflow got the most votes, but also flyte. I decided to first check out metaflow as it seems it is widely used in top companies like Netflix and LinkedIN.
+
+metaflow has an UI which can be set up in the cloud (prefered method) or locally (which I did following [this](https://medium.com/devops-techable/up-and-running-with-metaflow-ui-locally-with-docker-desktop-2b49b52828b3) article).
+
+I found a simple metaflow workflow, and modified it as follows:
+
+![image](https://github.com/user-attachments/assets/f8c6d2b1-bf29-495d-82de-86777837d691)
+
+The code for it is on my repo as well, under testing_metaflow.py
+
+I am running the metaflow UI locally and the below is what I get in the UI (given a success run)
+
+![image](https://github.com/user-attachments/assets/9674b6eb-03f6-435f-b171-aa0f26191fc1)
+
+Going into one of the 'green' runs:
+
+![image](https://github.com/user-attachments/assets/5b27ede4-df2b-4f6e-8d15-49770a67ca40)
+
+However, I am not seeing anything else. I am not sure if it is because I need to initialise it in my code, or maybe because I am running the UI locally. Nevertheless, it seems nice (I guess) but I need to figure out logging more info and maybe artifacts. 
+
+Tomorrow ~ I will try to explore metaflow a bit more. Hopefully I can figure out how to see more things in the UI.
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 7ad2ac7b343021952ee23096e1659e1bb0756330 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 20:56:29 +0900
Subject: [PATCH 293/638] Update 2024-07-04-day-185.md

---
 _posts/2024-07-04-day-185.md | 64 ++++++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-04-day-185.md b/_posts/2024-07-04-day-185.md
index cadf548b6..a6c80ce54 100644
--- a/_posts/2024-07-04-day-185.md
+++ b/_posts/2024-07-04-day-185.md
@@ -1,5 +1,65 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 185) Using prefect as my orchestrator for my MLOps project
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 185!
+A quick summary of today:
+* creating a flow for model training
+* creating a flow for uploading data to GCS
+
+Well, I decided to go with Prefect as my orchestrator. I have little experience using it and I liked it, so went with it. 
+
+![image](https://github.com/user-attachments/assets/1c40edac-42ae-4474-809a-74d8fbc9556d)
+
+All the code from today is on [my repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline).
+
+As seen from the pic, I ran jobs many times and have many successes and failures. 
+
+The first flow (as they are called in prefect) I created is for model training. 
+
+This is an example visualisation of the flow from one of the runs:
+
+![image](https://github.com/user-attachments/assets/8a41beb1-9d1c-498f-888a-e8413db688b3)
+
+It contains 4 tasks:
+* read data (not from GCS, I need to make it read from GCS)
+* split data
+* get best params for the model from mlflow
+* train the model, and produce reports on it
+
+The reports are quite nice as we can use markdown and python f-strings together to make them:
+
+Model report:
+
+![image](https://github.com/user-attachments/assets/25788162-6b8d-4339-9d03-9eff59644425)
+
+Mlflow info:
+
+![image](https://github.com/user-attachments/assets/d81e0f9a-2a74-43fb-a41b-70b99d766cfe)
+
+Deploy instructions:
+
+![image](https://github.com/user-attachments/assets/3ff03a7b-dc56-44dc-aafb-b1a4c9867ad0)
+
+Instead of going to mlflow to get info, I tried to make it a bit easier and clearer on how to get a model, and also showing basic performance stats. 
+
+I am using Prefect's latest (and recently) released version 3.x. So to deploy a flow - make it available to be ran from the UI there is a .serve method, and I spent some time figuring that out, but finally it works fine. 
+
+![image](https://github.com/user-attachments/assets/edd2dcc5-dc20-46ee-a1c6-67d49b4c8d78)
+
+Next, I created a flow to upload the data for the model to GCS. 
+
+First I needed to create a prefect 'block' (like env variables) for my GCP credentials in the UI, and a block to connect to my GCS bucket. Then, creating the flow is quite short:
+
+![image](https://github.com/user-attachments/assets/7fd1cbe9-14f4-43b6-bca4-1d3a8e15bf76)
+
+Here I faced an issue. The flow deployment (.serve()) method comes with Prefect 3.x. However, to upload to GCS, another library is used - prefect-gcp which uses an older version of Prefect (2.19.7) and I cannot deploy this workflow, but I guess it is fine for now because this data does not need to be re-uploaded multiple times to GCS. Nevertheless, I mentioned it in Prefect's community discord so hopefully it gets picked up and fixed. 
+
+Today, I spent some time figuring out how to run the prefect UI from docker, but ultimately I failed. Maybe I will tackle it again tomorrow.
+
+
+That is all for today!
+
+See you tomorrow :) 

From d8826e96d73324963172fc2fcabacb923424bd6b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:21:40 +0900
Subject: [PATCH 294/638] Update 2024-07-05-day-186.md

---
 _posts/2024-07-05-day-186.md | 79 +++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-05-day-186.md b/_posts/2024-07-05-day-186.md
index cadf548b6..b820720a6 100644
--- a/_posts/2024-07-05-day-186.md
+++ b/_posts/2024-07-05-day-186.md
@@ -1,5 +1,80 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 186) Prefect cloud, model serving with FastAPI, and SHAP values
+categories: [cloud,mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 186!
+A quick summary of today:
+* using FastAPI for model serving, and running it in Docker
+* starting to think about model monitoring
+
+All code, pictures are on my [github repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline).
+
+Firstly, yesterday I was having version issues between prefect and prefect-gcp - fixed that with the help of Prefect's community slack, and now I can run one file which runs both flows - upload_to_gcs flow and train_model flow. I also realised that for solo (and mini project) devs Prefect cloud is free, so I uploaded my flows to there just to be a bit safer - not having everything locally only. 
+
+![image](https://github.com/user-attachments/assets/a209a453-ca7c-48db-8061-067b4c9dddae)
+
+The UI is the same, but it is in the cloud. 
+
+Next, serving the model for inference
+
+I decided to use FastAPI as it seems very easy to create and run a server. Also when I run a fastapi server, there is a nice /docs page where I can quickly and easily test my endpoints. 
+
+![image](https://github.com/user-attachments/assets/c5db2008-398d-43f8-ab39-50140a837d20)
+
+It is pretty basic for now, and I just need to improve the prefict_fraud function. I also set up the server to run in docker. Again I had some issues with file location and ports, but I could more easily figure them out (given my experience with Docker from the data eng pipeline project). 
+
+The Dockerfile:
+
+![image](https://github.com/user-attachments/assets/bc16c969-3f34-4a08-b269-498179c2b1e9)
+
+And docker-compose.yml
+
+![image](https://github.com/user-attachments/assets/7ebac23a-f355-412f-a565-a7b3e73cd0cf)
+
+Given I got this server set up, and also having to serve Prefect flows, I decided to bring in make. 
+
+My makefile has the following options (for now at least):
+
+![image](https://github.com/user-attachments/assets/880a993a-c553-434a-aa37-a24129d7aa76)
+
+
+Maybe if I can figure out how to set up to run Prefect in docker, this will become a bit simpler. But I am yet to figure this out. Nevertheless for now, it is not bad ^^
+
+
+Now that I had model serving (with FastAPI) set up, I started to think about monitoring.
+
+But what can I monitor? The dataset is not only small, but no new data will come in. I can of course set aside some rows of data, or even generate fake data maybe. 
+
+I can monitor:
+
+* performance metrics (recall, accuracy, roc-auc, false positive rate, false negative rate, etc)
+* fraud/not fraud distribution
+* look for anomalies
+* optimal threshold
+* feature importance
+
+One interesting and new (for me) way to look at feature importance is to look at SHAP values. SHAP shows feature importance based on the contribution of each feature to the model's predictions, averaged over all possible feature combinations, ensuring a fair and consistent representation of their impact. 
+
+After some issues with generating visualisations, I got some charts
+
+Fraud SHAP values:
+
+![image](https://github.com/user-attachments/assets/16f31834-0619-44d7-920c-0820016974d0)
+
+Not Fraud SHAP values:
+
+![image](https://github.com/user-attachments/assets/3847febc-85ff-4624-84da-aea422d86526)
+
+There are some other png files that I managed to create (following [this](https://youtu.be/2xlgOu22YgE) video). All are on my repo. 
+
+On another note ~ I realised I might have made a mistake when preparing the dataset I used for the model. What I did was, I had all these variables: NumberOfSuppliments,AgeOfVehicle,AgeOfPolicyHolder,Month,Deductible,MonthClaimed,Make,AddressChange_Claim,PastNumberOfClaims,VehiclePrice,VehicleCategory,Fault,FraudFound_P
+
+And I just got dummies for all. However, I did this without thinking whether ordinal encoding might be better. After some review, doing dummies for all seems fine, but for other cases I need to keep this in mind. Maybe it is something to explore - doing ordinal encoding for some of the vars for which it might be interesting to see the effect on model performance (like vehicle price maybe). Nevertheless, for now doing OHE(dummies) for all seems fine, and the model performance is satisfactory for now.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From b55c099b1553882621fcef66bb3f20b6ca8fdb04 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:24:14 +0900
Subject: [PATCH 295/638] Update 2024-07-06-day-187.md

---
 _posts/2024-07-06-day-187.md | 67 ++++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-06-day-187.md b/_posts/2024-07-06-day-187.md
index cadf548b6..1766fae17 100644
--- a/_posts/2024-07-06-day-187.md
+++ b/_posts/2024-07-06-day-187.md
@@ -1,5 +1,68 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 187) Setting up postgres, pgAdmin, Grafana and FastAPI to run in Docker
+categories: [mlops,applying-knowledge,glaswegian]
 ---
+
+## Hello :) Today is Day 187!
+A quick summary of today:
+* creating a docker container with postgres db, pgadmin, grafana and fastapi
+* a small Glaswegian project update
+
+Docker and I are becoming closer and closer friends ^^
+
+Before going into that, I will go in order of things I did today.
+
+
+
+All files and code from today, and up until this point are on [my repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline).
+
+Firstly, monitoring with Evidently
+
+I learned about Evidently from the mlops zoomcamp, and it looked very easy to use, so I gave it a try. 
+
+In order to generate the reports, I decided to use my data before creating dummies. Because the balanced random forest classifier model uses a dataset with all dummies (all categorical data). So in order for the monitoring to be a bit more understandable, I decided to use the 'pre-dummy' dataset, and below are some parts of the generated reports by Evidently. 
+
+There is variable info:
+
+![image](https://github.com/user-attachments/assets/851e413e-948d-4a36-9776-b9c6e4703563)
+
+Missing values:
+
+![image](https://github.com/user-attachments/assets/c36c1990-05e6-448e-898c-75052620f7d2)
+
+Drift:
+
+![image](https://github.com/user-attachments/assets/b93e83b0-5b54-444d-80fa-8867888fcc56)
+
+Model performance:
+
+![image](https://github.com/user-attachments/assets/3a692fd0-9879-49d2-9093-b27d2dd140de)
+
+The reports (as html) are on my repo in the monitoring/evidently_reports folder.
+
+
+### Next ~ Grafana
+
+In the zoomcamp we were thought of both Evidently and Grafana. And I see that Grafana is a more flexible tool for monitoring that allows easier data integration from multiple resources.
+
+I *did* start a grafana server, however I started thinking about PostgreSQL. 
+
+I started to think that I do not want to use GCS for all my files - it might be better practice and a good exercise to think of GCS as a storage for keeping just some raw files, while I can use a postgres server for intermediate data.
+
+So I started to set up Grafana and a postgres server, and then I added pgAdmin using Docker. Then I decided to combine these services with the FastAPI docker service I established yesterday - so all runs together. And this is where I spent most of my time today. 
+
+I had little folder mapping probles, however the biggest one - I kept getting 'anonymous credentials cannot be refreshed' from google. I use GCS to download a model for FastAPI. It worked completely fine locally, but when I run the FastAPI server from docker the error kept coming up. I kept just chaning paths to try to point the env to the credentials file, but nothing worked. In the end... I went to what I had originally (this is after ~2 hours of trying to fix this) and it ran... haha 
+
+So now, with make the command `make start-all` I run postgres, pgAdmin, grafana, and fastAPI. 
+
+Since I have postgres now, I left some TODOs for tomorrow - removing all reads of local file data, now I should read from postgres (rather than GCS only) - I think I have 4 such TODOs. 
+
+### On another note, a small [Glaswegian](https://huggingface.co/datasets/divakaivan/glaswegian_audio) dataset update
+
+We are up to 86 minutes of audio. And our fine-tuned whisper is performing very well (according to my Scottish project partner), so I will try to help a bit with the transcription and send just some notes on what he might need to double-check to make sure whisper and I transcripbed a clip correctly. 
+
+
+That is all for today!
+
+See you tomorrow :)

From 1b181839fc4a3d02ff14cd51e6ffb0954c837408 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:27:06 +0900
Subject: [PATCH 296/638] Update 2024-07-07-day-188.md

---
 _posts/2024-07-07-day-188.md | 72 +++++++++++++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-07-day-188.md b/_posts/2024-07-07-day-188.md
index cadf548b6..99b04d69c 100644
--- a/_posts/2024-07-07-day-188.md
+++ b/_posts/2024-07-07-day-188.md
@@ -1,5 +1,73 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 188) Setting up automatically updated monitoring UI using streamlit
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 188!
+A quick summary of today:
+* setting up postgres properly
+* creating and hosting a monitoring interface using streamlit
+* creating a Grafana dashboard
+* setting up a batch model prediction flow in prefect
+
+All code from previous days and today is on [my repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline).
+
+### First, setting up a better way to upload data to postgres
+
+Before, everytime I ran docker the script checked if I had inserted data into postgres. Now I set up a proper sql script that inserts data once and I am set up (as long as I do not delete the built container of course)
+
+I put all the sql and commands I used to upload the data in the postgres folder in my repo.
+
+I first create the tables (pic is truncated)
+
+![image](https://github.com/user-attachments/assets/cafe3c96-4878-449e-aafc-f9dca25b7b81)
+
+And I also added a default column for FraudFound_P_prob just in case the user wants to get probabilities when they run a pred model. 
+
+I uploaded 2 datasets to postgres - the above one, and one ready for the balanced random forest classifier model - it is the same as the above one but all vars are in dummies. Given that, I removed all instances in my code where I read data from local sources -> now every instance reads code from the database. 
+
+### Second, onto monitoring
+
+I cleaned up the monitoring folder - removed LIME as I am not going to use it for this project, and I decided to use streamlit to automate the monitoring UI generation and serving. 
+
+And it is [here](https://insurance-fraud-model-monitoring.streamlit.app/). I figured out how to render the html reports generated by Evidently, and also added SHAP images. Luckily for me, recently streamlit introduced pages - so the UI looks nice.
+
+![image](https://github.com/user-attachments/assets/b3d11f7c-7a12-4659-8081-f6ffac53922f)
+
+And on each:
+
+![image](https://github.com/user-attachments/assets/645f8ef0-b3ab-481c-aff0-1df89d3b6a94)
+![image](https://github.com/user-attachments/assets/0be392f4-5062-4b56-9a2c-aec2b412a4d9)
+![image](https://github.com/user-attachments/assets/dae36be2-a66a-4947-a50a-f841d45b6914)
+
+I have 3 different reports. 
+Next, I created a prefect flow that updates the artifacts (html and png files) used for the monitoring UI, and after the user uploads them to github -> the streamlit UI will be updated as well. A nice little automation piece. 
+
+### Next, I decided to use Grafana to create some kind of a visualisation dashboard there. 
+
+I set up a connection to postgres to read data, and after some playing around, I created a basic dashboard. It is quite similar to Looker. 
+
+![image](https://github.com/user-attachments/assets/0156e71b-2180-4c8d-9fae-bb47489685d3)
+
+### Next, I started thinking how the model would be used
+
+Rather than streaming - i.e. something that users input single data points and get an output, this type of model might be better suited for batch - use a new whole dataset and predict Fraud/Not Fraud for the whole thing. 
+
+If I decide to (just for fun) do a web server using FastAPI, I found [this](https://youtu.be/xZ013IgK7Ts) nice video which shows how to host a FastAPI app for free. 
+
+Then, I just created a prefect flow that takes in new data, the model to use, adds prediction column (and optionally preds probabilities), and finally uploads it to postgres. 
+
+I have five prefect flows at the moment ^^
+
+![image](https://github.com/user-attachments/assets/c6d13f8d-3daa-4c6b-8b4d-e0fa30efed61)
+
+I am thinking now about adding tests, maybe creating a python documentation with sphinx since I added simple docstrings to all my code and data types. But those are tasks for tomorrow/next days.
+
+I need to add terraform as well so that the GCP infrastructure is initialised using terraform, rather than manually (a bit more automation). 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 41b985ba6c550cbb94c40dc27a458ea705f02958 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:29:37 +0900
Subject: [PATCH 297/638] Update 2024-07-08-day-189.md

---
 _posts/2024-07-08-day-189.md | 79 +++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-08-day-189.md b/_posts/2024-07-08-day-189.md
index cadf548b6..202ecfb99 100644
--- a/_posts/2024-07-08-day-189.md
+++ b/_posts/2024-07-08-day-189.md
@@ -1,5 +1,80 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 189) I finished the Car Insurance Fraud MLOps project. Thank you MLOps zoomcamp for teaching me so much!
+categories: [mlops,cloud,applying-knowledge]
 ---
+
+## Hello :) Today is Day 189!
+A quick summary of today:
+* completed the project and wrote project description
+
+The whole project and all info is on [my repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline)
+
+Here is a project diagram I created using lucid.app
+
+![image](https://github.com/user-attachments/assets/a2b57dac-2c64-45dd-99fb-f06b25234a41)
+
+Well ~ today I added some final things to my project. 
+
+
+
+First I added terraform code to create GCP buckets for mlflow articats and raw data, and also start a VM.
+
+The last one is kind of cool because when we start the VM, mlflow starts automatically as well because the code to start mlflow is included in the `metadata_startup_script` which is:
+
+![image](https://github.com/user-attachments/assets/ba0044d6-12c7-4ef1-8d6c-9686f35ec3f3)
+
+I also added the whole terraform init, plan and apply setup in the project's Makefile for easy set and start up. So now the make help looks like:
+
+![image](https://github.com/user-attachments/assets/1aa9f9eb-f693-41a0-9b99-42b8104b63b6)
+
+Next, I added pylint
+
+Following the suggestions from MLOps zoomcamp, I added pylint and fixed a lot of whitespace, and import order lines of code as per pylint's suggestions. 
+
+Next, I tried to add some tests using pytest
+
+I added 5 simple tests to some of the prefect flows. 
+
+The exact tests are:
+
+* test_read_new_data
+* test_predict
+* test_save_predictions
+* test_batch_model_predict
+* test_local_to_gcs
+
+But there is work to be done for more thorough testing. One thing Mage.ai was nice for is that in every block of code, at the end of it there was a dedicated test place so adding tests was very easy (which is not the case for Prefect)
+
+Next, I added python documentation
+
+I used sphinx back in my placement year with Lloyds Banking Group, and I have been adding small docstrings to my code, so adding a sphinx python docs seemed like a nice little addition. I easily set it up, but I could not get github actions to automate this, so I used an alternative:
+
+Use a new branch where the _build folder (normally not uploaded to github because it has many files) is uploaded and netlify watches that branch and updates the docs
+
+[Here](https://fraud-model-prefect-docs.netlify.app/) is the link to the netlify hosted python docs for the project.
+
+Finally, I added some basic git pre-commit hooks
+
+* trailing-whitespace
+* end-of-file-fixer
+* check-yaml
+* check-added-large-files
+* pytest-check
+* pylint
+
+These are ran and checked before every commit, and the commit is stopped if one fails.
+
+![image](https://github.com/user-attachments/assets/7067b6b9-c327-4894-80f7-2246677c4b3e)
+
+After this, I started putting things together and writing the README of the project, and creating the project diagram (top of this blog).
+
+There are things to improve, but nevertheless DataTalksClub - THANK YOU SO MUCH! MLOps zoomcamp is such an amazing course, and it taught me amazing tools to build amazing things.
+
+From now on, I will continue learning the ongoing LLM zoomcamp - hopefully I will build an amazing project from there as well!
+
+
+
+That is all for today!
+
+See you tomorrow :)

From d4d73c52e06dfd83daeada8b82273e08887aeac1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:32:34 +0900
Subject: [PATCH 298/638] Update 2024-07-09-day-190.md

---
 _posts/2024-07-09-day-190.md | 70 ++++++++++++++++++++++++++++++++++--
 1 file changed, 68 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-09-day-190.md b/_posts/2024-07-09-day-190.md
index cadf548b6..3867d7981 100644
--- a/_posts/2024-07-09-day-190.md
+++ b/_posts/2024-07-09-day-190.md
@@ -1,5 +1,71 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 190) Learning about evaluating vector search engines for RAG apps
+categories: [nlp,applying-knowledge]
 ---
+
+## Hello :) Today is Day 190!
+A quick summary of today:
+* covered [Module 3](https://github.com/DataTalksClub/llm-zoomcamp/tree/main/03-vector-search): vector search from the LLM zoomcamp
+
+All the code from today is on [my repo](https://github.com/divakaivan/llm-camp/tree/main/Module%203).
+
+The first part of the module was related to doing semantic search using dense vectors in Elasticsearch
+
+1. Loaded Q&A documents from a json file
+
+2. Created dense vectors for each document using a pre-trained model
+
+3. Created an index in Elasticsearch
+
+4. Indexed the documents in Elasticsearch
+
+5. Performed a semantic search using the dense vectors
+
+6. Filtered the results using a specific section
+
+![image](https://github.com/user-attachments/assets/1d064599-ef15-4ec3-b4d0-d739786719e6)
+
+(picture is from the course)
+
+### Next I learned about evaluating the retrieval mechanism
+
+1. Generate unique IDs for each document to distinguish each other
+
+2. Generate 5 sample questions for each document using the GPT API
+
+3. Save the results to a file to use for evaluation
+
+first 10 rows from the created dataset:
+
+![image](https://github.com/user-attachments/assets/eaa3a06f-f2ee-49ac-8f6d-05d9cb99ead5)
+
+The ID is needed to connect the sample created questions to the documenta they are related to.
+
+
+### Next, I learned about two evaluation metrics for evaluating the used search mechanism
+
+Recall
+
+* Measures the number of relevant documents retrieved out of the total number of relevant documents available.
+* Formula: Recall = (Number of relevant documents retrieved) / (Total number of relevant documents)
+
+Mean Reciprocal Rank
+* Evaluates the rank position of the first relevant document.
+* Formula: MRR = (1 / |Q|) * Σ (1 / rank_i) for i = 1 to |Q|
+
+To do evaluation, a few different engines were created and compared based on recall and MRR
+
+* base elastic search based on word similarity: recall: 0.74, mrr: 0.60
+* KNN using embedded question body: recall: 0.77, mrr: 0.66
+* KNN using embedded text body: recall: 0.83, mrr: 0.71
+* KNN using embedded question and text boxy: recall: 0.92, mrr: 0.82
+
+All ran with different times, so it is about evaluating what do we care more, speed vs a set % improvement
+
+At the end, I completed the [homework](https://github.com/divakaivan/llm-camp/blob/main/Module%203/homework3.ipynb) which covered similar questions like the above learned content.
+
+
+That is all for today!
+
+See you tomorrow :)

From c6b9cc7c612ebebde134accc93cd844d63dfaf9a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:39:10 +0900
Subject: [PATCH 299/638] Update 2024-07-10-day-191.md

---
 _posts/2024-07-10-day-191.md | 247 ++++++++++++++++++++++++++++++++++-
 1 file changed, 245 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-10-day-191.md b/_posts/2024-07-10-day-191.md
index cadf548b6..9616a7e55 100644
--- a/_posts/2024-07-10-day-191.md
+++ b/_posts/2024-07-10-day-191.md
@@ -1,5 +1,248 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 191) Starting the book - Effective Data Science Infrastructure
+categories: [mlops,applying-knowledge,theory]
 ---
+
+## Hello :) Today is Day 191!
+A quick summary of today:
+* today I started a book from Manning - Effective Data Science Infrastructure
+
+Below is a summary of the topics covered up to and including chapter 3.
+
+![image](https://github.com/user-attachments/assets/a3fa8df8-c079-479f-9569-5b6226e4be44)
+
+To enhance data scientists' productivity, prioritize a well-structured development environment over solely focusing on scalable production infrastructure. Many companies overlook this, managing code development, debugging, and testing haphazardly. Effective development environments, integral to infrastructure, enhance productivity by focusing on ergonomics, defined as optimizing efficiency in the working environment.
+
+
+
+Key activities to optimize include:
+
+1. Prototyping - Translating expertise into functional code and models.
+
+2. Interaction with Production Deployments - Connecting code and models to systems to generate business value.
+
+
+
+The prototyping loop (similar to the REPL loop in software engineering) involves developing, evaluating, and analyzing code iteratively. To boost productivity, streamline each step and transition in this loop.
+
+
+
+This prototyping loop integrates with the higher-order loop of production deployment, forming an infinite cycle of improvement and debugging, akin to continuous delivery (CD) in software engineering. However, data science differs from traditional software in:
+
+1. Correctness - Validated post-deployment through experiments, not pre-deployment tests.
+
+2. Stability - Affected by constantly changing data, unlike stable software environments.
+
+3. Variety - Requires iterative testing with new ideas and data, unlike static software functions.
+
+4. Culture - Data scientists are not typically trained in the deep culture of DevOps and infrastructure engineering, necessitating a human-centric approach to development environments.
+
+![image](https://github.com/user-attachments/assets/7bde405e-87c7-4cba-b1f8-c46d00650bb6)
+![image](https://github.com/user-attachments/assets/c2ce12a5-c717-47d5-ad7e-3235ba1b322f)
+
+I learned about:
+
+AWS Cloud9 - it is a cloud-based IDE that functions like a local editor and terminal, managing an EC2 instance to support prototyping and production interactions, offering cost-effectiveness but lacking built-in support for notebooks.
+
+Amazon SageMaker Studio - it is a JupyterLab-based environment focused on data science, managing ML-specific instances and integrating seamlessly with AWS data science services, making it ideal for users of those services but more costly than standard EC2 instances.
+
+
+Running notebooks (options)
+
+![image](https://github.com/user-attachments/assets/60bdd71b-a416-4e95-9331-1435030081df)
+
+Example workstation for a data scientist
+
+![image](https://github.com/user-attachments/assets/a6ac701b-b031-44ac-a0db-131e5f7ae65b)
+
+Workflows
+
+a workflow is a directed graph, that is a set of nodes or steps connected by directional arrows
+
+![image](https://github.com/user-attachments/assets/17762b53-83ed-4276-84c5-a0db9535ace8)
+
+executing workflows is the job of an orchestrator
+
+
+When evaluating orchestrators, we can think of the architecture, which includes the code structure and system interface; the job scheduler, which involves workflow triggering, execution, monitoring, and failure handling; and compute resources, which consider the system's ability to handle varying resource requirements and parallel execution capabilities.
+
+![image](https://github.com/user-attachments/assets/3e3e4ad1-8e99-4e8b-b669-4ad0fe8fb006)
+
+When choosing a workflow framework for data science, prioritize the productivity of data scientists by selecting an architecture tailored to data science use cases and considering the full data science stack. Operational concerns like robustness, scalability, and high availability are critical long-term factors, so opt for systems with proven reliability. Ensure the framework integrates seamlessly with your compute resources to enhance productivity.
+
+This book uses metaflow and I hope they provide clear instructions on setting it up.
+
+What dev env is best for your team size?
+
+![image](https://github.com/user-attachments/assets/7cdd4142-9a34-4c57-afca-53af3117f1f9)
+
+### Next, starting to use metaflow
+
+First, install with pip install metaflow
+
+Then create a hello world flow:
+
+![image](https://github.com/user-attachments/assets/553a1789-e4b0-4892-b596-4bba4786b0c1)
+
+Its output when run:
+
+![image](https://github.com/user-attachments/assets/a2c92229-5947-4fbe-9af8-94f3d5f26688)
+
+when we run the file with `show` added in the terminal, we see a clearer flow
+
+![image](https://github.com/user-attachments/assets/b79e3fc6-a60e-4c9d-ad23-471a3ded36bb)
+
+this is a textual representation of the DAG
+
+when we execute the flow, we see:
+
+![image](https://github.com/user-attachments/assets/ccb707c2-6920-44be-89be-0a875999ed67)
+![image](https://github.com/user-attachments/assets/c8568f26-e628-4548-9255-652bce5f4457)
+
+Using the run id, we can check the flow that was executed:
+
+![image](https://github.com/user-attachments/assets/01ecd7b3-300c-4409-8fe5-e7522bfdd3ec)
+
+### Next, metaflow artifacts
+
+Metaflow artifacts (seems they are like artifacts in MLflow) are data objects automatically persisted by Metaflow when assigned to self in step code. They can include any Python objects serializable with pickle. Artifacts are stored in a datastore for reproducibility, maintaining immutability for accurate experiment tracking across workflow steps.
+
+To illustrate artifacts, we created this simple CounterFlow
+
+![image](https://github.com/user-attachments/assets/806fd698-e5a2-465a-97a5-dc9122931282)
+
+RULE OF THUMB
+
+Use instance variables, such as self, to store any data and objects that may have value outside the step. Use local variables only for intermediary, temporary data. When in doubt, use instance variables because they make debugging easier.
+
+Flows may fail. Because of that metaflow saves each state of the flow prior to the failure - and by examining this we can better understand what happened.
+
+When we run the flow, we get the run id and step, and we can examine the state at that point
+
+![image](https://github.com/user-attachments/assets/ef2461d1-644b-47f3-b970-91a21cc3e3fb)
+
+how the metaflow datastore works:
+
+![image](https://github.com/user-attachments/assets/04b2c01a-887e-494a-9919-b11324c29efc)
+
+When designing my flows in Prefect, for my last project I had thought a lot about what to put in a step - do I have multiple things, do I make it simpler? And usually I sided to put less functionality in 1 single step - so that if a step fails I know exactly where it fails. Whereas if a lot of operations were in a single step it might require more debugging. And the book provides a similar rule of thumb: 
+
+Structure your workflow in logical steps that are easily explainable and understandable. When in doubt, err on the side of small steps. They tend to be more easily understandable and debuggable than large steps.
+
+Next - parameters
+
+Parameters are a flow level construct and are available for all the steps in the flow
+
+We can create params using the Parameter class
+
+![image](https://github.com/user-attachments/assets/a3e0dc72-cede-4f05-8207-6fbb90e77a53)
+
+The actual flow is printing info about the params, and when we do just python parameters.py run
+
+we get:
+
+![image](https://github.com/user-attachments/assets/b5b92488-505a-4053-94f0-3ff108f42c0d)
+
+And the rest of the params can be passed the same way but they are not required
+Params are constant immutable values and if they need to be changed in the code, we need to create a copy and assign it to another artifact.
+we can also set env vars to be used in runs. Like: `export METAFLOW_RUN_CREATURE=dinosaur` and this value will automatically be used instead and we don’t need to pass —creature when we run the flow
+
+Even if a param is set in the env vars, if we pass —creature new_creature then this will be used and has priority
+
+We can also add more complex params (not just int/float/str) using json:
+
+![image](https://github.com/user-attachments/assets/ff07ecba-9a41-45ee-94c6-4e8daa37a977)
+
+json file paths can also be passed using —mapping like: `python json_parameter.py run --mapping "$(cat myconfig.json)"`
+
+To include files as params, we can use the IncludeFile class from metaflow, and to add something like csv files as params we can use help="delimiter", default=',' when defining a param like:
+
+![image](https://github.com/user-attachments/assets/22231e8d-e5b7-4574-a961-45f6e2d3a0c1)
+
+In Metaflow, IncludeFile and delimiters serve distinct purposes:
+
+IncludeFile: Reads a file and stores it as an immutable artifact with the run. This means the original data is captured and versioned along with the run, ensuring reproducibility even if the source file changes or is removed.
+
+Delimiter: Specifies how fields in a CSV file are separated (e.g., comma-separated values). This ensures proper parsing of the CSV file's content.
+
+So, while IncludeFile preserves data integrity by snapshotting it with the run, delimiters ensure correct parsing of CSV data into fields.
+
+Running dynamic branches and data parallelism - can be done using the `foreach` keyword where a step is run for multiple cases in parallel
+
+![image](https://github.com/user-attachments/assets/03eb8987-f0f3-4fa5-bebd-f2bb1a58e8c2)
+
+foreach can be used to execute and run in parallel even tens of thousands of tasks - it is a powerful keyword and a key element that contributes to scalability that metaflow provides.
+To protect users from over-running tasks and making their machine heat up, metaflow has a preset concurrency limit
+
+There are also two variables that help: max_num_splits controls data partitioning for parallel processing, while max_workers limits the number of concurrent executions to manage system resources effectively during parallel execution in Metaflow workflows. Adjusting these parameters can optimize performance based on specific computational needs and available resources.
+
+Next, staring a simple classifier prediction model flow:
+
+![image](https://github.com/user-attachments/assets/175a5065-8a01-4ce2-b6ea-1d304d51c25a)
+
+After running the flow, we can inspect the logs for the start step and see the artifacts:
+
+![image](https://github.com/user-attachments/assets/e56edfcc-be6e-4314-9f44-87bb5dcc22e8)
+
+(we can see at least some of it, and ensure it is loaded)
+
+Since I ran the flow once, these artifacts are persisted and next time instead of loading them again from the API, I can load them from the metaflow datastore.
+
+I am writing this as I go, and now I see a nice definition of flow, step, task, etc:
+
+Metaflow—Contains all flows. You can use it to discover flows created by you and your colleagues.
+
+Flow—Contains all runs that have been executed with a FlowSpec class.
+
+Run—Contains all steps of a flow whose execution was started during this run. Run is the core concept of the hierarchy, because all other objects are produced through runs.
+
+Step—Contains all tasks that were started by this step. Only foreach steps contain more than one task.
+
+Task—Contains all data artifacts produced by this task.
+
+Data artifact—Contains a piece of data produced by task.
+
+![image](https://github.com/user-attachments/assets/8236c321-b4ee-4b2e-958f-ce5f36df17c8)
+
+To load a file there are 3 ways:
+
+Instantiating Objects: You can create any object using a path that uniquely identifies it in the structure. For example, Run("ClassifierTrainFlow/1611541088765447") accesses the data of a specific run.
+
+Accessing Child Objects: Using bracket notation (['start']), you can retrieve specific child objects. For instance, Run("ClassifierTrainFlow/1611541088765447")['start'] gives you the start step of that run.
+
+Iterating over Containers: You can iterate over any container to access its children. For example, list(Run("ClassifierTrainFlow/1611541088765447")) returns a list of all step objects associated with that run.
+
+And we can access the client API anywhere where python is supported - we can load files in another place like a jupyter notebook too.
+
+![image](https://github.com/user-attachments/assets/26ec6297-8e88-4e72-b7ff-33d9097358c7)
+
+When running a flow, if the flow fails for some reason, we can inspect - fix the big, and continue that specific run from the last successful step using python file.py resume
+
+resume also accepts a specific run id and a step from which to resume
+
+from chapter 3, the final full DAG looks like:
+
+![image](https://github.com/user-attachments/assets/bf985c50-ea97-42dc-aee6-9cf7744610cd)
+
+1. We obtained input data and split it into a train and test sets, which were stored as artifacts.
+
+2. We trained two alternative models as parallel branches . . .
+
+3.  . . and chose the best performing one based on the accuracy with test data.
+
+4. The chosen model was stored as an artifact, which we can inspect together with other artifacts in a notebook using the Client API.
+
+5. A separate prediction flow can be called as often as needed to classify new vectors using the trained model.
+
+
+
+Overall - very nice. After failing to setup metaflow (its UI to be specific) some days ago, following this chapter so far I had no issues and I can easily learn about metaflow. I hope the next chapters are just as good.
+
+All code from today is on [my repo](https://github.com/divakaivan/learn-metaflow).
+
+
+
+That is all for today!
+
+See you tomorrow :) 

From 3befef71ef3f6a40ffede740f284bca2d187d866 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:43:25 +0900
Subject: [PATCH 300/638] Update 2024-07-11-day-192.md

---
 _posts/2024-07-11-day-192.md | 151 ++++++++++++++++++++++++++++++++++-
 1 file changed, 149 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-11-day-192.md b/_posts/2024-07-11-day-192.md
index cadf548b6..6126ea689 100644
--- a/_posts/2024-07-11-day-192.md
+++ b/_posts/2024-07-11-day-192.md
@@ -1,5 +1,152 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 192) Chapter 4 - Scaling with the compute layer (from the book - Effective Data Science Infrastructure)
+categories: [cloud,mlops,applying-knowledge,theory]
 ---
+
+## Hello :) Today is Day 192!
+A quick summary of today:
+* set up AWS resources and configured metaflow with aws (finally)
+
+What is Scalability?
+
+Scalability refers to a system's ability to handle increasing amounts of work by adding resources. This concept is distinct from performance, which measures how well a system functions under a fixed workload. Scalability involves:
+
+
+
+1. Growth: It’s relevant only when discussing systems that need to handle more work.
+
+2. Efficient Resource Use: Adding resources like more computers or memory should proportionally increase the system's capacity.
+
+3. Different Measures: The dimensions of scalability (e.g., volume, velocity, validity, and variety) must be defined based on specific needs.
+
+When building infrastructure, it’s crucial to ensure scalability across all layers, supporting a large number of applications and users, and enabling quick development and deployment of data science projects.
+
+
+
+Culture of Experimentation
+
+Modern data science organizations should allow data scientists to innovate freely without being limited by compute resources. This is facilitated by the reduced cost of computing, particularly through cloud infrastructure, which offers near-unlimited resources.
+
+However, as organizations scale, communication overhead can become a significant issue. Traditional hierarchical structures limit this overhead but modern organizations prefer more flexible, less hierarchical models. Effective use of cloud resources can minimize the need for coordination by providing easy access to computing power, thus supporting more projects and enabling more experimentation
+
+Key Strategies for Scalability:
+
+1. Minimize Interference: Ensure that different workloads do not negatively impact each other. This can be achieved through:
+
+   - Proper isolation of workloads.
+
+   - Using modern cloud infrastructures and container management systems.
+
+   - Employing versioning, namespaces, and dependency management.
+
+2. Enable Autonomy: Allow data scientists to manage the entire prototyping and deployment process independently, reducing the need for extensive coordination.
+
+In summary, scalability is about efficiently handling growth by leveraging additional resources, while minimizing the need for coordination and ensuring system robustness. This fosters a productive environment where data scientists can experiment and innovate freely.
+
+
+
+Next, the compute layer
+
+![image](https://github.com/user-attachments/assets/358a2087-4bd4-4172-a51b-e2341360af5d)
+
+Infrastructure Stack and Workflow Execution:
+
+* Workflow Interface: Users define tasks via workflows, but the compute layer only handles individual tasks.
+* Workflow Orchestrator: Schedules tasks and determines execution timing.
+* Compute Layer: Focuses on where to execute tasks, based on resource requirements (CPUs, RAM).
+
+Compute Layer Requirements:
+
+* Must handle numerous concurrent tasks.
+* Manages a dynamic pool of physical computers.
+* Efficiently matches tasks to available resources, a complex problem (Bin Packing, Knapsack problem).
+* Ensures robustness against failures and errors.
+
+Evolution with Public Clouds:
+
+* Public clouds like AWS have simplified building scalable, robust compute layers.
+* They allow elastic scaling from small to supercomputer levels, paying only for usage.
+* However, the basic cloud interfaces are low-level and require additional architectural decisions for data science workloads.
+
+Diverse Compute Needs:
+
+* Different applications have varying compute requirements.
+* Data science infrastructure should support various compute layers (local laptops, cloud workstations, GPU clusters).
+* These layers can share a common interface and architecture to manage diversity effectively.
+
+![image](https://github.com/user-attachments/assets/26349301-7373-4131-979a-9c8f074d621d)
+
+* Instance Pool: Contains computers that run containers executing user code.
+* Cluster Management System: Manages the instance pool, adding or removing instances based on demand and health, accommodating diverse hardware configurations.
+* Container Orchestration System: Queues pending tasks and assigns them to suitable instances based on resource requirements, ensuring tasks are placed on available instances with the necessary resources (e.g., GPU).
+
+Examples of compute layers
+
+![image](https://github.com/user-attachments/assets/37b5e345-d6bc-4366-923f-86deb049c7ba)
+
+We should evaluate compute layers based on:
+
+* workload support
+* latency
+* workload management
+* cost-efficiency
+* operational complexity
+
+Popular compute layers:
+
+* kubernetes
+* AWS Batch
+* AWS Lambda
+* Apache Spark
+
+![image](https://github.com/user-attachments/assets/81c66405-e2da-4c59-8cf3-df17208d46e5)
+
+**RULE OF THUMB**
+
+Provide one general-purpose compute layer like Kubernetes or AWS Batch for heavy lifting and a low-latency system like local processes for prototyping. Use more specialized systems as required by your use cases.
+
+Cost?
+
+To optimize cloud-based compute costs, companies should maximize utilization, ensuring instances perform useful work as much as possible. This can be achieved by shutting down idle instances and sharing instances across multiple projects to avoid downtime. While instances are cheaper than data scientists, using more instance hours to save data scientist time can be more cost-effective, even if the code is inefficient.
+
+![image](https://github.com/user-attachments/assets/3d650e1c-29de-40d0-9d82-1fc28f1978fb)
+
+Configuring mlflow with aws
+
+The book did not provide *exact* way to do it but it pointed me towards a resource that can help. TLDR - I set up resources in AWS using [this](https://github.com/outerbounds/metaflow-tools/tree/master/aws/cloudformation) CloudFormation template and then configured metaflow to connect to them. 
+
+Now when I run a task locally it goes to AWS, and I see something like:
+
+![image](https://github.com/user-attachments/assets/799b792d-991f-4b07-9ca2-bc881379d15a)
+
+This is just metaflow serivce, I have not set up the UI yet (maybe the book will help me set it up in the next chapters).
+
+Then to finish up I learned about some decorators
+
+@batch
+
+This decorator specifies that a step should run on AWS Batch, a managed batch computing service. It allows you to define the computing environment and resources for that step.
+
+@retry
+
+This decorator specifies retry behavior for a step, allowing it to automatically retry on failure.
+@resources
+
+This decorator specifies the resources allocated to a step, such as CPUs and memory, typically used for local or Kubernetes execution.
+
+@catch
+
+This decorator handles exceptions for a step, specifying what should be done if the step fails.
+
+To see the decorators in action, I used some basic flows. All the code from today is in the chapter 4 directory in [my repo](https://github.com/divakaivan/learn-metaflow).
+
+
+
+Looking forward to chapter 5: Practicing scalability and performance
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 8ebda9d8cbe719e53514845470f1022a50fe668b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:49:33 +0900
Subject: [PATCH 301/638] Update 2024-07-12-day-193.md

---
 _posts/2024-07-12-day-193.md | 172 ++++++++++++++++++++++++++++++++++-
 1 file changed, 170 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-12-day-193.md b/_posts/2024-07-12-day-193.md
index cadf548b6..00415ed91 100644
--- a/_posts/2024-07-12-day-193.md
+++ b/_posts/2024-07-12-day-193.md
@@ -1,5 +1,173 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 193) Chapter 5, 6, and 7 from Effective Data Science Infrastructure
+categories: [mlops,theory,cloud]
 ---
+
+## Hello :) Today is Day 193!
+A quick summary of today:
+* covered chapter 5,6, and 7 from Effective Data Science Infrastructure
+
+### Chapter 5: Practicing scalability and performance
+
+![image](https://github.com/user-attachments/assets/c82268bd-9b03-4d9b-81c9-df5bbb29131d)
+
+Effective infrastructure must accommodate a wide range of projects. Rather than adopting a one-size-fits-all approach, it should offer a versatile toolbox of robust methods to achieve adequate scalability and performance. To enhance organizational scalability and ensure projects are comprehensible to the largest audience, our primary strategy is simplicity. Given that people's understanding is limited overengineering and overoptimizing can cause extra costs.
+
+
+
+Vertical scalability
+
+![image](https://github.com/user-attachments/assets/2ddb75ab-7beb-4338-bdc2-e8bc3f216520)
+
+it refers to the idea of handling more compute and larger datasets just by using larger instances
+
+To start things, we begin with a skeleton flow, and then keep adding new things till we get to the final solution. 
+
+The model uses Yelp review data and the goal is to group reviews together to find what kind of reviews are generally posted on the platform
+
+![image](https://github.com/user-attachments/assets/fa5a9531-6ccf-4c82-bb72-c8145302480a)
+
+In metaflow we can set a flow to run in a fixed environment by adding something like:
+@conda_base(python='3.8.3', libraries={'scikit-learn': '0.24.1'}) 
+
+above the class definition
+
+and when we run this file we need to add `--environment=conda` as an extra arg to tell metaflow that we want to use conda as env.
+
+We can also let metaflow use more resources with this tag:
+
+@resources(cpu=16, memory=4000)
+
+it tells AWS to allocate 16 CPU cores and 4 GB of memory to the workflow step to ensure it has adequate computational resources
+
+In contrast to the memory limit, the CPU limit is a soft limit on AWS Batch. The task can use all available CPU cores on the instance if there are no other tasks executing on it simultaneously. The limit applies when multiple tasks need to share the instance. In most cases, this behavior is desirable, but it makes benchmarking tricky, because a task with, for example, four CPUs, may end up using more CPU cores opportunistically.
+
+
+
+Horizontal scalability
+
+As a rule of thumb, you should consider horizontal scalability, that is, using multiple instances, if you answer “yes” to any of the following three questions:
+
+* Are there significant chunks of compute in your workflow that are embarrassingly parallel (little or no effort is needed to separate the problem into a number of parallel tasks), meaning they can perform an operation without sharing any data besides inputs?
+* Is the dataset size too large to be conveniently handled on the largest available instance type?
+* Are there compute-intensive algorithms, such as model training, that are too demanding to be executed on a single instance?
+
+
+By default, Metaflow will run at most 16 tasks in parallel
+
+PERFORMANCE TIP
+
+Premature optimization is the root of all evil. Don’t worry about performance until you have exhausted all other easier options. If you must optimize performance, know when to stop.
+
+
+
+### Chapter 6: Going to production
+
+“Deploying to production” means running workflows that are fully automated and highly available, starting and executing without human intervention and rarely failing, although specific use cases may have additional requirements such as low latency, handling massive datasets, or specific system integrations.
+
+![image](https://github.com/user-attachments/assets/a3028b5b-59ab-4dde-9575-159b1fafeb72)
+
+![image](https://github.com/user-attachments/assets/1ebc75c7-1c3c-4d27-a7c2-0fe27f6243de)
+
+AWS Step Functions (SFN) is a highly available and scalable workflow orchestrator offered by AWS. It is a fully managed, maintenance-free service that integrates seamlessly with other AWS services, and is cost-competitive compared to in-house solutions. However, defining workflows manually using SFN's native JSON-based syntax is challenging, and the GUI is somewhat clunky. Additionally, SFN has limitations on workflow size, which may impact workflows with extensive parallel operations.
+To run a flow in production: python some_flow.py step-functions create
+
+And the first thing we see is:
+
+![image](https://github.com/user-attachments/assets/acc09199-fc91-4db2-a777-5c4257d54aad)
+
+When step-functions create was ran the following was executed
+
+![image](https://github.com/user-attachments/assets/08e50264-2676-4565-8f31-feb50bbb4bc2)
+
+1. Metaflow packaged all Python code in the current working directory and uploaded it to the datastore (S3) so it can be executed remotely. More about this in the next section
+2. Metaflow parsed the workflow DAG and translated it to a syntax that SFN understands. In other words, it converted your local Metaflow workflow to a bona fide SFN workflow
+3. Metaflow made a number of calls to AWS APIs to deploy the translated workflow to the cloud
+
+To trigger a run we can use: python some_flow.py step-functions trigger
+
+The some_flow.py is a simple flow with start and end steps, and since we created and trigerred a run, in the AWS console we can see the execution details and also a graph view:
+
+![image](https://github.com/user-attachments/assets/26e6b11c-c67e-41ce-aca8-1d28650590a2)
+
+Also an event list:
+
+![image](https://github.com/user-attachments/assets/6995876f-c1be-4d7e-86b3-605edce2e1f7)
+
+And input/output
+
+![image](https://github.com/user-attachments/assets/658eb038-fcfb-4035-9dbc-f3808ab22122)
+
+Using our terminal we can see some more infor about a specific flow
+`python sfntest.py step-functions list-runs`
+
+![image](https://github.com/user-attachments/assets/07376ba9-57b5-420e-9a47-29b1f317e3a7)
+
+Also, we can set a flow to run on a schedule using the @schedule decorator
+
+* Use containers and the @conda decorator to manage third-party dependencies in production deployments.
+* User namespaces help isolate prototypes that users run on their local workstations, making sure that prototypes don’t interfere with each other.
+* Production deployments get a namespace of their own, isolated from prototypes. New users must obtain a production token to deploy new versions to production, which prevents accidental overwrites.
+* The @project decorator allows multiple parallel, isolated workflows to be deployed to production concurrently.
+* Use @project to create a unified namespace across multiple workflows.
+
+### Chapter 7: Processing data
+
+![image](https://github.com/user-attachments/assets/3df5b9c0-e1c2-4779-ba6c-358babf494e3)
+
+This chapter is about loading data from different sources and thinking about the size of our project and what is the best option for a data warehouse - local? db? cloud?
+
+Using local files:
+
+* Fast Loading: Loading data from a local file is significantly quicker than executing a SQL query
+* Stable Dataset: For effective prototyping, it's crucial that the dataset remains consistent. Systematic experimentation and iterations become impossible if the underlying data changes unexpectedly
+* User-Friendly: Loading data from local files doesn’t require special clients. It avoids random failures or slowdowns due to concurrent experiments by colleagues and is compatible with almost all off-the-shelf libraries
+
+Unfortunately, local files have many downsides: they are unsuitable for production deployments or scaled-out experiments in the cloud and require manual updates, bypassing data security and governance policies. Using a cloud-based object store like AWS S3 can overcome these issues, making data compatible with cloud-based compute and governance, and with some effort, the user experience can be nearly as seamless as accessing local files, often even faster.
+
+
+
+Comparison between common file types their loading types:
+
+![image](https://github.com/user-attachments/assets/21c8cd07-1025-4830-91be-7eb95afa4612)
+
+RECOMMENDATION
+
+Whenever feasible, use the Parquet format to store and transfer tabular data instead of CSV files.
+
+![image](https://github.com/user-attachments/assets/f401aa12-531b-40de-9c26-f46a55e6ce4c)
+
+RECOMMENDATION
+
+If memory consumption is a concern, avoid storing individual rows as Python objects. Converting to pandas can be costly as well. The most efficient option is to use Arrow and NumPy, if possible.
+
+Interfacing with data infrastructure
+
+![image](https://github.com/user-attachments/assets/c30840e1-0c26-403f-be9e-c170c1d2ab7c)
+
+* Data: At the core, you have the data asset itself, stored as CSV files, Parquet files, or tables in a database.
+* Durable storage: Use durable storage systems like AWS S3 or replicated databases instead of a USB thumb drive; a modern data lake with a metadata layer like Apache Hive or Iceberg can facilitate access.
+* Query engine: A query engine processes SQL statements to select, filter, and join data; newer systems like Trino or Apache Spark are loosely coupled with storage, while streaming data can use systems like Apache Druid or Pinot.
+* Data loading and transformations: ETL is essential for data engineering; newer systems like Snowflake or Spark support ELT, and tools like DBT simplify data transformations, while tools like Great Expectations ensure data quality.
+* Workflow orchestrator: ETL pipelines are expressed as DAGs and executed by orchestrators like AWS Step Functions or Apache Airflow; using a centralized orchestrator can manage both data science and data workflows.
+* Data management: Increasing data complexity requires data management components like catalogues (e.g., Amundsen), governance systems for security and policies, and monitoring systems for overall data health and ETL pipeline quality.
+
+![image](https://github.com/user-attachments/assets/bd522999-3fa8-4cad-a628-828a0b336194)
+
+As projects and teams grow the distinction between a data engineer and a data scientist becomes more unclear
+
+![image](https://github.com/user-attachments/assets/002a215d-c9de-4e5b-b0d3-06534696414b)
+
+The data engineer is responsible for data acquisition, that is, gathering raw data, data quality, and any data transformations that are needed to make the data available as widely consumable, authoritative, carefully curated datasets. In the case of structured data, the datasets are often tables with a fixed, stable schema. Notably, these upstream datasets should focus on facts—data that corresponds as closely as possible to directly observable raw data—leaving interpretations of data to downstream projects. The organization shouldn’t underestimate the demands of this role or undervalue it. This role is directly exposed to the chaos of the real world through raw data, so they play a critical role in insulating the rest of the organization from it. Correspondingly, the validity of all downstream projects depends on the quality of upstream data.
+
+The data scientist focuses on building, deploying, and operating data science applications. They are intimately familiar with the specific needs of each project. They are responsible for creating project-specific tables based on the upstream tables. Because they are responsible for creating these tables, they can iterate on them independently as often as needed.
+
+![image](https://github.com/user-attachments/assets/2e896756-b24c-4eb0-b3ca-450fd8c95aac)
+
+I definitely need to do some kind of project using metaflow so I can experience these things first hand. I tried to go over as much as possible today because I am getting charged by AWS as we speak haha. But when I decide to use metaflow for a project hopefully I can either set it up in GCP where I have free credits, or perfectly - just locally. 
+
+
+That is all for today!
+
+See you tomorrow :)

From 9059f6efa6d0716d41cb06c6dd5073bb5335e2e2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 21:54:25 +0900
Subject: [PATCH 302/638] Update 2024-07-13-day-194.md

---
 _posts/2024-07-13-day-194.md | 40 ++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-13-day-194.md b/_posts/2024-07-13-day-194.md
index cadf548b6..910517c22 100644
--- a/_posts/2024-07-13-day-194.md
+++ b/_posts/2024-07-13-day-194.md
@@ -1,5 +1,41 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 194) Using Video Generation Models for Taxi OD Demand Matrix Prediction
+categories: [reading-research,deep-learning,applying-knowledge]
 ---
+
+## Hello :) Today is Day 194!
+A quick summary of today:
+* I finished the paper for which I read many papers and posted them throughout May/June/July
+
+![image](https://github.com/user-attachments/assets/4cf2dfbe-7be4-485b-adec-d2207baf91e6)
+
+Since May I have been talking about reading research related to predicting OD demand matrix using either graph neural networks or next-frame (video) prediction models. Well fast forward to today ~ and I finished it. Everything is on my [github repo](https://github.com/divakaivan/taxi-demand-video-models-paper).
+
+I ran data through 3 models: historical average, ConvLSTM, PredRNN - HA the most common baseline, and ConvLSTM and PredRNN - two of the best next-frame prediction models.
+
+Here is the abstract of the paper:
+
+_Predicting taxi demand is essential for managing urban transportation effectively. This study explores the application of next-frame prediction models—ConvLSTM and PredRNN—to forecast Origin-Destination (OD) taxi demand matrices using a concatenated dataset of NYC taxi data from early 2024. ConvLSTM achieved an RMSE of 1.27 with longer training times, while PredRNN achieved 1.59 with faster training. These models offer alternatives to traditional graph-based methods, showing strengths and trade-offs in real-world scenarios. Additionally, an open-source framework for model deployment is introduced, aiming to bridge the gap between research and practical implementation in taxi demand forecasting._
+
+_Keywords: Taxi, Demand, forecasting, OD Matrix, Next-Frame Prediction Models_
+
+The training was tidious ... I am on a mac so it was very slow to train models on MPS. For the future I need to look into other next-frame prediction models and include them in the comparison. 
+
+The results at the moment are:
+
+![image](https://github.com/user-attachments/assets/f3417dd9-9a32-4819-a17a-742fc054fde8)
+
+Some example predictions:
+
+![image](https://github.com/user-attachments/assets/5055e7b2-c7de-4788-91a9-1e5a00c55268)
+
+The full paper is in the repository - [paper.pdf](https://github.com/divakaivan/taxi-demand-video-models-paper/blob/main/paper.pdf).
+
+Hopefully I get to develop this further, but for now I am happy I got to read so many papers on a specific topic. I learned a lot about graph neural networks, next-frame prediction models, and specifically their application in traffic forecasting and demand. Such models are transferrable to other topics so I am glad I did this research and wrote a small paper. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From fc1c32b78058f20b8a62cc02cc7d56a6d6ac7bfa Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:00:10 +0900
Subject: [PATCH 303/638] Update 2024-07-14-day-195.md

---
 _posts/2024-07-14-day-195.md | 103 ++++++++++++++++++++++++++++++++++-
 1 file changed, 101 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-14-day-195.md b/_posts/2024-07-14-day-195.md
index cadf548b6..7a30a986b 100644
--- a/_posts/2024-07-14-day-195.md
+++ b/_posts/2024-07-14-day-195.md
@@ -1,5 +1,104 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 195) Reading about bank term deposit subscription prediction models
+categories: [reading-research]
 ---
+
+## Hello :) Today is Day 195!
+
+One of my lab mates sent me a few papers to skim through to help for his team's project related to predicting bank deposit subscriptions. Thanks to ChatGPT skimming is very easy now. Below are the outputs from ChatGPT on the five papers I got.
+
+**Predictive Analytics and Machine Learning in Direct Marketing for Anticipating Bank Term Deposit Subscriptions**
+
+- **Introduction:**
+  - Direct marketing is essential for personalized client communication in banking.
+  - Predictive analytics and machine learning offer new opportunities for refining marketing strategies.
+  - The research aims to enhance direct marketing's effectiveness by applying sophisticated analytical models.
+
+- **Literature Review:**
+  - Examines eight studies on machine learning and data mining in banking.
+  - Highlights methodologies like the S_Kohonen network, Improved Whale Optimization Algorithm, META-DES-AAP, and various machine learning models.
+  - Emphasizes the importance of time deposits, customer credit products, and telemarketing success in banks.
+
+- **Proposed Methodology:**
+  - **Data Collection and Exploration:** Utilizes Kaggle datasets and explores them using visualization tools.
+  - **Exploratory Data Analysis (EDA):** Uses techniques like crosstabulation and heatmaps to uncover patterns and relationships in the data.
+  - **Feature Engineering and Preprocessing:** Transforms and prepares data for machine learning models through encoding and scaling.
+  - **Model Implementation:** Applies models like SGD Classifier, K-Nearest Neighbors, Logistic Regression, Gaussian Naive Bayes, Decision Tree, and Random Forest Classifiers. Evaluates their performance using accuracy, precision, and F1 score.
+
+- **Results and Evaluation:**
+  - Presents classification results, highlighting the Random Forest Classifier as the best performer.
+  - Evaluates models based on metrics like accuracy, sensitivity, specificity, PPV, NPV, and F1 score.
+
+- **Conclusion:**
+  - Predictive analytics and machine learning significantly impact direct marketing in banking.
+  - The study provides a comprehensive analysis and evaluation of different machine learning models.
+  - Findings guide future strategies for direct marketing in the banking sector.
+
+**Bank Predictions for Prospective Long-Term Deposit Investors Using Machine Learning LightGBM and SMOTE**
+
+- **Objective:** Investigates methods for predicting potential long-term deposit investors using historical bank data and machine learning techniques.
+- **Methods:**
+  - Uses machine learning algorithms such as logistic regression, Random Forest, and LightGBM to predict consumer interest based on features like age, job, marital status, and more.
+  - Applies SMOTE (Synthetic Minority Over-sampling Technique) to balance the dataset, improving prediction accuracy.
+- **Data:** Kaggle dataset with 32,950 entries and 16 features. Data preprocessing includes cleaning and removing irrelevant data.
+- **Results:**
+  - **LightGBM:** Achieved the highest accuracy at 90.63%.
+  - **Random Forest:** Accuracy at 90.34%.
+  - **Logistic Regression:** Accuracy at 88.89%.
+- **Conclusion:** LightGBM combined with SMOTE provides the most accurate predictions for long-term deposit investments, demonstrating higher prediction accuracy compared to previous studies.
+
+**Identifying the Best Machine Learning Model for Predicting Bank Term Deposits: An Empirical Study Using Public, Post Financial Crisis Data**
+
+- **Introduction:**
+  - Bank term deposits are a strategy for banks to secure stable funds for lending and investment.
+  - Importance of net interest margin, the difference between interest earned from borrowers and interest paid to depositors.
+  - Research question: "Which classification model can accurately predict bank term deposits from the public?"
+
+- **Methodology:**
+  - **Bias-Variance Tradeoff:** Essential in machine learning to minimize overfitting and underfitting.
+  - **Accuracy Paradox:** Imbalanced dataset with many more instances of customers not making deposits.
+  - **Statistical Models:**
+    - **Binomial Logistic Regression:** Predicts the binary target variable using the logistic function.
+    - **Decision Tree Classifier:** Uses a tree-like structure of nodes to make decisions, avoiding overfitting with pruning and cross-validation.
+    - **Artificial Neural Network (ANN):** Uses interconnected neurons in layers, adjusting weights and biases through training.
+    - **Support Vector Machine (SVM):** Finds the optimal hyperplane to separate classes using kernel functions.
+
+- **Conclusion:** The paper compares models to determine the best predictor of bank term deposits, focusing on balancing precision and recall to effectively target potential customers.
+
+**Bank Deposit Prediction Using Ensemble Learning**
+
+- **Background:** Predicting bank deposits is crucial for managing liquidity and making informed decisions.
+- **Objective:** Develop an ensemble learning approach to predict bank deposits using a combination of machine learning algorithms.
+- **Methodology:**
+  - **Data Collection:** 10,000 customer records from a Bangladeshi bank with 14 attributes.
+  - **Data Preprocessing:** Cleaning, transforming, and normalizing the dataset.
+  - **Feature Selection:** Correlation-based feature selection method to select relevant features.
+  - **Ensemble Learning:**
+    - Trained three base models: Decision Tree, Random Forest, and Gradient Boosting.
+    - Combined these models using stacking and voting techniques to create an ensemble model.
+- **Results:**
+  - Ensemble model achieved a higher accuracy of 92.15% in predicting bank deposits.
+  - Stacking ensemble approach outperformed the voting approach.
+  - Most significant predictors: average balance, age, and occupation.
+- **Conclusion:** The study demonstrates the effectiveness of ensemble learning in predicting bank deposits, highlighting the potential of machine learning techniques in improving prediction accuracy.
+
+**Applying Machine Learning to the Development of Prediction Models for Bank Deposit Subscription**
+
+- **Background:** Bank deposit subscription is crucial for banking, and predicting customer behavior is essential for effective marketing strategies.
+- **Objective:** Develop prediction models using machine learning techniques to forecast bank deposit subscription behavior.
+- **Methodology:**
+  - Collected a dataset of 10,000 customers from a commercial bank, including demographic, financial, and behavioral features.
+  - Applied four machine learning algorithms: Decision Trees, Random Forest, Support Vector Machines (SVM), and Gradient Boosting.
+  - Trained and evaluated models using a 70:30 split of the dataset for training and testing.
+  - Used performance metrics like accuracy, precision, recall, F1-score, and area under the ROC curve (AUC).
+- **Results:**
+  - Gradient Boosting algorithm achieved the best performance with an accuracy of 86.2%, precision of 84.5%, recall of 88.1%, F1-score of 86.3%, and AUC of 0.93.
+  - Most important features: customer age, income, credit score, and deposit history.
+- **Conclusion:** The study shows the effectiveness of machine learning in predicting bank deposit subscription behavior, suggesting banks can use these models for targeted marketing and improved customer retention.
+
+On another note, I am staying up late today to watch the EURO final, but while waiting for 4am to come, I am transferring posts to my [GitHub blog](https://ivanstudyblog.github.io/). I hope I get a decent amount in.
+
+That is all for today!
+
+See you tomorrow :) 

From 598fde66f5ca4da836a064173e3780e68651fbcc Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:07:28 +0900
Subject: [PATCH 304/638] Update 2024-07-15-day-196.md

---
 _posts/2024-07-15-day-196.md | 205 ++++++++++++++++++++++++++++++++++-
 1 file changed, 203 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-15-day-196.md b/_posts/2024-07-15-day-196.md
index cadf548b6..d57aa98f5 100644
--- a/_posts/2024-07-15-day-196.md
+++ b/_posts/2024-07-15-day-196.md
@@ -1,5 +1,206 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 196) Learned about 'ML canvas' and more about MLOps
+categories: [theory,mlops]
 ---
+
+## Hello :) Today is Day 196!
+A quick summary of today:
+* Today I found [this](https://ml-ops.org/) great resource for MLOps. Below I will summarise the posts I read
+
+![image](https://github.com/user-attachments/assets/249809b1-d41d-4b53-a3cb-9725e353ef9c)
+
+A better pic of the above is on my [repo](https://github.com/divakaivan/insurance-fraud-mlops-pipeline). I learned about the above 'ML Canvas' concept from the below resources.
+
+### Motivating MLOps
+Why MLOps?
+
+Machine learning (ML) models are increasingly being used in production environments, but their development and deployment are often disconnected from traditional software development and operations practices. This disconnection leads to various pain points, such as:
+
+* Lack of collaboration: Data scientists, engineers, and operators work in silos, leading to inefficiencies and errors.
+* Inconsistent workflows: Ad-hoc processes and manual interventions hinder reproducibility, scalability, and maintainability.
+* Inadequate infrastructure: Insufficient infrastructure and tools lead to difficulties in deploying, monitoring, and updating ML models.
+
+![image](https://github.com/user-attachments/assets/beba2ac5-84ad-4fa2-ad49-f860a5d18c6d)
+
+Motivation for MLOps
+
+To address these challenges, MLOps aims to bridge the gap between ML model development and deployment by:
+
+* Streamlining workflows: Automating and standardizing ML workflows to improve efficiency, reproducibility, and collaboration.
+* Improving infrastructure: Providing scalable, secure, and efficient infrastructure for ML model deployment and management.
+* Enhancing collaboration: Fostering collaboration between data scientists, engineers, and operators to ensure seamless ML model development and deployment.
+
+By adopting MLOps, organizations can:
+
+* Increase productivity: Reduce time-to-market and improve resource utilization.
+* Improve model quality: Enhance model accuracy, reliability, and explainability.
+* Reduce risks: Minimize errors, bias, and security vulnerabilities.
+
+### Designing ML-powered Software
+
+Phase Zero: Problem Formulation
+
+Phase Zero is the initial stage of the Machine Learning Operations (MLOps) lifecycle, focusing on problem formulation. This phase is crucial in setting the foundation for a successful ML project. The primary goal of Phase Zero is to define a problem worth solving and identify the opportunities where machine learning can add value.
+
+Key Activities:
+
+* Problem Identification: Identify business problems or opportunities that can be addressed using machine learning.
+* Stakeholder Alignment: Engage with stakeholders to understand their needs, expectations, and constraints.
+* Problem Formulation: Clearly define the problem, including the objectives, key performance indicators (KPIs), and success metrics.
+* Data Exploration: Initial exploration of available data to understand its quality, relevance, and potential for ML model development.
+* Feasibility Study: Assess the technical feasibility of the project, including the availability of resources, data, and infrastructure.
+
+Deliverables:
+
+* Problem Statement: A clear, concise description of the problem to be addressed.
+* Project Charter: A document outlining the project's objectives, scope, stakeholders, and timelines.
+* Initial Data Report: A summary of the initial data exploration, highlighting data quality, relevance, and potential issues.
+
+Best Practices:
+
+* Involve Stakeholders Early: Engage with stakeholders throughout the problem formulation phase to ensure their needs are met.
+* Be Iterative: Be prepared to refine the problem statement and project charter as new information becomes available.
+* Keep it Simple: Focus on a specific, well-defined problem to increase the chances of success.
+
+There is this very nice 'ML canvas' that once filled can provide an overview of the project, and I might do it for my MLOps zoomcamp project.
+
+![image](https://github.com/user-attachments/assets/fb130511-8951-49f4-8ba0-fd55be7160bd)
+
+### End-to-End ML Workflow Lifecycle
+
+![image](https://github.com/user-attachments/assets/a38daa8c-6821-4de0-9fc1-c9aaa2ca266d)
+
+The end-to-end ML workflow is a comprehensive process that covers all stages of a machine learning project, from data preparation to model deployment and maintenance. It involves multiple steps, including:
+
+* Data Preparation: Collecting, cleaning, and preprocessing data for model training.
+* Model Training: Training and testing machine learning models using various algorithms and techniques.
+* Model Evaluation: Evaluating the performance of trained models and selecting the best one.
+* Model Deployment: Deploying the selected model in a production environment, making it accessible to end-users.
+* Model Monitoring: Continuously monitoring the model's performance, identifying issues, and retraining or updating the model as needed.
+* Model Maintenance: Maintaining and updating the model over time, ensuring it remains accurate and effective.
+
+The end-to-end ML workflow is crucial for successful machine learning projects, as it ensures that models are properly trained, validated, and deployed to achieve their intended goals.
+
+### Three Levels of ML-based Software
+
+Level 1: Data and Model
+
+Focuses on data preparation, model training, and model evaluation. This level is concerned with building and training ML models.
+
+Level 2: Data, Model, and Code
+
+In addition to Level 1, this level involves writing code to deploy and integrate the ML model into a larger software system.
+
+Level 3: Data, Model, Code, and Deployment Pipelines
+
+The highest level of ML software, which includes all the previous levels and adds automated deployment pipelines to production.
+
+![image](https://github.com/user-attachments/assets/2e8fbc73-09f2-4998-8de2-13854aa8efcd)
+
+Model serialisation formatting software summary:
+
+![image](https://github.com/user-attachments/assets/ec8b1cb1-972a-45f4-86b9-db2206e62dd3)
+![image](https://github.com/user-attachments/assets/9eca2bbd-5a1e-452e-9ff2-0a27a7dd6a7c)
+
+### MLOps Principles
+
+![image](https://github.com/user-attachments/assets/6909bbbc-4518-4f01-89ed-ff85799abb8f)
+
+Automation
+
+* manual process
+* ML pipeline automation
+* CI/CD pipeline automation
+
+The below shows the automated ML pipeline with CI/CD routines:
+
+![image](https://github.com/user-attachments/assets/8c5b61fd-cd3b-4c4c-a1bc-f51db59892b8)
+
+Continuous X
+
+* Continuous Integration (CI) extends the testing and validatin code and components by adding testing and validating data and models
+* Continuous Delivery (CD) concerns with delivery of an ML training pipeline that automatically deploys an ML model prediction service
+* Continuous Training (CT) is unique to ML systems which automatically retrain ML models for re-deployment
+* Continuous Monitoring (CM) concerns with monitoring production data and model performance metrics, which are bound to business metrics
+
+Versioning
+
+The goal of the versioning is to treat ML training scrips, ML models and data sets for model training as first-class citizens in DevOps processes by tracking ML models and data sets with version control systems.
+
+Experiments Tracking
+
+Testing
+
+![image](https://github.com/user-attachments/assets/e5a79a38-5be5-4f27-a4be-2afe7b3ba876)
+
+Monitoring
+
+![image](https://github.com/user-attachments/assets/ee99987f-94f4-40d5-9de2-ae5cd9abd010)
+
+ML-based Software Delivery Metrics
+
+![image](https://github.com/user-attachments/assets/7f8146da-251d-4aed-a847-ad5a4d57a9bb)
+
+Summary of MLOps Principles and Best Practices
+
+![image](https://github.com/user-attachments/assets/99782ac7-36ae-436e-b0b4-42ad5bad8b22)
+
+### MLOps and Model Governance
+
+![image](https://github.com/user-attachments/assets/3c6626f4-4c82-408e-89af-526b3771d190)
+
+Model Governance
+
+Model governance is a crucial aspect of Machine Learning Operations (MLOps) that ensures the integration of model governance processes throughout the ML lifecycle. It involves recording, auditing, validating, approving, and monitoring models to ensure compliance with legal and corporate requirements.
+
+Importance of MLOps Governance
+
+MLOps governance is essential for deploying ML models successfully. It provides fine-grained control and visibility into how ML models and pipelines operate in production, offering benefits such as:
+
+* Elevated control and transparency
+* Structured ability to trace activities
+* Improved collaboration and decision-making
+* Reduced risks and improved operational efficiencies
+
+Model Governance Framework
+
+![image](https://github.com/user-attachments/assets/4600539d-b2ec-4cff-a7a7-942f37305dc1)
+
+A model governance framework encompasses the systems and processes in place to fulfill model governance needs for every operational ML model. It should be integrated into every stage of the ML lifecycle, covering both legal and corporate requirements.
+
+Key Components of Model Governance
+
+* Recording: tracking model development and deployment
+* Auditing: monitoring model performance and identifying issues
+* Validation: verifying model accuracy and reliability
+* Approval: ensuring models meet regulatory and corporate standards
+* Monitoring: tracking model performance and retraining as needed
+
+Connection with MLOps
+
+Model governance and MLOps are closely linked. The level of governance required depends on the number of models used and the regulations in the field. MLOps provides the infrastructure for implementing model governance processes, ensuring seamless integration and reuse of governance processes across multiple ML production pipelines.
+
+Establishing a Model Governance Framework
+
+To establish a model governance framework, it's essential to:
+
+* Integrate model governance into every stage of the ML lifecycle
+* Implement MLOps to provide the necessary infrastructure
+* Define clear roles and responsibilities for model governance
+* Establish processes for tracking, monitoring, and validating models
+
+
+That is all for today!
+
+See you tomorrow :)
+
+
+
+
+
+
+
+
+
+

From 2c2ff713990166df54bc4b24550424f51a4d50a4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:09:34 +0900
Subject: [PATCH 305/638] Update 2024-07-16-day-197.md

---
 _posts/2024-07-16-day-197.md | 53 ++++++++++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-16-day-197.md b/_posts/2024-07-16-day-197.md
index cadf548b6..d1e693200 100644
--- a/_posts/2024-07-16-day-197.md
+++ b/_posts/2024-07-16-day-197.md
@@ -1,5 +1,54 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 197) Learning about Kafka
+categories: [data-eng,applying-knowledge]
 ---
+
+## Hello :) Today is Day 197!
+A quick summary of today:
+* finally decided to learn about kafka and streamed data
+
+I started the day by watching and replicating [this](https://youtu.be/Nu9fNyI8-d0) video on Building a Real-Time Data Streaming Pipeline using Kafka,Postgres and Streamlit
+
+![image](https://github.com/user-attachments/assets/590c0431-3f1b-4c66-81a2-9c7340cb145f)
+
+Kafka, and in general streaming (data/models) have been in my zone of interest for a while, and this video finally gave me a good starting point.
+
+The video helped me set up a Kafka service, a Zookeeper (which manages metadata), a producer (which reads data from a streaming source like a live weather feed and sends it to Kafka), and a consumer (which reads data from Kafka). 
+
+The whole thing was around sentiment of generated sentences.
+
+First we generate sentences, send them to kafka using the producer, read them from kafka using the consumer and use a sentence sentiment analyser to get a score -> then upload the sentence + score to postgres -> then as each new data comes in, show it in a UI (streamlit). I took some pics but I do not have the code for it as I started overwriting the code to fit it to my project. 
+
+This is what the producer produced:
+
+![image](https://github.com/user-attachments/assets/9ea17382-61c4-494c-8fee-fbdff21004aa)
+
+The consumer read the data, added sentiment score:
+
+![image](https://github.com/user-attachments/assets/8ddc589a-6182-4f12-bd27-30ce87daaf19)
+
+Then each line as it was processed was added to the db
+
+![image](https://github.com/user-attachments/assets/cc443d26-1252-4ad8-a885-4899f1c8c2b7)
+
+And then a live feed of new sentences could be seen on streamlit
+
+![image](https://github.com/user-attachments/assets/9688d6db-fe15-4800-9b9e-cb5861efee8e)
+
+Now ~ for my project idea
+
+This was my idea: get full transaction data from stripe api (kafka python producer) > read with a kafka python consumer > use pyspark to clean it to keep only what i need > store results in a db > use grafana to visualise data from db. Also maybe use mage for orchestration
+I spent most my day setting up java/pyspark/spark in Docker. At first after I finally managed to get pyspark/spark running, I used batch processing, but I wanted to use the spark streaming api, so I tried that, but I kept getting empty data as I was not sure what is going on.
+
+My producer was a basic while loop that sends that to kafka every 3 seconds, and after the consumer reads it I wanted to do some basic manipulation. However given I could not see any data when using the streaming api, I gave up on it and decided to not use spark. So I went back to just basic cleaning of data in python and then sending to postgres. I was not happy with that. I did not want to give up on using the spark api, so now I decided to try a different route - use it in the producer to process data from stripe (to potentially utilise spark's parallelism in case millions of transactions can come in a second - this is not possible because the stripe API limits me to 25 transactions at a time, but still)
+
+All my latest code (now at 1.30am right before I sleep) is on [this](https://github.com/divakaivan/transaction-stream-data-pipeline) repo.
+
+While I appreciate the fact that I learned a lot about kafka today, I have not given up on using spark (even a little bit). I know the tech might not be that important - I also need to do documentation and data modelling. I need to think about the transferrable skills and how the things I am doing might be scaled to millions of transactions per second. But rather than implementing anything that can handle that, maybe just knowing about it is fine too ...
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 2614f96986242477d4914f200c2a95bdcded5f00 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:37:47 +0900
Subject: [PATCH 306/638] Update 2024-07-17-day-198.md

---
 _posts/2024-07-17-day-198.md | 51 ++++++++++++++++++++++++++++++++++--
 1 file changed, 49 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-17-day-198.md b/_posts/2024-07-17-day-198.md
index cadf548b6..0b5d235b3 100644
--- a/_posts/2024-07-17-day-198.md
+++ b/_posts/2024-07-17-day-198.md
@@ -1,5 +1,52 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 198) Transactions Data Streaming Pipeline Porject (v1 completed)
+categories: [data-eng,cloud,applying-knowledge]
 ---
+
+## Hello :) Today is Day 198!
+A quick summary of today:
+* data streaming pipeline project [v1 done]
+
+![image](https://github.com/user-attachments/assets/243649ae-d0ab-4e56-a61f-5db942623229)
+
+[Here is a link to the project](https://github.com/divakaivan/transaction-stream-data-pipeline)
+
+Well ... I did not know I can do it in a day (~14 hours) after yesterday's issues but here we are.
+Turns out in order to insert the full (~70 variables with nested/list structure), I need the proper pyspark schema. And yesterday I did not have that and that is why when I was reading data in the kafka producer I was getting NULL in the columns - my schema was wrong. Well today I not only fixed the schema for the 4 variables I had yesterday, but included *all* the variables that come from the Stripe API ~ 70 (for completeness). 
+
+When I run docker-compose, the data streams and is input into the postgres db (and is still running). Unfortunately, the free Stripe API for creating realistic transactions has a limit of 25, so every 3 seconds, 25 new transactions are sent to the db. It has been running half the day (since I got that set up) and as I am writing the db has (checking db) 28675 rows of transactions. 
+
+After I got this set up, I decided to move onto dbt for a bit of data documentation and modelling.
+
+![image](https://github.com/user-attachments/assets/ea3b8368-a1bf-481f-a29c-b09c4886b8f4)
+
+This is the data model I settled for now. And using dbt I created the dimension tables and the fact table. I decided to use dbt core now, rather than dbt cloud - just to see what is the difference - and it is not that different but the cloud version has its conveniences. I also wanted to document as much as possible (there is probably more to do), and I generated and hosted the dbt docs on netlify. [Here is the link](https://transaction-stream-data-docs.netlify.app/)
+
+As I was writing the dbt models, I thought about adding some more data - to make this whole thing a bit more interesting. And I thought to add currency exchange data. Since the transactions are in GBP, if a customer is paying from overseas we might need to convert the charge into some other currency. I looked around, and saw forex-python - it did not work for some reason, and then I saw an API url from which I can get daily exchange rates. I quickly set up a python script and put it in docker to be ran every 24hr (to get the current day's exchange rates for some chosen currencies). 
+
+The final dbt data lineage looks like:
+
+![image](https://github.com/user-attachments/assets/58e52f84-8f1e-4128-895c-96a0f194c970)
+
+Finally, I tried making some kind of a dashboard in Grafana. I have little data, and it is only for less than a day. So I tried to make something. 
+
+![image](https://github.com/user-attachments/assets/079753be-12d8-4d71-86fb-cc54f99971a5)
+
+However, I will keep running the docker compose to get new transaction data into the db and after a few days I can create a bit more interesting dashboard with differences in exchange rates between days, transaction amounts. 
+
+There are of course things to improve:
+
+* add PySpark tests
+* use an orchestrator
+* use more data for a better Grafana visualisation
+  * Spark might be an overkill due to the data amount limitations, but I wanted to learn how to set Spark Streaming up in case data is much more
+  * maybe find an alternative transactions data source because the Stripe API has a 25 rate limit
+  * also many of the generated values in a transaction from the Stripe API are null
+
+maybe others too. I did post on dataengineering on reddit, and asked in DataTalksClubs for feedback, so I look forward to improving this.
+
+
+That is all for today!
+
+See you tomorrow :)

From f881d316fefdc443c7366e845fd8f97a18f6d819 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:40:57 +0900
Subject: [PATCH 307/638] Update 2024-07-18-day-199.md

---
 _posts/2024-07-18-day-199.md | 74 +++++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-18-day-199.md b/_posts/2024-07-18-day-199.md
index cadf548b6..c965fdb2f 100644
--- a/_posts/2024-07-18-day-199.md
+++ b/_posts/2024-07-18-day-199.md
@@ -1,5 +1,75 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 199) Continuing with Build an LLM from scratch
+categories: [nlp,theory,glaswegian]
 ---
+
+## Hello :) Today is Day 199!
+A quick summary of today:
+* saw that all the chapters from the book [Build an LLM from scratch](https://www.manning.com/books/build-a-large-language-model-from-scratch) have been published so I decided to continue with it (after a few moths of waiting)
+
+![image](https://github.com/user-attachments/assets/8a81af33-87d5-429b-a87a-51377b04e9ef)
+
+I like that even though we are in chapter 5 (out of 7), the author still reminds us the learners the process of going from input text to LLM generated text. Making sure we are still on the same page
+
+![image](https://github.com/user-attachments/assets/b6f2089d-500c-415a-b6b3-c65438f7b331)
+
+The goal of this chapter is to train a model because at the moment, when we try to generate some text we get gibberish
+
+![image](https://github.com/user-attachments/assets/5add5280-745c-49ea-8867-604ef02834ed)
+
+At the moment the untrained model is given: Every effort moves you; and the model continues this with rentingetic wasnم refres RexMeCHicular stren
+
+By the way, the current (untrained) model has the following config: 
+
+![image](https://github.com/user-attachments/assets/7f0bc4e1-b2e2-4b44-8a93-b2e01f94d3c4)
+
+How can the model learn ? It's weights need to be updated so they start to predict the target tokens. Here comes good ol' backpropagation. And it requires a loss function which calculates the dif between desired and actual output (i.e. how far off the model is)
+
+![image](https://github.com/user-attachments/assets/81b5bad8-e125-420b-b09b-69ab09f7d0f6)
+
+Perplexity is a metric used with cross-entropy loss to evaluate model performance in tasks like language modeling. It reflects the model's uncertainty in predicting the next token, with lower perplexity indicating better performance. Perplexity is calculated as torch.exp(loss). For example, a loss resulting in 48725 indicates the model is uncertain about predicting among approximately 47,678 possible tokens. This makes perplexity a more interpretable measure than raw loss, as it signifies the effective vocabulary size over which the model is uncertain.
+
+
+This is an interesting fact from the book:
+"The cost of pretraining LLMs
+To put the scale of our project into perspective, consider the training of the 7 billion parameter Llama 2 model, a relatively popular openly available LLM. This model required 184,320 GPU hours on expensive A100 GPUs, processing 2 trillion tokens. At the time of writing, running an 8xA100 cloud server on AWS costs around $30 per hour. A rough estimate puts the total training cost of such an LLM at around $690,000 (calculated as 184,320 hours divided by 8, then multiplied by $30)." - wow
+
+Starting to train ~ we can see the model output as well. We give it Every effort moves you 
+
+![image](https://github.com/user-attachments/assets/60a34689-1071-4d63-9fca-bcc53733c075)
+
+And by the 10th last epoch it starts to look like normal text
+
+![image](https://github.com/user-attachments/assets/4468633c-e2ea-48f6-8eed-e598305754d6)
+
+A reminder that the text used is "The Verdict" short story by Edith Wharton. The train loss starts at 10.058 and converges to 0.502. 
+
+![image](https://github.com/user-attachments/assets/92af08c9-8434-4dae-bab5-331a1bae87d4)
+
+Seems the model is overfitting after the 2nd epoch which is not that surprising given the small datataset. 
+
+Sampling strategies
+
+Temperature - the lower, the 'less creative' the model is
+
+![image](https://github.com/user-attachments/assets/816459b0-7bbf-4b63-b20d-8b9388580358)
+
+Top-k - picking from only top-k tokens with the highest logits when generating
+
+![image](https://github.com/user-attachments/assets/3b0bd88d-b821-49ac-9031-05b04ab0a329)
+
+### On another note ~ I added new clips to the Glaswegian dataset. 
+
+We are now at ~1000 clips and total of 86 minutes of audio. [Here](https://huggingface.co/datasets/divakaivan/glaswegian_audio) is the link to the dataset on hugging face
+
+
+Another note ~ a lab mate told me about a banking AI competition.
+
+The bank is Kukmin Bank (국민은행 in Korean) and the competition started yesterday and the submission is 11th of Aug. We would have to develop some kind of project related to helping customers/workers using AI. We will discuss it more in the coming days/weeks. I am excited for it. [Here](http://www.kbdatory.com/notice/view) is a link to the competition (it is in Korean).
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 17659a16bbb9260a713e75aa4ed435a05c2e97e0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:45:10 +0900
Subject: [PATCH 308/638] Update 2024-07-19-day-200.md

---
 _posts/2024-07-19-day-200.md | 80 +++++++++++++++++++++++++++++++++++-
 1 file changed, 78 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-19-day-200.md b/_posts/2024-07-19-day-200.md
index cadf548b6..e72abcfd9 100644
--- a/_posts/2024-07-19-day-200.md
+++ b/_posts/2024-07-19-day-200.md
@@ -1,5 +1,81 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 200) Kukmin Bank AI competition project idea
+categories: [gnn,applying-knowledge]
 ---
+
+## Hello :) Today is Day 200!
+A quick summary of today:
+* creating a project plan, timeline and learning about graph DBs
+
+My lab mate and I considered two project ideas. First I will talk and show our plan for the project we chose, and then for the other one which I did a simple demo for.
+
+### Project idea we chose: Real-time fraud analysis using graph data and graph database
+
+My lab mate said he trusts me to choose the technologies for our project and to set up a plan to follow:
+
+Training Pipeline
+
+Data Collection and Preprocessing:
+
+* Raw dataset obtained from Kaggle containing transaction data.
+* Raw data is preprocessed before going into the database.
+
+Storage in ArangoDB:
+
+* Transactions are stored in ArangoDB, a popular graph database.
+* Customers and merchants are represented as nodes, and transactions as edges.
+
+Graph Neural Network with PyG:
+
+* PyTorch Geometric (PyG) is used to build and train a Graph Neural Network (GNN) for fraud detection on the graph data.
+* The model is trained on a batch of the dataset and saved for inference.
+
+Real-time inference pipeline
+
+Simulated Data Streaming:
+
+* Emulating a live transaction environment with continuous data flow using the downloaded full Kaggle dataset.
+
+Data Ingestion with Kafka:
+
+* Kafka handles the real-time streaming of transaction data, ensuring efficient data flow and availability.
+
+Storage in ArangoDB:
+
+* Real-time transactions are stored in ArangoDB, maintaining the graph structure.
+
+Real-Time Fraud Detection:
+
+* The trained GNN model analyses new transactions in real-time as they are streamed and stored.
+* The model uses the saved GNN model to perform real-time inference and detect fraudulent transactions.
+
+![image](https://github.com/user-attachments/assets/43096804-d041-4b5f-8149-882228524952)
+
+Also in excel, I used a project timeline template to create:
+
+![image](https://github.com/user-attachments/assets/7ab5c7ce-772b-4c23-b25f-41d296572995)
+
+The competition started on Wednesday and submission deadline is 11th of Aug, so we are starting from today. The first task is finding a decent dataset with a decent amount of transactions.
+
+One of the new things for me is using a graph db. I saw there is neo4j nd this ArangoDB, I decided to use ArangoDB because the python setup was easier. From playing with it, I included sample transactions and the db webapp (similar to postgres' pgAdmin) and got a limited (because there are too many nodes and edges) visualisation:
+
+![image](https://github.com/user-attachments/assets/653d8221-dbe7-4227-bebf-6a983a18aeb5)
+
+I am really looking forward to this project and working with my lab mate - Jae-hyeok Choi.  
+
+
+### As for the idea that we dismissed - banking voice assistant
+
+I was up for both, and my lab mate preferred the first one, and now looking back - I am glad haha because it is so exciting.
+
+Nevertheless, to set up a voice assistant demo was surprisingly easy. I set it up using huggingface spaces. [Here is the link.](https://huggingface.co/spaces/divakaivan/korean_voice_assistant)
+
+![image](https://github.com/user-attachments/assets/561e49d5-da80-441f-8b43-206282ad6d25)
+
+The above ~30 lines of code take voice -> turn it into text -> a language model answers the text -> the answer is transformed to speech and returned to the user. To try the demo, one needs an OpenAI api key. In order for this simple voice assistant to a bank once, I need to use the LM for a RAG app that talks to a db. Given I made [db2chat](https://github.com/divakaivan/db2chat) it should not be that difficult. But this is a project for later. 
+
+
+That is all for today!
+
+See you tomorrow :) 

From 15714590921d8ec205cf4205f99e3e36d04d67d5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:46:53 +0900
Subject: [PATCH 309/638] Update 2024-07-20-day-201.md

---
 _posts/2024-07-20-day-201.md | 44 ++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-20-day-201.md b/_posts/2024-07-20-day-201.md
index cadf548b6..d3c3743e9 100644
--- a/_posts/2024-07-20-day-201.md
+++ b/_posts/2024-07-20-day-201.md
@@ -1,5 +1,45 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 201) Struggling with neo4j and a fraud GNN
+categories: [gnn,applying-knowledge,mlops]
 ---
+
+## Hello :) Today is Day 201!
+A quick summary of today:
+* started using neo4j as a graph database
+
+It is ~3.30am when I am starting to write this as I was occupied with neo4j and trying to get a GNN model to work so I will keep it brief and go to bed.
+
+Why did we switch from arangoDB to neo4j? 
+Neo4j has a free cloud version with up to 400,000 edges so we can use this one for our real-time inference pipeline. Whereas in arangoDB there is no *permanent* free version. There *is* a free one but it expires after 14 days, and if anything I'd like to keep the data in the cloud db just in case we need it. 
+
+Inserting data into neo4j
+
+[This](https://www.kaggle.com/datasets/kartik2112/fraud-detection) is the dataset we will go with for now. It has a lot of info about the credit card user, transaction, and also we can extract location info about the merchant.
+
+I set up neo4j locally and inserted some data (inserting was easy and fast).
+
+![image](https://github.com/user-attachments/assets/307099e5-d047-4e24-bbb6-623c89560c51)
+
+In the red square, you can see the amount of nodes, and edges. The right side is an sql-esque type of query space but we need to use neo4j's language. 
+This is the query which I used to insert credit card nodes and their attributes:
+
+![image](https://github.com/user-attachments/assets/3ad43df6-3e24-4561-8254-9bc7476b285f)
+
+And for merchants and transactions - it was similar.
+
+
+### Setting up data for a Graph Neural Network
+
+Academia vs reality strikes. During XCS224W: ML with Graphs, I never really had to worry or think about the data, we used good data and we learned about implementing different models. Well today I got struck with the reality of data. Tldr - I was not sure how to create my graph data exactly for some type of GNN. Yes I knew what were the nodes, what are the edges, node and edge indecis and attributes, but whenever I created some Data or HeteroData (as torch geometric calls the two data class options I tried) I did not manage to get a model working with it. Tomorrow I will continue with this. At the moment my idea is: nodes are CreditCard and Merchant, and edge is TRANSACTION which goes only one way: from CreditCard to Merchant. 
+
+That was not the main focus of today, it was about finding a good dataset, which we did and then learning a bit about graph databases. Tomorrow I will try to figure out a way and create even a simple GNN that works.
+
+
+I am working all locally today so there is not any code I can share at the moment. But hopefully in the next days we will have something set up on github.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 818740ea2e41a16913b2bbae816d58f836671d86 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:49:11 +0900
Subject: [PATCH 310/638] Update 2024-07-21-day-202.md

---
 _posts/2024-07-21-day-202.md | 61 ++++++++++++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-21-day-202.md b/_posts/2024-07-21-day-202.md
index cadf548b6..b0f107e7b 100644
--- a/_posts/2024-07-21-day-202.md
+++ b/_posts/2024-07-21-day-202.md
@@ -1,5 +1,62 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 202) Setting up a Graph Convolution Network model to detect fraud credit card transactions
+categories: [applying-knowledge,mlops,gnn]
 ---
+
+## Hello :) Today is Day 202!
+A quick summary of today:
+* finally got a GNN to work
+* learned how to use Mage for data streaming pipelines
+
+I started today where I ended yesterday - trying to create some kind of a graph neural network to predict whether a transaction is fraud or not. 
+
+Tldr (as it is ~3.20am, another late night)
+
+I ended up using torch geometric's Homogeneous data class and the resulting data looks something like:
+
+`Data(x=[4290, 18], edge_index=[2, 23278], y=[4290], train_mask=[4290], test_mask=[4290])`
+
+The preprocessing involves undersampling the majority class and we end up with a balanced dataset. 
+
+The dataset has the following amount of edges and nodes
+
+![image](https://github.com/user-attachments/assets/50633574-4945-4249-b27b-377ed7dee8ad)
+
+Neo4j is nice. 
+
+The model I found that works (at least for now, version 0.1) is:
+
+![image](https://github.com/user-attachments/assets/9afe2f24-5aca-435e-8ad0-5567fae25beb)
+
+After splitting data into train and test, the best model so far achieved the following results:
+
+![image](https://github.com/user-attachments/assets/41e3230a-a7f4-4d8d-b179-244ed917f461)
+
+`Accuracy: 0.8833, Precision: 0.8151, Recall: 0.9909, F1: 0.8945`
+
+![image](https://github.com/user-attachments/assets/63958f89-28f7-41e4-be71-9e111e5149bc)
+
+Today I experimented with creating the training pipeline, but nothing is final yet. These are just experiments.
+
+
+On another note, after getting a model to work - I decided to check out mage's streaming pipelines.
+
+I set up kafka services in docker compose, and with a python script:
+
+![image](https://github.com/user-attachments/assets/1a0d61c3-3a19-425a-9753-c73366aa1138)
+
+I started sending sample data to try to access it through mage.
+On the mage side it is quite simple: using a Kafka data_loader block and a python transformer block to read the messages from the kafka stream
+
+![image](https://github.com/user-attachments/assets/17f49f80-cfaa-47a3-ab83-8d718f106ad2)
+
+When I start running this pipeline we see:
+
+![image](https://github.com/user-attachments/assets/96bb5af6-f98b-4639-9e0b-4d469906c0c1)
+
+Nice ^^ at least I know we can use mage for the real-time inference pipeline.
+
+That is all for today!
+
+See you tomorrow :)

From c0ad3bab58c45d246861c36d5c93909192aa2724 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:54:20 +0900
Subject: [PATCH 311/638] Update 2024-07-22-day-203.md

---
 _posts/2024-07-22-day-203.md | 219 ++++++++++++++++++++++++++++++++++-
 1 file changed, 217 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-22-day-203.md b/_posts/2024-07-22-day-203.md
index cadf548b6..078bd91ed 100644
--- a/_posts/2024-07-22-day-203.md
+++ b/_posts/2024-07-22-day-203.md
@@ -1,5 +1,220 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 203) Starting LLM zoomcamp module 4 - Monitoring
+categories: [nlp,applying-knowledge]
 ---
+
+## Hello :) Today is Day 203!
+A quick summary of today:
+* meeting with my lab mate for the KB AI competition project
+* starting LLM zoomcamp module 4
+
+Today we met to talk about neo4j and doing EDA for the Kukmin Bank AI competition. We talked about the benefits of docker and docker-compose, and how to use neo4j. The next step we will do is EDA. This might sound a bit odd, given that yesterday I created a GNN. Then I just use raw data without many features. The reason being I wanted to make sure I can get even a simple GNN to work.
+
+ My lab mate suggested to split the columns of the dataset in half, and to see what we can find in terms of needing to change in the dataset. In total there are 22 columns, so we split it down the middle. 
+
+These are the columns assigned to myself: 
+`trans_date_trans_time,cc_num,merchant,category,amt,first,last,gender,street,city,state,zip,lat,long,city_pop,job,dob,trans_num,unix_time,merch_lat,merch_long,is_fraud `
+
+### On another note, today I covered about 90% of [LLM zoomcamp](https://github.com/DataTalksClub/llm-zoomcamp)'s 4th module. Below is a summary
+
+We started by loading a document where each item contains a question, from which course it is, from which document it is, original answer, and answer from the llm. (the answer_llm is the newly added).
+
+![image](https://github.com/user-attachments/assets/92fab6e8-c579-4023-8b58-630da89a0509)
+
+To get an LLM answer for each doc (total docs were ~1800) we used gpt3.5-turbo, gpt4o and gpt4o-mini to compare. Out of them, the fastest was gpt3.5-turbo running just for a few minutes, while the longest was gpt4o which took ~2 hours. gpt4o cost 9.79 dollars. gpt3.5-turbo was $0.79
+
+Here we can see a comparison of the distribution of cosine similarity between original answer and llm answer:
+
+![image](https://github.com/user-attachments/assets/8647e6ca-b5c6-4f86-9945-042b0b43e238)
+
+the recently released gpt4o-mini which is cheaper than even gpt3.5, has the following results:
+
+it cost 0.28$ for a smarter model compared to gpt3.5. and it is similar to gpt4o
+
+![image](https://github.com/user-attachments/assets/556513b3-169a-4f42-b455-62e1e25d020a)
+
+4o-mini seems similar to 4o and is way cheaper.
+The next topic was about using an LLM as a judge for answers
+
+We developed two prompts:
+
+Two prompts:
+
+1. When we have the original answer 
+
+```
+
+You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.
+
+Your task is to analyze the relevance of the generated answer compared to the original answer provided.
+
+Based on the relevance and similarity of the generated answer to the original answer, you will classify
+
+it as "NON_RELEVANT", "PARTLY_RELEVANT", or "RELEVANT".
+
+
+
+Here is the data for evaluation:
+
+
+
+Original Answer: {answer_orig}
+
+Generated Question: {question}
+
+Generated Answer: {answer_llm}
+
+
+
+Please analyze the content and context of the generated answer in relation to the original
+
+answer and provide your evaluation in parsable JSON without using code blocks:
+
+
+
+{{
+
+  "Relevance": "NON_RELEVANT" | "PARTLY_RELEVANT" | "RELEVANT",
+
+  "Explanation": "[Provide a brief explanation for your evaluation]"
+
+}}
+
+```
+
+and
+
+
+
+2. when we want to do online (in prod-like) evaluation
+
+```
+
+You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.
+
+Your task is to analyze the relevance of the generated answer to the given question.
+
+Based on the relevance of the generated answer, you will classify it
+
+as "NON_RELEVANT", "PARTLY_RELEVANT", or "RELEVANT".
+
+
+
+Here is the data for evaluation:
+
+
+
+Question: {question}
+
+Generated Answer: {answer_llm}
+
+
+
+Please analyze the content and context of the generated answer in relation to the question
+
+and provide your evaluation in parsable JSON without using code blocks:
+
+
+
+{{
+
+  "Relevance": "NON_RELEVANT" | "PARTLY_RELEVANT" | "RELEVANT",
+
+  "Explanation": "[Provide a brief explanation for your evaluation]"
+
+}}
+
+```
+
+
+
+Here is one example case with included actual question/answer:
+
+```
+
+You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.
+
+Your task is to analyze the relevance of the generated answer compared to the original answer provided.
+
+Based on the relevance and similarity of the generated answer to the original answer, you will classify
+
+it as "NON_RELEVANT", "PARTLY_RELEVANT", or "RELEVANT".
+
+
+
+Here is the data for evaluation:
+
+
+
+Original Answer: Scikit-learn offers another way: precision_recall_fscore_support
+
+Example:
+
+from sklearn.metrics import precision_recall_fscore_support
+
+precision, recall, fscore, support = precision_recall_fscore_support(y_val, y_val_pred, zero_division=0)
+
+(Gopakumar Gopinathan)
+
+Generated Question: What is the syntax for using precision_recall_fscore_support in Python?
+
+Generated Answer: The syntax for using `precision_recall_fscore_support` in Python is as follows:
+
+
+from sklearn.metrics import precision_recall_fscore_support
+
+precision, recall, fscore, support = precision_recall_fscore_support(y_val, y_val_pred, zero_division=0)
+
+
+Please analyze the content and context of the generated answer in relation to the original
+
+answer and provide your evaluation in parsable JSON without using code blocks:
+
+
+
+{
+
+  "Relevance": "NON_RELEVANT" | "PARTLY_RELEVANT" | "RELEVANT",
+
+  "Explanation": "[Provide a brief explanation for your evaluation]"
+
+}
+
+```
+
+and gpt’s response:
+
+![image](https://github.com/user-attachments/assets/da1d90c5-fb34-455d-98bc-fbb1b1317be0)
+
+Prompt engineering is important here. Because GPT can sometimes return data not in the format we need, so finding a prompt that gets results with a consistent format is key.
+
+We can turn the evaluations to a df for further analysis and visualisations
+
+![image](https://github.com/user-attachments/assets/91fbfb96-cfae-4ddc-8bae-cbd47e9fb567)
+
+Example NON_RELEVANT case
+
+![image](https://github.com/user-attachments/assets/bb19cc55-f51e-4c2e-804f-a3c302d34b04)
+
+because answer_llm does not answer the question and is not close to the original answer
+
+
+Then we set up a simple chat bot using streamlit:
+
+![image](https://github.com/user-attachments/assets/990231f3-e117-4c43-9110-34f1c233dc0f)
+
+And the data is saved into a db for further analysis and monitoring
+
+![image](https://github.com/user-attachments/assets/d35521c1-a8b9-41d9-8506-4cd061f68d24)
+
+and the feedback (+1 or -1 in streamlit) table has:
+
+![image](https://github.com/user-attachments/assets/a5dbe4b6-1fc2-4835-90a0-14190a9ca442)
+
+The last bit is related to grafana. But I did not get to cover that today.
+
+
+That is all for today!
+
+See you tomorrow :) 

From 5e9cfa76d6ec40b38c8bbbb1b5640e7eb0d340e8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 22:58:14 +0900
Subject: [PATCH 312/638] Update 2024-07-23-day-204.md

---
 _posts/2024-07-23-day-204.md | 73 +++++++++++++++++++++++++++++++++++-
 1 file changed, 71 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-23-day-204.md b/_posts/2024-07-23-day-204.md
index cadf548b6..afaa42660 100644
--- a/_posts/2024-07-23-day-204.md
+++ b/_posts/2024-07-23-day-204.md
@@ -1,5 +1,74 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 204) Transaction data EDA + MLflow & minIO docker setup
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 204!
+A quick summary of today:
+* some EDA on the KB AI competition data
+* setting up mlflow and minIO
+
+### Firstly, about doing some basic cleaning and EDA on my part of the data for the Kukmin Bank project
+
+These are the variables assigned to myself: `trans_date_trans_time,cc_num,merchant,category,amt,first,last,gender,street,city,state`
+
+![image](https://github.com/user-attachments/assets/3d883bd3-e5ae-4a2a-819e-462843592057)
+
+No missing/null values. 
+
+Box plot of log(amount) in Not Fraud vs Fraud
+
+![image](https://github.com/user-attachments/assets/e4177149-a1d2-4b20-915c-8e271fedfaed)
+
+Distribution of Not Fraud vs Fraud
+
+![image](https://github.com/user-attachments/assets/d84c9815-e476-435a-883e-bc1b49a34739)
+
+Other graphs
+
+![image](https://github.com/user-attachments/assets/8d079c3e-a1fb-4022-af85-c3e775d2bdf4)
+![image](https://github.com/user-attachments/assets/661733ed-3fe7-47ac-82db-73c5ad4ae28c)
+![image](https://github.com/user-attachments/assets/4aa61734-5c03-4d80-9a81-0b2773d5150c)
+![image](https://github.com/user-attachments/assets/3f6dfc90-c032-4db7-84a5-cfa065dbaa3e)
+![image](https://github.com/user-attachments/assets/5c737411-7671-4108-83eb-8a7698c9453f)
+![image](https://github.com/user-attachments/assets/6189aaa4-7cf7-4c17-9308-7a1cc580bf3e)
+
+Interesting ~ only Fraud transactions in the state of Delaware (of course this is not real data, but interesting nonetheless)
+
+We should also do some basic transformation and cleanup before the raw data goes to the db. In my columns, an example case is: all merchant's name start with fraud_, so removing it would be fine, just for a bit more clarity. 
+
+
+### On another note ~ mlflow and minIO
+
+I found [this](https://mlops-for-mle.github.io/tutorial/docs/model-registry/mlflow-setup) website (in Korean but can be translated) that provides an easy "plug-n-play" Dockerfile and docker-compose serivces code for setting up mlflow, and minIO as an artifact and backend store. I have never used minIO before but from a quick online search (before using it) it seemed like a UI similar to any cloud provider's storage, and is built on AWS so it seems that it is scalable too.
+
+Following the guide I set up the Dockerfile and the services for mlflow-backend-store
+
+![image](https://github.com/user-attachments/assets/c485a51d-dbf9-4305-a9e4-e315ab834396)
+
+mlflow-artifact-store
+
+![image](https://github.com/user-attachments/assets/14b3aeee-1dd6-4459-8737-56a3fad3c1ed)
+
+mlflow-server
+
+![image](https://github.com/user-attachments/assets/89502d07-5e74-4ebf-bdcc-28d5cd1df31e)
+
+ran it and it works fine. This was the 1st time I saw the UI of minIO
+
+![image](https://github.com/user-attachments/assets/194a3cec-2720-4786-b559-bee451b11a25)
+
+It is empty for now, but reminds me of AWS S3 and GCS
+
+From the setup, there is already a created bucket as well:
+
+![image](https://github.com/user-attachments/assets/d7a1f383-95ae-415e-bdd5-67704e090cb6)
+
+Another thing related to kafka. For some reason the setup I had on this project was using some schema-registry image which was ~1.5GB and taking lots of my space in Docker, and it stopped working, so I switched the services in my docker-compose file with the same as from my [transaction-stream-data-pipeline project](https://github.com/divakaivan/transaction-stream-data-pipeline). At the moment my docker containers are:
+
+![image](https://github.com/user-attachments/assets/ef0148ff-f57c-4ca4-ad89-651eb6e3372d)
+
+That is all for today!
+
+See you tomorrow :)

From 4a51a471187e646343b63236bf6f4f3d3098f9a8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:00:50 +0900
Subject: [PATCH 313/638] Update 2024-07-24-day-205.md

---
 _posts/2024-07-24-day-205.md | 47 ++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-24-day-205.md b/_posts/2024-07-24-day-205.md
index cadf548b6..603416ad6 100644
--- a/_posts/2024-07-24-day-205.md
+++ b/_posts/2024-07-24-day-205.md
@@ -1,5 +1,48 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 205) Going back to a basic mlflow service and another meeting for the KB project
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 205!
+A quick summary of today:
+* no minIO, just plain old postgres and local storage for mlflow
+* helping my friend better understand github's workflow
+
+Yesterday I mentioned I set up minIO as an artifact store for mlflow ~ 
+
+Well today I removed it haha. The problem was that i wanted some extra AWS credentials in order to work, and to get that I need a working AWS account. I can easilty set that up again (because I deleted my aws account recently), but in order for my project collaborator to set it up, he needs to do it as well. We have enough work as it is, and this extra tech is not essential so I just removed it and set up mlflow just using a local postgres docker service and a folder as the artifact store. Maybe later, we can swap out this whole mlflow to run on GCP but it is not crucial if we leave it as is for now. 
+
+While setting up mlflow and testing the connection, I was reminded of the value of "network" in dockere-compose. I had the other services using a network called 'bridge', but when I set up mlflow I did not add the network to the docker compose service like:
+
+![image](https://github.com/user-attachments/assets/d7170129-0e16-4d16-bd92-64df7600e545)
+
+So within Mage, I could not access mlflow. I thoyght it had something to do with how I set up the ports, and after about 30 mins of trying to fix it, I saw that networks: bridge is missing from the mlflow-server service.
+
+### On another note ~ we had another Kukmin Bank (KB) project meeting
+
+We shared what we learned from the data and doing some visualisations. I shared mine in yesterday's post, but as for his - they are local on his machine, but I will share a link to them when we get them on github.
+
+Talking about GitHub, we set that up too. I also got the chance to help my team partner with his understanding of the git workflow - commits, pushes, fetches, pulls, branches, merge conflicts. I was reminded of my days in Lloyds Banking Group in 2020 when back then I was helping team members learn about GitHub as well. 
+
+We set up this repo:
+
+![image](https://github.com/user-attachments/assets/1edb4752-eebd-488a-afce-630c64591612)
+
+And just practiced commit, push, fetch, pull, switching branches. The repo is private at the moment, but we will make it public once we submit it to the competition. 
+
+As for next steps, on Friday we will meet again. Till then our tasks are as follows:
+
+* create a script to do basic data transformation, upload data to neo4j, and extract as well - my task
+* translate his visualisations written in R to python and put all visualisations (his and mine) in one notebook so it can live in the repo under a folder called EDA
+
+
+### Another thing ~ I saw Meta released a 405B param model ?!
+
+405 billion ?! I checked on huggingface and its files are near 1TB. But that is big for open source AI as they claim it can compete with the best closed source models like GPT4o
+
+
+
+That is all for today!
+
+See you tomorrow :)

From d9145d71d31bb3ce8fde026f8ac58f669792d35c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:04:05 +0900
Subject: [PATCH 314/638] Update 2024-07-25-day-206.md

---
 _posts/2024-07-25-day-206.md | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-25-day-206.md b/_posts/2024-07-25-day-206.md
index cadf548b6..4ef60c727 100644
--- a/_posts/2024-07-25-day-206.md
+++ b/_posts/2024-07-25-day-206.md
@@ -1,5 +1,33 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 206) Finishing the Stock Market Analysis zoomcamp (for now)
+categories: [applying-knowledge]
 ---
+
+## Hello :) Today is Day 206!
+A quick summary of today:
+* last homework from Stock Market Analysis Zoomcamp
+* Uploading neo4j scripts to the KB project repo
+
+Over the past 4 weeks, my lab mate (Jae-Hyeok, the same guy with whom I am doing the KB project) and I have been covering another couse by DataTalksClub - the [Stock Market Analysis Zoomcamp](https://github.com/DataTalksClub/stock-markets-analytics-zoomcamp). The course introduces basic concepts and strategies for creating models that can potentially invest in stocks. Today was the 4th week since we started it and was the last homework, related to working with a financial model workbook to modify and analyse stocks, focusing on Random Forest tuning, reducing feature sets (to see difference in results), predicting strong future growth (investing in cases where the stock growth is above a fixed treshold), and developing an ideal trading strategy.
+
+![image](https://github.com/user-attachments/assets/cfcad00e-7c55-4497-8c8e-4b8404b548d1)
+
+Above is an example resulting graph from one of the questions. We compare models based on CAGR (Compound annual growth rate), and there are various models - random forest, decision trees, a basic rule of thumb model with CCI > 200.
+
+Given this is the last homework (noting there is a 5th video but no homework, related to deployment) we ought be thinking about a project. However given that we have to concentrate on our Kukmin Bank AI competition project we will delay this until after we submit our competition project (11th of Aug).
+
+### On another note ~ the KB project
+
+Tomorrow is our final meeting for this week (as I am writing I realised we met every day this week haha), and we will discuss next steps, as next week I am going on holiday and what we can do separately during that time. 
+
+The plan was for me to set up database interaction scripts (upload to and extract from neo4j), so that while I am away JH (JaeHyeok for short) can use the data to create different models that try to predict fraudulent transactions. I did this today, and I uploaded them to out GitHub (the repo is private for now, until we submit it for the competition). 
+
+![image](https://github.com/user-attachments/assets/3ca3e0fb-6d06-4210-812c-2ba95d439300)
+
+The scripts upload data into neo4j, creating CreditCard and Merchant nodes, and using Transaction as the edge going from a CreditCard to a Marchant. 
+
+
+That is all for today!
+
+See you tomorrow :) 

From a54d7a149cff3fd82e9abd3afd2b43af513abefe Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:05:58 +0900
Subject: [PATCH 315/638] Update 2024-07-26-day-207.md

---
 _posts/2024-07-26-day-207.md | 37 ++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-26-day-207.md b/_posts/2024-07-26-day-207.md
index cadf548b6..1516fb92c 100644
--- a/_posts/2024-07-26-day-207.md
+++ b/_posts/2024-07-26-day-207.md
@@ -1,5 +1,38 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 207) Finished with neo4j (for now) and thinking about fraud detection models
+categories: [reading-research,applying-knowledge]
 ---
+
+## Hello :) Today is Day 207!
+A quick summary of today:
+* setting up neo4j
+* reading some papers on bank telemarketing classification
+
+Firstly, about the Kukmin Bank (KB) AI competition project
+We set up  the database on my partner's laptop
+
+* insert CreditCard nodes
+* insert Merchant nodes
+* insert Transaction edge
+
+We also had a look at the final EDA notebook.
+
+At the moment the repo looks like:
+
+![image](https://github.com/user-attachments/assets/43d88395-c313-4f03-a3ab-855e5bd55e38)
+
+As for next steps, we will start developing models, and try to get something that can identify Fraud transactions well. And more importantly, explainability - what features help the model determine that a transaction is fraud. So we will start with some basic models like logistic reg, decision trees, random forest, then add hyperparam tuning, over/undersampling, etc. And try to get a model that detects Fraud well. 
+
+As I want to try out a GNN for this project's model. I saw there is a GNNExplainer by torch-geometric, so I need to try using it in practice and see if the explainability it provides is good enough for our project. 
+
+### Secondly, reading papers on bank telemarketing classification
+
+![image](https://github.com/user-attachments/assets/f6e68e0d-fce9-46d7-9308-5f06507da6ff)
+
+I put 5 papers that use Portuguese Bank telemarketing data and approach it from different perspectives. Tomorrow I will present it to a professor that is advising this paper idea by one of my lab mates. 
+
+
+That is all for today!
+
+See you tomorrow :)

From b3979ebdd812a238e687c2c4c0f7b8d81d0d8fd5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:08:49 +0900
Subject: [PATCH 316/638] Update 2024-07-27-day-208.md

---
 _posts/2024-07-27-day-208.md | 53 ++++++++++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-27-day-208.md b/_posts/2024-07-27-day-208.md
index cadf548b6..c7dd6368a 100644
--- a/_posts/2024-07-27-day-208.md
+++ b/_posts/2024-07-27-day-208.md
@@ -1,5 +1,54 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 208) Setting up docker-services for the KB project, streaming transactions, and the Scottish dataset
+categories: [mlops,glaswegian,applying-knowledge]
 ---
+
+## Hello :) Today is Day 208!
+A quick summary of today:
+* making progress with the Kukmin Bank (KB) project
+* Scottish dataset 2hr audio mark is close
+
+### First, about the KB project for the KB AI competition
+
+All code from today is on the project's repo on branch ivan.
+
+Today I managed to set up the (what I think are the) needed services going forward.
+
+![image](https://github.com/user-attachments/assets/c88c7066-c700-4dfd-8540-8dcc3753e022)
+
+I hit the Docker storage space limit a few times, so I had to compromise on complexixty. For example instead of using postgres as a db for mlflow, even using a simple sqlite3 db for this project is fine. Later, plugging in a better storage for mlflow, and its artifacts is trivial.
+
+One new thing, that was not there before is grafana. I saw online that there is a neo4j grafana plugin - so I tried to install it. At first, by default, I could not, so I looked in grafana cloud (free for 14 days) and on there, the plugin could easily be loaded, and I could easily connect to my neo4j db. However, I wanted to do it locally, so I found a way to install the plugin directly in my docker-compose grafana service definition, like: 
+
+![image](https://github.com/user-attachments/assets/65981b52-b46e-4d1d-a1f3-2339c5a049f5)
+
+This long url leads to the [neo4j plugin](https://github.com/denniskniep/grafana-datasource-plugin-neo4j/releases/tag/v1.3.2) download which is automatically downloaded. So with this, I could connect to my docker hosted neo4j db. 
+
+Next, I decided to try to set up the pseudo streaming script that will emulate transactions occurring. 
+
+![image](https://github.com/user-attachments/assets/236ce18e-4d7d-4bb6-8d35-cd60ea28108c)
+
+Every second, it sends 5 transactions to the kafka stream in the form of dictionaries. 
+
+Its Dockerfile:
+
+![image](https://github.com/user-attachments/assets/7681618e-5c36-4b87-9cd6-610ac3ad0e7c)
+
+And requirements.txt
+
+![image](https://github.com/user-attachments/assets/ef10d756-b107-4ccb-b939-c0fc74b43cc0)
+
+For reproducability, I will try to add version to every package being used, just to ensure that when the project code is in the hands of the judges, if they want, they can reproduce everything. 
+
+Next, I will try to set up Mage (orchestration) a bit more so we have defined pipelines, and more importantly - working pipelines for training a GCN, and using that model for real-time streaming predictions. 
+
+### Second, about the Scottish dataset
+
+I have two scripts, the audio of which I need to cut into clips. Along with these we will get to that 2 hour mark which we have been aiming for. [Here](https://huggingface.co/datasets/divakaivan/glaswegian_audio) is the current version, but I hope over the next week to preprocess the audio and get the full dataset on huggingface. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 591e8f622842af7e065ea7352f04e90a958cbbc2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:13:04 +0900
Subject: [PATCH 317/638] Update 2024-07-28-day-209.md

---
 _posts/2024-07-28-day-209.md | 89 +++++++++++++++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-28-day-209.md b/_posts/2024-07-28-day-209.md
index cadf548b6..abfe6c210 100644
--- a/_posts/2024-07-28-day-209.md
+++ b/_posts/2024-07-28-day-209.md
@@ -1,5 +1,90 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 209) Using Mage for pipeline orchestration in the KB project
+categories: [mlops,gnn,applying-knowledge]
 ---
+
+## Hello :) Today is Day 209!
+A quick summary of today:
+* creating Mage pipelines for the KB AI competition project
+
+The repo after today
+
+![image](https://github.com/user-attachments/assets/d4c78ff5-e86c-4a13-b2df-af6c49c0fdd7)
+
+Today morning/afternoon I went on a bit of a roll ~ 
+
+![image](https://github.com/user-attachments/assets/f521a12c-c8bb-455e-b323-a1d5cacd41f7)
+
+I set up all the above pipelines in Mage. Below I will go over each one
+
+get_kaggle_data
+
+![image](https://github.com/user-attachments/assets/ba0f52b1-91a3-4f56-a552-f08484eeccb6)
+
+it is just one block that downloads the data for the project from [Kaggle](https://www.kaggle.com/datasets/kartik2112/fraud-detection) using the kaggle python package
+
+load_batch_into_neo4j
+
+![image](https://github.com/user-attachments/assets/a80d9c66-59af-4ddd-be7e-4b40d813f9a6)
+
+Gets the loaded data from the get_kaggle_data pipeline and inserts it into neo4j. This is the fraudTrain.csv from the Kaggle website (because the fraudTest.csv will be used for the pseudo-streaming pipeline).
+
+train_gcn
+
+![image](https://github.com/user-attachments/assets/8ac1e2a6-9861-4288-93a9-31952a94df45)
+
+I tried to split this into more blocks, but at the moment the way I structured the code, the most optimal solution was to do it all at once. That is - create a torch-geometric dataset, create node and edge index, train the model, test it, and save summary info. At the moment, because everything is local, I am using mlflow just for easy comparison but later (after the project submission) I will add a proper mlflow server with a cloud db and artifact store where I can save models, and from where I can load models and other data. 
+
+When this pipe is run, locally and in mlflow such folder is created:
+
+![image](https://github.com/user-attachments/assets/f1feb992-3787-41bd-a91f-b96a5975b1b2)
+
+Here are the images/text from this model run:
+
+Confusion matrix:
+
+![image](https://github.com/user-attachments/assets/0ae1f7b6-17e5-40f7-bc02-800515e3d864)
+
+Feature importance (I need to modify this because besides 5 of the vars, the other are categorical dummies and I need to add a suffix to make it clear):
+
+![image](https://github.com/user-attachments/assets/ba2a3141-c126-47b4-a5d9-8cdabe3a251d)
+
+Accuracy: 0.8809, Precision: 0.8133, Recall: 0.9883, F1: 0.8923
+
+Model summary:
+
+![image](https://github.com/user-attachments/assets/d3d66a04-0642-45a4-9927-ea8fa9749272)
+
+stream_predictions
+
+For this pipeline to work I needed to set up a kafka producer to send data to the kafka stream. I created this and shared it in yesterday's post. 
+
+![image](https://github.com/user-attachments/assets/ca2c5726-d45a-46c7-a8f6-c96d21c6be55)
+
+Mage has a nice setup for streaming pipelines. The 1st block establishes a connect to read data from the kafka stream (to which I am sending transactions). In the 2nd block, using the data from the stream, I create a Data object that can be used as input to the trained GCN, and add it to the message:
+
+![image](https://github.com/user-attachments/assets/88bcff5d-b987-4619-8e43-ca3c1193d6c4)
+
+And in the final block - insert the new data to neo4j. 
+
+
+Another thing - creating a simple model dictionary using streamlit
+
+![image](https://github.com/user-attachments/assets/89c1593d-3eb4-4757-b636-676bfedf7741)
+
+To create it I added a new option to the Makefile. Yesterday I thought I would use docker to create a service, but because I am not using mlflow with a proper db and artifact store, if I want to load images/text (as the above confusion matrix, model summary, etc.) I need to upload those files to the streamlit docker image as well, and this seems unnecessary. Once mlflow is set up properly then using streamlit from docker might be a viable choice. 
+
+As for how the streamlit model dictionary looks:
+
+![image](https://github.com/user-attachments/assets/03dfb635-9678-4ce3-be1a-34a09080e0b4)
+
+it includes all the model info that is produced from the train_gcn pipeline. 
+And once we have other models, their info can easily be added. 
+
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 2a4ecd70e50ddcf047a8c5d544777076d6be0ab8 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:18:30 +0900
Subject: [PATCH 318/638] Update 2024-07-29-day-210.md

---
 _posts/2024-07-29-day-210.md | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-29-day-210.md b/_posts/2024-07-29-day-210.md
index cadf548b6..8a6e0361e 100644
--- a/_posts/2024-07-29-day-210.md
+++ b/_posts/2024-07-29-day-210.md
@@ -1,5 +1,29 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 210) 118 minutes of Glaswegian accent audio clips
+categories: [glaswegian,applying-knowledge]
 ---
+
+## Hello :) Today is Day 210!
+A quick summary of today:
+* final audio clips preprocessing to reach our audio dataset mark
+
+![image](https://github.com/user-attachments/assets/5b167a4c-0a29-4fca-97f9-ba01d5121171)
+
+Final dataset for the glaswegian voice assistant AI ([link to HuggingFace](https://huggingface.co/datasets/divakaivan/glaswegian_audio)). Today I preprocessed the final audios from 2 of Limmy's youtube videos (Limmy accidentaly kills the city and The writer of Saw called Limmy a ...). 
+Just an update on how the process goes now ~ 
+
+Since our transcription AI is pretty good (according to my Glaswegian speaking project partner), we pass the full raw audio to our [fine-tuned whisper](https://huggingface.co/spaces/divakaivan/glaswegian-whisper) model hosten on HuggingFace spaces. Then the transcript is put into a docs file (where first I check over it for obious mistakes and flag if I see something odd and cannot understand it from re-listening to the audio) and split into sensible (small) bits while listening to the audio, like:
+
+![image](https://github.com/user-attachments/assets/4dc85181-d691-4b04-af57-6aeb3244e094)
+
+(this is the start from Limmy accidentaly kills the city)
+
+Then using an audio tool, I cut the full audio length into clips according to the cut text, then I match clip name and transcript in excel, then using python I get the clip length and sampling rate. Finally I add static info like gender, age, class, location, speaker id to the data, and finally I get a csv which I upload onto the hugging face dataset along with the cut audio clips.
+
+Next steps are to train a final whisper model, and then a Text-To-Speech model using this final dataset. Maybe tomorrow when we go to breakfast I will leave whisper to fine-tune. 
+
+
+That is all for today!
+
+See you tomorrow :)

From 2ba6c5b0592a019c66b4e3d2a81dc50ec0f0dcee Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:20:28 +0900
Subject: [PATCH 319/638] Update 2024-07-30-day-211.md

---
 _posts/2024-07-30-day-211.md | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-30-day-211.md b/_posts/2024-07-30-day-211.md
index cadf548b6..b01e171c0 100644
--- a/_posts/2024-07-30-day-211.md
+++ b/_posts/2024-07-30-day-211.md
@@ -1,5 +1,28 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 211) 2 hour mark !!! Glaswegian dataset goal - accomplished! + whisper-small fine-tuned
+categories: [glaswegian,applying-knowledge]
 ---
+
+## Hello :) Today is Day 211!
+A quick summary of today:
+* fine-tuning the final whisper-small model for Glaswegian  automatic speech recognition (ASR)
+
+After yesterday, it the audio was 118 and I was not happy that we did not *officially* hit the 2 hour mark, so today I went on youtube and got a 3min clip from Limmy which I transcribed and added to our dataset. [Here](https://huggingface.co/datasets/divakaivan/glaswegian_audio) is a link to the final 2 hour Glaswegian dataset on HuggingFace.
+
+After a few months of working on this side project we hit the set goal!~
+
+After I got the dataset I started [fine-tuning](https://huggingface.co/divakaivan/glaswegian-asr) whisper-small. It took around 4 hours ~ and the fine-tuned model is ready and deployed for using on [HuggingFace Spaces](https://huggingface.co/spaces/divakaivan/glaswegian-whisper). 
+
+![image](https://github.com/user-attachments/assets/023c0c10-fd27-4c16-b0bd-652fb50f330b)
+
+Looking at the metrics, training could be improved. 
+
+![image](https://github.com/user-attachments/assets/49902663-190d-4794-ada8-6c790499bc60)
+
+As for what is next - training a Text-To-Speech model using our final dataset. This was the hard part before because the voice at the end was, while understandable, a bit robotic. So this is the next step and hopefully we get better results now. 
+
+
+That is all for today!
+
+See you tomorrow :)

From 1151887ed7b6368ad5a051361dfb12c0856989f4 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:22:31 +0900
Subject: [PATCH 320/638] Update 2024-07-31-day-212.md

---
 _posts/2024-07-31-day-212.md | 37 ++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-07-31-day-212.md b/_posts/2024-07-31-day-212.md
index cadf548b6..7a08dfede 100644
--- a/_posts/2024-07-31-day-212.md
+++ b/_posts/2024-07-31-day-212.md
@@ -1,5 +1,38 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 212) Final Glaswegian TTS model
+categories: [glaswegian,applying-knowledge]
 ---
+
+## Hello :) Today is Day 212!
+A quick summary of today:
+* creating a simple Glaswegian assistant app
+
+![image](https://github.com/user-attachments/assets/b86200e5-f57e-491e-b4c4-20546324b0e2)
+
+After finetuning whisper to get the final version of the Glaswegian ASR model, the next task was to do a final fine-tuning on the T5 speech model to get a final version of the Glaswegian TTS model. Well I did that today. 
+
+[Here](https://huggingface.co/divakaivan/glaswegian_tts) is a link to the model on HuggingFace. And its training results:
+
+![image](https://github.com/user-attachments/assets/a9c612e2-18d3-4f4d-85aa-54b330742c1a)
+
+Now that we have the final 2 hour dataset, I was hoping for better results. Before, the generated audio (while with a little accent) sounded robotic. First thing I had to do was fix the HuggingFace space where the previous version of the glaswegian TTS was running. The issue was related to voice embeddings, and after a quick fix ~
+
+![image](https://github.com/user-attachments/assets/bb34466e-1c76-48f8-8eeb-827ee9bf1cb9)
+
+It was up again, and I loaded the latest glaswegian_tts model. Well now, it *does* sound better. There are cases where it is robotic, but there is definitely improvement compared to the previous version. That previous version was trained or around 30 mins of audio, compared to now 2 hours.
+
+
+Next - create a full assistant app
+
+![image](https://github.com/user-attachments/assets/631fbf4a-ca9a-4e20-a601-f5db7a4d5290)
+
+Gradio and HF spaces make it very easy -> [here](https://huggingface.co/spaces/divakaivan/glaswegian-assistant).
+Audio input -> transcribed using glaswegian_asr -> send to gpt2 -> answer from gpt2 is turned to speech and returned to the user
+At the start I used gpt2, but as its not that good, I switched to using gpt3.5-turbo.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From f54f90aeffb3510c0290110fbf88c4fc8c63b837 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:24:29 +0900
Subject: [PATCH 321/638] Update 2024-08-01-day-213.md

---
 _posts/2024-08-01-day-213.md | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-01-day-213.md b/_posts/2024-08-01-day-213.md
index cadf548b6..e5e5c7cf8 100644
--- a/_posts/2024-08-01-day-213.md
+++ b/_posts/2024-08-01-day-213.md
@@ -1,5 +1,26 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 213) Creating a grafana dashboard for the KB project
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 213!
+A quick summary of today:
+* using Cypher to create a grafana dashboard for my neo4j database
+
+After setting up neo4j's plugin in grafana a few days ago today I decided to create a live dashboard in grafana. The result:
+
+![image](https://github.com/user-attachments/assets/e1ef266e-9f1c-48a0-8937-50bc5f638db0)
+
+Each graph requires using Cypher (neo4j's query language). One cool graph I was surprised I could make is the geography map which uses the long and lat of merchants where there was fraud. What else is cool is that the graph updates as new transactions come in. It can update at different intervals, but I tested the 5 second one, and as new data comes in from kafka and sent to the db, the dashboard reflects it. What is more, this time, I saved the dashboard as a json are uploaded it to github.
+
+As seen from the next pic, besides the graph, the other things I did today are a bit trivial - just updating variable names, clean up data preprocessing and make the Mage pipelines better. 
+
+![image](https://github.com/user-attachments/assets/81767e88-cbd7-476f-b70a-8ed4529a58c1)
+
+The deadline for the project is 11th of Aug, so till then the repo will be kept as private, but after we will make it public and I can share a link.
+
+
+That is all for today!
+
+See you tomorrow :)

From 1b77ec447dd64eb25ce769ab57ebff673ee9564d Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:26:50 +0900
Subject: [PATCH 322/638] Update 2024-08-02-day-214.md

---
 _posts/2024-08-02-day-214.md | 56 ++++++++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-02-day-214.md b/_posts/2024-08-02-day-214.md
index cadf548b6..982a8253d 100644
--- a/_posts/2024-08-02-day-214.md
+++ b/_posts/2024-08-02-day-214.md
@@ -1,5 +1,57 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 214) The evaluation of my MLOps zoomcamp project arrived - max points
+categories: [mlops,data-eng,applying-knowledge]
 ---
+
+## Hello :) Today is Day 214!
+A quick summary of today:
+* documenting the KB project data using arrow.app
+* small repo changes
+* ML zoomcamp announcement
+
+I tried to find some graph documentation tool for the KB project. The most popular I could find is this arrows.app application where we can create sample nodes with edges between and add attributes with descriptions. I created the below pic using that app
+
+![image](https://github.com/user-attachments/assets/dd60ea64-da54-4000-9d77-c7807f45daaa)
+
+Also I made some small changes to the repo:
+
+![image](https://github.com/user-attachments/assets/406b89cf-d12b-432b-ae38-0e20f45d7275)
+
+the readme I added:
+
+* Docker - used to containarise and self-host the below services
+* Mlflow - used for easy model comparison during development. Can be improved by using cloud services (like GCP) for hosting, database and artifact store
+* Mage - used for pipeline orchestration of the model training and real-time inference pipelines. Can be improved by hosting mage on the cloud
+* Neo4j - used as a Graph database to store transaction data as nodes and edges. Can be improved by using Neo4j's AuraDB (hosted on the cloud)
+* Kafka - used to ensure real-time transaction data processing
+* Grafana - used for real-time dashboard creating and monitoring. Can be improved by using Grafana's cloud services
+* Streamlit - used to host a model dictionary showing models, evaluation metrics, and feature importance graphs
+
+Terraform can be used to provide Infrastructure as Code (IaC) for services that can be hosted on the cloud
+
+I also removed .env from gitignore just so that reproducers can clearly see the env variables used. At the moment there is no actual secret variables so it is fine. 
+
+A task I added to my TODOs is to figure out how to setup the neo4j db connection on grafana startup. I saw I could use a grafana.ini file, but I will figure out this tomorrow.
+
+
+
+On another note, today the evaluation of my [MLOps zoomcamp project](https://github.com/divakaivan/insurance-fraud-mlops-pipeline/) came back.
+
+I got 32, which I believe is the max. The points are given for: describing the problem, usage of cloud services, mlflow and exp. tracking, orchestration, model deployment, monitoring, reproducability and best practices. 
+
+The certificate is not issued yet, but I will share it when it is sent. 
+
+
+
+Today I saw that Alexey Grigorev, the creator of the DataTalksClub, announced the next cohort for their ML zoomcamp. It will cover the below:
+
+![image](https://github.com/user-attachments/assets/eab9ea51-5cfb-46e3-b4ec-9ddb3d2adac1)
+![image](https://github.com/user-attachments/assets/a9d0bb33-20e3-4498-93d1-cf0195b17718)
+
+I might sign up to see if I can learn something new, and keep my skills sharp ^^ 
+
+
+That is all for today!
+
+See you tomorrow :)

From 57e93a1cba0ba508940e92b25443f5fd906cae5b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:29:20 +0900
Subject: [PATCH 323/638] Update 2024-08-03-day-215.md

---
 _posts/2024-08-03-day-215.md | 58 ++++++++++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-03-day-215.md b/_posts/2024-08-03-day-215.md
index cadf548b6..98bee2608 100644
--- a/_posts/2024-08-03-day-215.md
+++ b/_posts/2024-08-03-day-215.md
@@ -1,5 +1,59 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 215) Trying out 'traditional' models on the KB project transaction fraud data
+categories: [traditional-machine-learning]
 ---
+
+## Hello :) Today is Day 215!
+A quick summary of today:
+* trying out different models to classify non fraud/fraud transactions
+
+In addition to the Graph Convolutional Network model, I wanted to create some 'traditional' (non-neural net) models for better comparison and judgement. 
+
+The features I used for the below models are:
+
+numerical: amt (amount)
+
+categorical: category, merchant, city, state, job, trans_hour, trans_dow
+
+Then I downsampled the majority class (non fraud cases; just like for the GCN model).
+
+
+
+### Logistic Regression
+
+![image](https://github.com/user-attachments/assets/6b8e63b9-abf4-494d-bf46-a3c72dcbe869)
+
+`Best params: {'C': 1, 'penalty': 'l1', 'solver': 'liblinear'}`
+
+### CatBoost
+
+![image](https://github.com/user-attachments/assets/a2d5447c-ec8e-4d1e-9410-528981c24d37)
+
+`Best params: {'depth': 6, 'iterations': 300, 'learning_rate': 0.3}`
+
+### XGBoost
+
+![image](https://github.com/user-attachments/assets/8da48e61-3e18-4a60-8c94-feaf69b02eb9)
+
+`Best params: {'learning_rate': 0.3, 'max_depth': 6, 'n_estimators': 300, 'subsample': 0.9}`
+
+### RandomForest
+
+![image](https://github.com/user-attachments/assets/d4b11e27-31bb-4e6f-aba8-df62fa495f96)
+
+`Best params: {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 200}`
+
+### LGBMClassifier
+
+![image](https://github.com/user-attachments/assets/e1db284d-0b2a-4d89-b20c-47857bb7be24)
+
+`Best params: {'learning_rate': 0.01, 'max_depth': 10, 'n_estimators': 300, 'num_leaves': 70}`
+
+Among the top, XGBoost and CatBoost seem to perform the best, and for tomorrow I will add them to individual Mage training pipelines, and then in the Mage inference pipeline. The idea is that having different models' predictions on a transaction can help with decision making and also that kind of info can be shown in Grafana as well. What is more from these models, we can get feature importance, and even SHAP values to better understand their decision making.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From bae80fa051ef993c3472c82e07e1ff9bcc584f23 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:41:15 +0900
Subject: [PATCH 324/638] Update 2024-08-04-day-216.md

---
 _posts/2024-08-04-day-216.md | 73 +++++++++++++++++++++++++++++++++++-
 1 file changed, 71 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-04-day-216.md b/_posts/2024-08-04-day-216.md
index cadf548b6..622524bd6 100644
--- a/_posts/2024-08-04-day-216.md
+++ b/_posts/2024-08-04-day-216.md
@@ -1,5 +1,74 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 216) Pipelines for XGBoost and CatBoost training, and using the models in the real-time inference pipeline
+categories: [mlops,applying-knowledge]
 ---
+
+## Hello :) Today is Day 216!
+A quick summary of today:
+* created pipelines to train XGBoost and CatBoost models
+* added XGBoost and Catboost models' predictions in the real-time inference pipeline
+* update model dictionary UI
+* created project README
+
+The project submission deadline is 11th of Aug, and after that we will make the repo public, but until then I just have to share pictures from the project. 
+
+All pipelines including today's:
+
+![image](https://github.com/user-attachments/assets/8f3d4fd9-15a6-4316-85a1-d0960f7ca4a0)
+
+Setting up model training pipes was easy, the tricky part was using the models. Because the models require OHE data (or dummy vars), so the approach I ended up with was from the development notebook, I took all the columns that are used for the model training data, to emulate a OHE dataset, then I created a dict with key: col, value:0. When a message gets processed, the code goes over the data, and updates the 0 to a 1 if a particular value is there. It is easier to understand with an example - if the transaction has category: entertainment, then in the dictionary, the key category_entertainment gets a value of 1 - and the same for all others. Then this dictionary is used as input for the XGBoost and CatBoost models. It might not be the most elegant solution, but it works. 
+
+
+Next, I updated the model dictionary streamlit UI
+
+Now, it automatically reads folders that start with models_. This is not that great, the best would be to read artifacts for each model from mlflow, but given this is all local, it is fine. Later, after the submission, when I do some extra work on this project I might setup mlflow to run on GCP, and use a proper artifact and backend store. Now the UI provides info like:
+
+![image](https://github.com/user-attachments/assets/0066fc42-4116-4960-a111-72bcd8f8ad63)
+
+Finally, I set up a project README
+It'd be best for me to just share a link to it, but again - the repo is private for now.
+
+Project overview
+
+Kaggle - Transaction data is taken from here
+Docker - used for containerization to ensure that the application is consistent across different environments
+Mlflow - used for easy model comparison during development
+Model training - models are trained using a combination of libraries: sklearn, xgboost, catboost, PyG
+Mage - used for pipeline orchestration of the model training and real-time inference pipelines
+Neo4j - used as a Graph database to store transaction data as nodes and edges
+Kafka - used to ensure real-time transaction processing
+Grafana - used for real-time dashboard creation and monitoring
+Streamlit - used to host a model dictionary showing models, evaluation metrics, and feature importance graphs
+
+Data description
+
+![image](https://github.com/user-attachments/assets/4d82347a-fd41-4b7e-8a0f-938b2c76ab99)
+
+Real-time inference pipeline
+
+To simulate real-time information, we use the Kaggle fraudTest.csv dataset and send 5 transactions every second through Kafka. The transactions are received and new columns, where the models predict not fraud/fraud, are created. This data is sent to neo4j and used in the Grafana dashboard, which updates every 5 seconds with the latest information.
+
+Monitoring
+
+Information from Neo4j is extracted and visualised through a real-time dashboard in Grafana
+
+![image](https://github.com/user-attachments/assets/8dd67bd6-a956-47e8-bb40-ceddb871f91c)
+
+Future improvements
+
+* Use a cloud provider for the following services (and use Terraform to serve the resources):
+  * MLflow database and artifact store
+  * Mage
+* Use the official cloud option for:
+  * Neo4j database
+  * Grafana
+* Deploy Streamlit for improved accessibility
+* Add unit/integration tests
+* Load models from Mlflow in the model dictionary
+
+There is also a section on Reproducability, but I will share that once the repo is public. 
+
+That is all for today!
+
+See you tomorrow :)

From cb32e904e19317ba29f35ca60d111669e30d3a56 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:44:42 +0900
Subject: [PATCH 325/638] Update 2024-08-05-day-217.md

---
 _posts/2024-08-05-day-217.md | 52 ++++++++++++++++++++++++++++++++++--
 1 file changed, 50 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-05-day-217.md b/_posts/2024-08-05-day-217.md
index cadf548b6..fdb68eef6 100644
--- a/_posts/2024-08-05-day-217.md
+++ b/_posts/2024-08-05-day-217.md
@@ -1,5 +1,53 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 217) KB project meeting and reading bank telemarketing papers
+categories: [mlops,reading-research,applying-knowledge]
 ---
+
+## Hello :) Today is Day 217!
+A quick summary of today:
+* talking about the KB project
+* reading papers related to bank telemarketing models
+
+Today my KB project partner and I met to catch up. We looked over what he had done during the past week. In addition, we talked about what happens after developing a model in a jupyter notebook - creating a script that takes a new row of features, preprocessing them, and then putting that through the model to get a prediction for that new case. Then we went over the below graph
+
+![image](https://github.com/user-attachments/assets/9b40d747-c664-4623-8dc0-f146f452dae5)
+
+Giving him an overview of what each technology does. He is going on vacation from tomorrow to Friday, so the tasks left are to write up the project ppt that is needed for the submission in Korean. 
+
+
+
+In addition, for reproducability - I noticed that the kafka-producer uses local data to send transactions. For better reproducability, I uploaded the dataset to HuggingFace and used in the code I am reading the csv from the HF url
+
+![image](https://github.com/user-attachments/assets/1270dc4c-9af6-4b74-9540-71d1d2f25c5c)
+
+I also read some papers related to bank term deposit telemarketing
+
+### A dynamic ensemble selection method for bank telemarketing sales prediction
+
+The paper proposes a method called META-DES-AAP to predict the success of bank telemarketing campaigns for selling time deposits. This method stands out by not only focusing on prediction accuracy but also on maximizing average profit. It uses a multi-objective optimization algorithm to select and integrate the most suitable base classifiers dynamically for each campaign. The results show that META-DES-AAP outperforms other state-of-the-art methods in both accuracy and profit maximization​.
+
+![image](https://github.com/user-attachments/assets/2f976ed1-9621-4150-90c7-1c5a0d71bd32)
+
+### Prediction of bank telephone marketing results based on improved whale algorithms optimizing S_Kohonen network
+
+The paper investigates a method to enhance the success rate of bank telemarketing for time deposits. It introduces an improved Whale Optimization Algorithm (WOA) to optimize an S_Kohonen network, which is a variation of the traditional Kohonen network with added supervised learning capabilities. This approach aims to improve the prediction accuracy of whether customers will subscribe to time deposit products based on telemarketing efforts. One cool thing from this paper is that it uses Chi-square and T-tests to determine which variables are worth using in the models.
+
+![image](https://github.com/user-attachments/assets/7565fd01-2911-4ed0-ad74-3ffa04accf84)
+
+### How to improve the success of bank telemarketing? Prediction and interpretability analysis based on machine learning
+
+The paper focuses on enhancing the effectiveness of telemarketing campaigns in banks using machine learning techniques. It develops and tests three machine learning models: Random Subspace (RS), Multi-Boosting (MB), and a combined Random Subspace-Multi-Boosting (RS-MB) model. Among these, the RS-MB model achieved the best performance in predicting the success of telemarketing calls.
+
+Key findings of the paper include:
+
+* Model Performance: The RS-MB model outperformed others, providing more accurate predictions.
+* Variable Importance: The study identified critical variables influencing the success of telemarketing, such as the customer's job type, the month of contact, and the day of the week.
+* False Negatives vs. False Positives: The research emphasized the importance of reducing false negatives (missing potential customers) over false positives (contacting uninterested customers) for maximizing campaign success.
+* Interpretability: Partial Dependence Plots (PDP) were used to show how important variables affect the likelihood of a customer subscribing to a term deposit, aiding in the interpretability of the models and providing actionable insights for bank marketing strategies.
+
+![image](https://github.com/user-attachments/assets/811729a1-c798-4501-99a1-9c91d44a689e)
+
+That is all for today!
+
+See you tomorrow :)

From 276e52ad0141e6f67e09b7b788eee6e30546d14e Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:46:42 +0900
Subject: [PATCH 326/638] Update 2024-08-06-day-218.md

---
 _posts/2024-08-06-day-218.md | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-06-day-218.md b/_posts/2024-08-06-day-218.md
index cadf548b6..993ce9970 100644
--- a/_posts/2024-08-06-day-218.md
+++ b/_posts/2024-08-06-day-218.md
@@ -1,5 +1,25 @@
 ---
 layout: post
-title: 
-categories:
+title: (Day 218) ML canvas for the KB fraud transaction detection project
+categories: [mlops,nlp]
 ---
+
+## Hello :) Today is Day 218!
+A quick summary of today:
+* creating an ML canvas for the KB project
+* rewatching LLM zoomcamp module 4 - monitoring
+
+Just like with the fraud insurance claims [project](https://github.com/divakaivan/insurance-fraud-mlops-pipeline), I created an ML canvas:
+
+![image](https://github.com/user-attachments/assets/3e3cbb4b-c233-4568-91d5-db83b7168406)
+
+Finally, I can share the repo as it is public. [Here is the link](https://github.com/divakaivan/kb_project)
+
+The ML canvas took most my time today and also just small checks to make sure that the repo can be easily reproduced using the current code and instructions.
+Today I also rewatched 4th module on monitoring for LLMs from the LLM zoomcamp. How come? I saw that the 5th module is finished, and before I cover it I wanted to check everything from the 4th module again. The [5th module](https://github.com/DataTalksClub/llm-zoomcamp/tree/main/05-orchestration) covers orchestration using my good ol' friend Mage. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From da56ddcdd5a7f82a8499de8be3d59a7584f05709 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:54:10 +0900
Subject: [PATCH 327/638] Create 2024-08-07-day-219.md

---
 _posts/2024-08-07-day-219.md | 161 +++++++++++++++++++++++++++++++++++
 1 file changed, 161 insertions(+)
 create mode 100644 _posts/2024-08-07-day-219.md

diff --git a/_posts/2024-08-07-day-219.md b/_posts/2024-08-07-day-219.md
new file mode 100644
index 000000000..a0c873e79
--- /dev/null
+++ b/_posts/2024-08-07-day-219.md
@@ -0,0 +1,161 @@
+---
+layout: post
+title: (Day 219) Fundamentals of Data Eng and LLM data preprocessing pipelines in Mage
+categories: [data-eng,theory,nlp,mlops]
+---
+
+## Hello :) Today is Day 219!
+A quick summary of today:
+* starting 'Fundamentals of Data Engineering'
+* covered module 5: orchestration of the LLM zoomcamp
+
+![image](https://github.com/user-attachments/assets/9e1c9f59-1eb9-421f-bc03-cf79710f3ac8)
+
+When I woke up today I saw that DeepLearning.AI is launching a new course - [DE Professional Certificate](https://www.coursera.org/professional-certificates/data-engineering#courses) at the end of August. The instructor - Joe Reis, is one of the writers of the infamous 'holy book' of DE - Fundamentals of Data Engineering.
+Thankfully, I found the book officially published for free by Redpand. Below is a summary of what I read today (Chapter 1).
+
+### What is Data Engineering?
+
+There is a lot of definitions of the term, but all have a similar idea. The book combines it all in this one:
+
+_Data engineering is the development, implementation, and maintenance of systems and processes that take in raw data and produce high-quality, consistent information that supports downstream use cases, such as analysis and machine learning. Data engi‐ neering is the intersection of security, data management, DataOps, data architecture, orchestration, and software engineering. A data engineer manages the data engineering lifecycle, beginning with getting data from source systems and ending with serving data for use cases, such as analysis or machine learning._
+
+### The Data Engineering Lifecycle
+
+![image](https://github.com/user-attachments/assets/779e2274-551f-4ea5-ad4e-fcc0a4fab79c)
+
+Rather than fixating on the tech, this book focuses on the ideas and thinking behind the tech and the processes in the data's lifecycle. It summarises the lifecycle stages as:
+* generation
+* storage
+* ingestion
+* transformation
+* serving
+
+### Evolution of the Data Engineer
+The early days: 1980 to 2000, from data warehousing to the web
+
+Data engineering's roots lie in the development of data warehousing in the 1980s, spearheaded by innovations such as relational databases and SQL. The rise of the internet in the mid-1990s necessitated robust data systems for emerging web-first companies, leading to the evolution of scalable analytics and roles focused on data warehousing and business intelligence.
+
+The early 2000s: The birth of contemporary data engineering
+
+Post-dot-com bust, surviving tech companies like Yahoo, Google, and Amazon pushed traditional data systems to their limits, necessitating scalable, cost-effective solutions. This period saw the advent of big data, driven by innovations like Google's MapReduce and the emergence of Hadoop, along with the rise of public cloud services such as AWS, which revolutionized data storage and computing
+
+The 2000s and 2010s: Big data engineering
+
+The birth of open-source big data tools and the transition to real-time data streaming defined this era. Hadoop and its ecosystem enabled companies to handle massive data sets, although the hype around "big data" sometimes led to its misapplication. The focus shifted from maintaining complex infrastructures to delivering business insights, paving the way for today's data engineering practices.
+
+The 2020s: Engineering for the data lifecycle
+
+Data engineering has evolved towards modular, managed, and highly abstracted tools, forming the modern data stack. The role now emphasizes data lifecycle management, including security, governance, and compliance, while leveraging improved tools and techniques to handle diverse and growing data sources efficiently. This period marks a golden age of data management, focusing on agility and decentralization
+
+### Data Engineering and Data Science
+
+There is some debate, but like I thought, the writers believe that DS is a downstream task of DE
+
+![image](https://github.com/user-attachments/assets/d4354daf-3fd3-4bcd-be3a-e566cc41da92)
+![image](https://github.com/user-attachments/assets/b9605198-3d9e-49e2-b924-581d4911809c)
+
+Although data scientists might want to spend time modelling, most of their work happens at the bottom of this triangle (possibly due to lack of support from data engineers).
+
+![image](https://github.com/user-attachments/assets/3d2fd386-e864-4132-8941-25e1adde074d)
+
+### Data Engineering Skills and Activities
+
+Data engineers used to focus on managing complex, monolithic technologies and low-level details, but now they utilize modern, simplified tools to create agile, cost-effective data solutions. They do not typically build ML models, create reports, perform data analysis, or develop software, though they must understand these areas to support stakeholders.
+
+### Data Maturity and the Data Engineer
+
+The level of data engineering complexity within a company depends a great deal on the company’s data maturity. This significantly impacts a data engineer’s day-to-day job responsibilities.
+
+Data maturity is the progression toward higher data utilization, capabilities, and integration across the organization, but data maturity does not simply depend on the age or revenue of a company. 
+
+![image](https://github.com/user-attachments/assets/1436263c-aac0-4875-bb34-9d7a64dd5fa1)
+
+
+Stage 1: Starting with Data
+
+* Characteristics: Early data maturity, vague goals, nascent data architecture, low adoption, small data team
+* Data Engineer's Role: Generalist, aiming to move quickly and add value
+* Challenges: Avoid premature ML projects, prevent working in silos, and steer clear of unnecessary complexity
+
+Stage 2: Scaling with Data
+* Characteristics: Transition from ad hoc data requests to formal practices, scaling architectures
+* Data Engineer's Role: Specialist, focusing on scalable, robust architectures and formal practices
+* Challenges: Avoid chasing bleeding-edge tech, focus on practical solutions, and lead pragmatic data initiatives
+
+Stage 3: Leading with Data
+* Characteristics: Data-driven company with automated pipelines, self-service analytics, seamless data introduction
+* Data Engineer's Role: Deep specialist, ensuring data availability and leveraging data for competitive advantage
+* Challenges: Avoid complacency, prevent technology distractions, and focus on business value
+
+Business Responsibilities
+* Know how to communicate with nontechnical and technical people
+* Understand how to scope and gather business and product requirements
+* Understand the cultural foundations of Agile, DevOps, and DataOps
+* Control costs
+* Learn continuously
+
+Technical Responsibilities
+
+A data engineer should know how to code and possess production-grade software engineering skills. Although managed services and SaaS have simplified some tasks, strong coding abilities and understanding of software engineering best practices are still crucial for handling complex technical needs and gaining a competitive edge.
+
+Main languages: SQL, Python, Java, Scala, bash
+
+Secondary languages: R, JavaScript, Go, Rust, C/C++, C#, and Julia
+
+How do you keep your skills sharp in a rapidly changing field like data engineering?
+
+Focus on the fundamentals to understand what’s not going to change; pay attention to ongoing developments to know where the field is going. New paradigms and practices are introduced all the time, and it’s incumbent on you to stay current. Strive to understand how new technologies will be helpful in the lifecycle.
+
+The Continuum of Data Engineering Roles, from A to B
+
+In Data Science there are type A and type B data scientists - Analysis (understanding and deriving value from data), and Building (building systems that work in production, while sharing skills with type A).
+
+In DE, there are type A and B engineers too. 
+
+Type A data engineers 
+
+_A stands for abstraction. In this case, the data engineer avoids undifferentiated heavy lifting, keeping data architecture as abstract and straightforward as possible and not reinventing the wheel. Type A data engineers manage the data engineering lifecycle mainly by using entirely off-the-shelf products, managed services, and tools. Type A data engineers work at companies across industries and at all levels of data maturity. _
+
+Type B data engineers 
+
+_B stands for build. Type B data engineers build data tools and systems that scale and leverage a company’s core competency and competitive advantage. In the data maturity range, a type B data engineer is more commonly found at companies in stage 2 and 3 (scaling and leading with data), or when an initial data use case is so unique and mission-critical that custom data tools are required to get started._
+
+It can be different people or 1 person that performs both.
+
+Data Engineers Inside an Organization
+
+![image](https://github.com/user-attachments/assets/d771b56a-228f-4567-bfbf-b74d48e94bc9)
+
+An external-facing data engineer designs and manages systems for collecting, storing, and processing data from user-facing applications like social media, IoT devices, and e-commerce platforms. These systems face challenges such as high concurrency, query limits to manage infrastructure impact, and complex security issues, especially with multitenant data.
+
+![image](https://github.com/user-attachments/assets/45fd8626-a758-4277-9c65-fbe327633f66)
+
+An internal-facing data engineer typically focuses on activities crucial to the needs of the business and internal stakeholders. Examples include creating and maintaining data pipelines and data warehouses for BI dashboards, reports, business processes, data science, and ML models.
+
+![image](https://github.com/user-attachments/assets/6334a504-0ece-4b6c-87e3-b07d60e36c42)
+
+How Data Engineers Interact with Other Roles
+
+![image](https://github.com/user-attachments/assets/62ac1787-5fb8-43ae-9282-08bfb4e85b5b)
+
+### Next, Module 5: Orchestration of the LLM zoomcamp
+
+In mage, there is a new pipeline type for RAG apps:
+
+![image](https://github.com/user-attachments/assets/9aa8ecaa-8ce2-44ec-9486-22878fa4e64b)
+
+In such a pipeline there is a data prep and inference stage.
+
+The first data prep stage includes:
+
+![image](https://github.com/user-attachments/assets/bdb71814-57fb-431e-81e0-5f48e5221f83)
+
+Loading -> Chunking -> Tokenization -> Embedding -> Saving in a vector db
+To be honest, just like with the original MLOps zoomcamp videos that introduce mage, these videos seemed rushed and do not explain much while there being a lot of moving pieces in the UI. In total it is 8 videos, each ~2 minutes. Seems that this RAG pipeline will be part of Mage Pro (a paid version) so I will try not to be too attached to it as I doubt I am going to pay to use Mage. Nevertheless, I might end up using this feature for my LLM zoomcamp project. Or I might just go with a normal pipeline with normal blocks. 
+
+
+
+That is all for today!
+
+See you tomorrow :)

From 996450d0f54b019c4aef5fb67bd17078f55f59de Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Thu, 8 Aug 2024 23:56:42 +0900
Subject: [PATCH 328/638] Create 2024-08-08-day-220.md

---
 _posts/2024-08-08-day-220.md | 80 ++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 _posts/2024-08-08-day-220.md

diff --git a/_posts/2024-08-08-day-220.md b/_posts/2024-08-08-day-220.md
new file mode 100644
index 000000000..e80475216
--- /dev/null
+++ b/_posts/2024-08-08-day-220.md
@@ -0,0 +1,80 @@
+---
+layout: post
+title: (Day 220) Chapter 2 The Data Engineering Lifecycle
+categories: [theory,data-eng]
+---
+
+## Hello :) Today is Day 220!
+A quick summary of today:
+* read chapter 2 from the book 'Fundamentals of Data Engineering'
+* transferred more posts onto the new blog UI
+
+### What Is the Data Engineering Lifecycle?
+The data engineering lifecycle by getting data from source systems and storing it. Next, we transform the data and then proceed to our central goal, serving data to analysts, data scientists, ML engineers, and others. In reality, storage occurs throughout the lifecycle as data flows from beginning to end.
+
+There are 5 stages:
+
+* Generation
+* Storage
+* Ingestion
+* Transformation 
+* Serving data
+
+Generation: Source Systems
+
+Sources produce data consumed by downstream systems, including humangenerated spreadsheets, IoT sensors, and web and mobile applications. Each source has its unique volume and cadence of data generation. A data engineer should know how the source generates data, including relevant quirks or nuances. Data engineers also need to understand the limits of the source systems they interact with.
+
+Storage
+
+Storage runs across the entire data engineering lifecycle, often occurring in multiple places in a data pipeline, with storage systems crossing over with source systems, ingestion, transformation, and serving. In many ways, the way data is stored impacts how it is used in all of the stages of the data engineering lifecycle. For example, cloud data warehouses can store data, process data in pipelines, and serve it to analysts. Streaming frameworks such as Apache Kafka and Pulsar can function simultaneously as ingestion, storage, and query systems for messages, with object storage being a standard layer for data transmission.
+
+Ingestion
+
+After you understand the data source, the characteristics of the source system you’re using, and how data is stored, you need to gather the data. The next stage of the data engineering lifecycle is data ingestion from source systems.
+
+Source systems and ingestion represent the most significant bot‐ tlenecks of the data engineering lifecycle. The source systems are normally outside your direct control and might randomly become unresponsive or provide data of poor quality. Or, your data ingestion service might mysteriously stop working for many reasons. As a result, data flow stops or delivers insufficient data for storage, processing, and serving.
+
+Transformation
+
+After you’ve ingested and stored data, you need to do something with it. The next stage of the data engineering lifecycle is transformation, meaning data needs to be changed from its original form into something useful for downstream use cases. Without proper transformations, data will sit inert, and not be in a useful form for reports, analysis, or ML. Typically, the transformation stage is where data begins to create value for downstream user consumption. Immediately after ingestion, basic transformations map data into correct types (changing ingested string data into numeric and date types, for example), putting records into standard formats, and removing bad ones. Later stages of transformation may transform the data schema and apply normalization. Downstream, we can apply large-scale aggregation for reporting or featurize data for ML processes.
+
+Serving Data
+
+Now that the data has been ingested, stored, and transformed into coherent and useful structures, it’s time to get value from your data. “Getting value” from data means different things to different users. Data has value when it’s used for practical purposes. Data that is not consumed or queried is simply inert. Data vanity projects are a major risk for companies. Many companies pursued vanity projects in the big data era, gathering massive datasets in data lakes that were never consumed in any useful way. The cloud era is triggering a new wave of vanity projects built on the latest data warehouses, object storage systems, and streaming technologies.
+
+* Serving for analytics:
+
+![image](https://github.com/user-attachments/assets/8c1617d2-18b4-4e60-9e0b-8a62a9819c3a)
+
+* Serving for ML
+* Reverse ETL
+
+### Major Undercurrents Across the Data Engineering Lifecycle
+
+![image](https://github.com/user-attachments/assets/a585ff0a-fcfc-49ab-9eb0-314e294a809e)
+
+Data engineers must understand both data and access security, exercising the principle of least privilege. The principle of least privilege means giving a user or system access to only the essential data and resources to perform an intended function. A common antipattern we see with data engineers with little security experience is to give admin access to all users.
+
+Data management is the development, execution, and supervision of plans, policies, programs, and practices that deliver, control, protect, and enhance the value of data and information assets throughout their lifecycle.
+
+DataOps maps the best practices of Agile methodology, DevOps, and statistical pro‐ cess control (SPC) to data. Whereas DevOps aims to improve the release and quality of software products, DataOps does the same thing for data products.
+
+![image](https://github.com/user-attachments/assets/9f51e738-4ba4-49a8-a2a9-7f25a0109767)
+
+A data architecture reflects the current and future state of data systems that support an organization’s long-term data needs and strategy. Because an organization’s data requirements will likely change rapidly, and new tools and practices seem to arrive on a near-daily basis, data engineers must understand good data architecture
+
+Orchestration is the process of coordinating many jobs to run as quickly and effi‐ ciently as possible on a scheduled cadence. For instance, people often refer to orchestration tools like Apache Airflow as schedulers. This isn’t quite accurate. A pure scheduler, such as cron, is aware only of time; an orchestration engine builds in metadata on job dependencies, generally in the form of a directed acyclic graph (DAG). The DAG can be run once or scheduled to run at a fixed interval of daily, weekly, every hour, every five minutes, etc.
+
+
+
+As for the new blog
+
+
+
+I am almost done transferring posts. Maybe from tomorrow I will start posting there new ones.
+
+
+
+That is all for today!
+
+See you tomorrow :)

From e305ddd08fe08d420a6d6d1c31da3a6b554c92c3 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:08:47 +0900
Subject: [PATCH 329/638] Update default.html

---
 _layouts/default.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index 9ae27593a..7b60a7282 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -33,6 +33,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
               <a href="{{ site.baseurl }}/archive">Archive</a>
+              <a href="{{ site.baseurl }}/categories">Categories</a>
             </nav>
           </header>
         </div>

From dcba70e0f2a77748b996550e6cd8540b8e5a7d6f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:15:46 +0900
Subject: [PATCH 330/638] Update categories.md

---
 _pages/categories.md | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index eccf3624d..76cf87a22 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -4,21 +4,16 @@ permalink: /categories/
 title: Categories
 ---
 
-
 <div id="archives">
-{% for category in site.categories %}
-  <div class="archive-group">
-    {% capture category_name %}{{ category | first }}{% endcapture %}
-    <div id="#{{ category_name | slugize }}"></div>
-    <p></p>
-    
-    <h3 class="category-head">{{ category_name }}</h3>
-    <a name="{{ category_name | slugize }}"></a>
-    {% for post in site.categories[category_name] %}
-    <article class="archive-item">
-      <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
-    </article>
-    {% endfor %}
-  </div>
-{% endfor %}
-</div>
\ No newline at end of file
+  {% for category in site.categories %}
+    <div class="archive-group">
+      {% capture category_name %}{{ category | first }}{% endcapture %}
+      <div id="#{{ category_name | slugize }}"></div>
+      <p></p>
+      
+      <!-- Display the category name with the count of posts -->
+      <h3 class="category-head">{{ category_name }} ({{ site.categories[category_name].size }})</h3>
+      <a name="{{ category_name | slugize }}"></a>
+    </div>
+  {% endfor %}
+</div>

From 095a191ba94f38c95e204ce8bf31f42ba0d31362 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:20:19 +0900
Subject: [PATCH 331/638] Update categories.md

---
 _pages/categories.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 76cf87a22..d3c24e398 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -10,10 +10,13 @@ title: Categories
       {% capture category_name %}{{ category | first }}{% endcapture %}
       <div id="#{{ category_name | slugize }}"></div>
       <p></p>
-      
-      <!-- Display the category name with the count of posts -->
       <h3 class="category-head">{{ category_name }} ({{ site.categories[category_name].size }})</h3>
       <a name="{{ category_name | slugize }}"></a>
+      {% for post in site.categories[category_name] %}
+      <article class="archive-item">
+        <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
+      </article>
+      {% endfor %}
     </div>
   {% endfor %}
 </div>

From ae3152ad9fed5c43be3a78553ec389a4887761c2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:22:28 +0900
Subject: [PATCH 332/638] Update categories.md

---
 _pages/categories.md | 51 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index d3c24e398..5ffb3124f 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -5,18 +5,43 @@ title: Categories
 ---
 
 <div id="archives">
-  {% for category in site.categories %}
-    <div class="archive-group">
+  <label for="category-select">Choose a category:</label>
+  <select id="category-select" onchange="location = this.value;">
+    <option value="">--Select a category--</option>
+    {% for category in site.categories %}
       {% capture category_name %}{{ category | first }}{% endcapture %}
-      <div id="#{{ category_name | slugize }}"></div>
-      <p></p>
-      <h3 class="category-head">{{ category_name }} ({{ site.categories[category_name].size }})</h3>
-      <a name="{{ category_name | slugize }}"></a>
-      {% for post in site.categories[category_name] %}
-      <article class="archive-item">
-        <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
-      </article>
-      {% endfor %}
-    </div>
-  {% endfor %}
+      <option value="#{{ category_name | slugize }}">
+        {{ category_name }} ({{ site.categories[category_name].size }})
+      </option>
+    {% endfor %}
+  </select>
+
+  <div id="category-list">
+    {% for category in site.categories %}
+      {% capture category_name %}{{ category | first }}{% endcapture %}
+      <div id="{{ category_name | slugize }}" style="display:none;" class="category-content">
+        <h3 class="category-head">{{ category_name }} ({{ site.categories[category_name].size }})</h3>
+        <a name="{{ category_name | slugize }}"></a>
+        {% for post in site.categories[category_name] %}
+        <article class="archive-item">
+          <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
+        </article>
+        {% endfor %}
+      </div>
+    {% endfor %}
+  </div>
 </div>
+
+<script>
+  // JavaScript to show selected category
+  document.getElementById('category-select').addEventListener('change', function() {
+    var selectedCategory = this.value;
+    var contents = document.querySelectorAll('.category-content');
+    contents.forEach(function(content) {
+      content.style.display = 'none';
+    });
+    if (selectedCategory) {
+      document.querySelector(selectedCategory).style.display = 'block';
+    }
+  });
+</script>

From 54698185245f95e9cc8096608f33f67b4f1867bd Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:24:56 +0900
Subject: [PATCH 333/638] Update categories.md

---
 _pages/categories.md | 47 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/_pages/categories.md b/_pages/categories.md
index 5ffb3124f..e47150437 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -4,6 +4,53 @@ permalink: /categories/
 title: Categories
 ---
 
+<style>
+  /* Styling the dropdown */
+  #category-select {
+    width: 100%;
+    padding: 10px;
+    font-size: 16px;
+    border: 2px solid #ccc;
+    border-radius: 5px;
+    background-color: #f9f9f9;
+    margin-bottom: 20px;
+    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
+  }
+
+  /* Styling the category content */
+  .category-content {
+    margin-top: 20px;
+    padding: 15px;
+    border: 1px solid #e1e1e1;
+    border-radius: 5px;
+    background-color: #ffffff;
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05);
+  }
+
+  /* Styling the category titles */
+  .category-head {
+    font-size: 22px;
+    font-weight: bold;
+    margin-bottom: 10px;
+    color: #333;
+  }
+
+  /* Styling the post titles */
+  .archive-item h4 {
+    font-size: 18px;
+    margin: 5px 0;
+    color: #0073e6;
+  }
+
+  .archive-item h4 a {
+    text-decoration: none;
+  }
+
+  .archive-item h4 a:hover {
+    text-decoration: underline;
+  }
+</style>
+
 <div id="archives">
   <label for="category-select">Choose a category:</label>
   <select id="category-select" onchange="location = this.value;">

From 9736ef3007b643d4fb57347821dd50626e9bdd32 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:27:24 +0900
Subject: [PATCH 334/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 7b60a7282..a51891ed2 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -32,7 +32,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
               <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
-              <a href="{{ site.baseurl }}/archive">Archive</a>
+<!--               <a href="{{ site.baseurl }}/archive">Archive</a> -->
               <a href="{{ site.baseurl }}/categories">Categories</a>
             </nav>
           </header>

From a3d2590118277839730ec54a0079d0db320397a2 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:28:09 +0900
Subject: [PATCH 335/638] Update default.html

---
 _layouts/default.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index a51891ed2..87181434b 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -32,7 +32,6 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
               <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
-<!--               <a href="{{ site.baseurl }}/archive">Archive</a> -->
               <a href="{{ site.baseurl }}/categories">Categories</a>
             </nav>
           </header>

From 65640a38c0c3a8c5f881ca9af16e4ff108c1c91a Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:30:56 +0900
Subject: [PATCH 336/638] Update style.scss

---
 assets/style.scss | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/assets/style.scss b/assets/style.scss
index b2f080b39..a2fda59f8 100644
--- a/assets/style.scss
+++ b/assets/style.scss
@@ -32,7 +32,7 @@ body {
 
 .container {
   margin: 0 auto;
-  max-width: 740px;
+  max-width: 800px;
   padding: 0 10px;
   width: 100%;
 }

From ed09a9c0630e18fd934d195f6e4ecec90622ccb5 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:33:30 +0900
Subject: [PATCH 337/638] Update _config.yml

---
 _config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_config.yml b/_config.yml
index a4cdbf12b..3c07e4e1c 100644
--- a/_config.yml
+++ b/_config.yml
@@ -3,7 +3,7 @@
 #
 
 # Name of your site (displayed in the header)
-name: 50+ days of AI
+name: Study and Blog
 
 # Name of the author
 author: Ivan Ivanov

From 638ea0f9d7483ada1e68ca944e1195869dcd05b1 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:37:06 +0900
Subject: [PATCH 338/638] Update 2024-06-26-day-177.md

---
 _posts/2024-06-26-day-177.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-06-26-day-177.md b/_posts/2024-06-26-day-177.md
index 6b4f9e321..832d9a6a0 100644
--- a/_posts/2024-06-26-day-177.md
+++ b/_posts/2024-06-26-day-177.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 177) Spark for batch processing
-categories: [data-eng,applying-knowlege]
+categories: [data-eng,applying-knowledge]
 ---
 
 ## Hello :) Today is Day 177!

From db760eeccb77cb62481cbe7ecd78745d6efbddfd Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:38:06 +0900
Subject: [PATCH 339/638] Update 2024-06-05-day-156.md

---
 _posts/2024-06-05-day-156.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-06-05-day-156.md b/_posts/2024-06-05-day-156.md
index 73fc89658..c8370c0a9 100644
--- a/_posts/2024-06-05-day-156.md
+++ b/_posts/2024-06-05-day-156.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 156) Final XCS224W - ML with Graphs homework
-categories: [gnn,applying-theory,reading-research]
+categories: [gnn,applying-knowledge,theory,reading-research]
 ---
 
 ## Hello :) Today is Day 156!

From 9e154857bfbe0efbff1cdf21d3c894081e73d2ee Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:38:47 +0900
Subject: [PATCH 340/638] Update 2024-2-14-day-44.md

---
 _posts/2024-2-14-day-44.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-14-day-44.md b/_posts/2024-2-14-day-44.md
index e0c3c5906..ac6c60e32 100644
--- a/_posts/2024-2-14-day-44.md
+++ b/_posts/2024-2-14-day-44.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 44) Batch vs Layer vs Group Normalization and GANs (+ found a free KAIST AI course)
-categories: [applying-knowledge.theory]
+categories: [applying-knowledge,theory]
 ---
 
 ## Hello :) Today is Day 44!

From 4b06b3bdd10862d0a4783cbfa86939eaaa69c15c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:39:28 +0900
Subject: [PATCH 341/638] Update 2024-3-2-day-61.md

---
 _posts/2024-3-2-day-61.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-3-2-day-61.md b/_posts/2024-3-2-day-61.md
index 56502471b..2ae202561 100644
--- a/_posts/2024-3-2-day-61.md
+++ b/_posts/2024-3-2-day-61.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 61) Stanford CS224N (NLP with DL) - Machine translation, seq2seq + a side CDCGAN mini project
-categories: [applying-knowledge,theory,practice,cv]
+categories: [applying-knowledge,theory,practice,cnn]
 ---
 
 ## Hello :) Today is Day 61!

From 48425e01532c7eca4fc5069fd60362390d2c8a9c Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:40:31 +0900
Subject: [PATCH 342/638] Update 2024-2-3-day-33.md

---
 _posts/2024-2-3-day-33.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-3-day-33.md b/_posts/2024-2-3-day-33.md
index 45a70f337..b705e5935 100644
--- a/_posts/2024-2-3-day-33.md
+++ b/_posts/2024-2-3-day-33.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 33) Tensorflow deployment specialization and a webapp for recognizing the Korean alphabet
-categories: [deep-learning, cnn, model-deployment]
+categories: [deep-learning, cnn, mlops]
 ---
 
 ## Hello :) Today is Day 33!

From f4acbe449f6c83c9315a712f8a757c2fa9fa315b Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:40:49 +0900
Subject: [PATCH 343/638] Update 2024-2-4-day-34.md

---
 _posts/2024-2-4-day-34.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-2-4-day-34.md b/_posts/2024-2-4-day-34.md
index 19affa3cc..be2af3915 100644
--- a/_posts/2024-2-4-day-34.md
+++ b/_posts/2024-2-4-day-34.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 34) Deploying ML models and TensorBoard
-categories: [model-deployment]
+categories: [mlops]
 ---
 
 ## Hello :) Today is Day 34!

From dc53958a590a445fa20b0b32585a97304ed7e5ca Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:41:13 +0900
Subject: [PATCH 344/638] Update 2024-04-11-day-101.md

---
 _posts/2024-04-11-day-101.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-04-11-day-101.md b/_posts/2024-04-11-day-101.md
index af70c8271..63188d6fe 100644
--- a/_posts/2024-04-11-day-101.md
+++ b/_posts/2024-04-11-day-101.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 101) Doing the Google Cloud Digital Leader Learning Path
-categories: [cloud,model-deployment]
+categories: [cloud,mlops,applying-knowledge]
 ---
 
 ## Hello :) Today is Day 101!

From bd4683c45515018200ae90f65424a645de77726f Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 00:42:20 +0900
Subject: [PATCH 345/638] Update 2024-3-2-day-61.md

---
 _posts/2024-3-2-day-61.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-3-2-day-61.md b/_posts/2024-3-2-day-61.md
index 2ae202561..a3e8c76bd 100644
--- a/_posts/2024-3-2-day-61.md
+++ b/_posts/2024-3-2-day-61.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 61) Stanford CS224N (NLP with DL) - Machine translation, seq2seq + a side CDCGAN mini project
-categories: [applying-knowledge,theory,practice,cnn]
+categories: [applying-knowledge,theory,applying-knowledge,cnn]
 ---
 
 ## Hello :) Today is Day 61!

From 1fac460282d5a46c166442090895a56f51685716 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 00:44:43 +0900
Subject: [PATCH 346/638] replace stats with statistics

---
 _posts/2024-04-29-day-119.md | 2 +-
 _posts/2024-05-01-day-121.md | 2 +-
 _posts/2024-05-07-day-127.md | 2 +-
 _posts/2024-05-08-day-128.md | 2 +-
 _posts/2024-05-09-day-129.md | 2 +-
 _posts/2024-05-10-day-130.md | 2 +-
 _posts/2024-05-11-day-131.md | 2 +-
 _posts/2024-05-13-day-133.md | 2 +-
 _posts/2024-05-20-day-140.md | 4 ++--
 _posts/2024-05-21-day-141.md | 4 ++--
 _posts/2024-05-22-day-142.md | 2 +-
 _posts/2024-06-15-day-166.md | 4 ++--
 _posts/2024-06-18-day-169.md | 4 ++--
 _posts/2024-06-25-day-176.md | 2 +-
 _posts/2024-07-04-day-185.md | 2 +-
 _posts/2024-2-28-day-58.md   | 2 +-
 16 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/_posts/2024-04-29-day-119.md b/_posts/2024-04-29-day-119.md
index dd6d3cea4..b3181d0b0 100644
--- a/_posts/2024-04-29-day-119.md
+++ b/_posts/2024-04-29-day-119.md
@@ -25,7 +25,7 @@ Assuming links(edges) are missing, we do not have an explicit adjecency matrix A
 
 ![](https://blogger.googleusercontent.com/img/a/AVvXsEhZv7HtQbijV3AykI7FrDVVcztJTGgRvsBD6cnJQguwGHhmJzr4axb4eA7ZDh9aFNVb8louFX1Ue3hfYCaDoK44xaZCfOxSdAWWReSH9HubheougA_tYa242kxx2N8Mm8Ii5w9p9O4cuwue7w0gizbyEnUei052xZ3hU6JwcKxhh_ba3nDRq_avi2DwCnqe)
 
-To help the transformer not to go around looking at all potential edges, based on: (1) data stats (i.e. treatment nodes can only be connected to diagnosis nodes); and (2) conditional probs from the data (i.e. p(EKG|chest pain) is usually larger than p(EKG|fever)) - they restrict the space where the search can look at.
+To help the transformer not to go around looking at all potential edges, based on: (1) data statistics (i.e. treatment nodes can only be connected to diagnosis nodes); and (2) conditional probs from the data (i.e. p(EKG|chest pain) is usually larger than p(EKG|fever)) - they restrict the space where the search can look at.
 
 ![](https://blogger.googleusercontent.com/img/a/AVvXsEgKHBFludmwNHBlKWK28BEYn_RHUN3Vpb8AjKzawMl0AVTwkP18L5Dzj19wXgXn42ObGoP9Sp3ezfo_vMXlDRAsbPG00fxBqLWCGiiQQrDLxbeYNO6wFAfWJcgszcIqqhCv1zOaKHII-yM4O7BboYS6I_qOCsxDOs9P4xfLl1ql8SbZK6Pa8RpyJ5Qe8SsV)
 
diff --git a/_posts/2024-05-01-day-121.md b/_posts/2024-05-01-day-121.md
index a3a1a0788..bb114eeae 100644
--- a/_posts/2024-05-01-day-121.md
+++ b/_posts/2024-05-01-day-121.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 121) Uncovering the full reason behing multicollinearity + Frequent itemset mining lecture
-categories: [theory,stats,math]
+categories: [theory,statistics,math]
 ---
 
 ## Hello :) Today is Day 121!
diff --git a/_posts/2024-05-07-day-127.md b/_posts/2024-05-07-day-127.md
index 9c31a2657..414cbbdc8 100644
--- a/_posts/2024-05-07-day-127.md
+++ b/_posts/2024-05-07-day-127.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 127) Serving an API endpoint for news classification + Stanford's CS109
-categories: [theory,applying-knowledge,stats]
+categories: [theory,applying-knowledge,statistics]
 ---
 
 ## Hello :) Today is Day 127!
diff --git a/_posts/2024-05-08-day-128.md b/_posts/2024-05-08-day-128.md
index a149de8b8..61fa5bd75 100644
--- a/_posts/2024-05-08-day-128.md
+++ b/_posts/2024-05-08-day-128.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 128) IBM Consulting Insights Virtual Careers event + More of CS109
-categories: [theory,stats]
+categories: [theory,statistics]
 ---
 
 ## Hello :) Today is Day 128!
diff --git a/_posts/2024-05-09-day-129.md b/_posts/2024-05-09-day-129.md
index eb19de0ad..8a9397b3f 100644
--- a/_posts/2024-05-09-day-129.md
+++ b/_posts/2024-05-09-day-129.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 129) AI with a Scottish accent? + MLE lecture by Chris Piech (Stanford CS109)
-categories: [applying-knowledge,theory,stats]
+categories: [applying-knowledge,theory,statistics]
 ---
 
 ## Hello :) Today is Day 129!
diff --git a/_posts/2024-05-10-day-130.md b/_posts/2024-05-10-day-130.md
index d207eca3c..d68175ec0 100644
--- a/_posts/2024-05-10-day-130.md
+++ b/_posts/2024-05-10-day-130.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 130) CS109 - MAP, Naive Bayes, Logistic Regression
-categories: [theory,stats]
+categories: [theory,statistics]
 ---
 
 ## Hello :) Today is Day 130!
diff --git a/_posts/2024-05-11-day-131.md b/_posts/2024-05-11-day-131.md
index 7e0cb3a8c..c163c6355 100644
--- a/_posts/2024-05-11-day-131.md
+++ b/_posts/2024-05-11-day-131.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 131) Meeting for the Scottish dataset project + CS109 - Deep learning
-categories: [theory,applying-knowledge,stats]
+categories: [theory,applying-knowledge,statistics]
 ---
 
 ## Hello :) Today is Day 131!
diff --git a/_posts/2024-05-13-day-133.md b/_posts/2024-05-13-day-133.md
index 652d7eade..888dcc25c 100644
--- a/_posts/2024-05-13-day-133.md
+++ b/_posts/2024-05-13-day-133.md
@@ -25,7 +25,7 @@ What I did was, make short clips around each expression. I am not sure what thes
 
 ### Secondly, today I read about eigenvalues' role in factor analysis
 
-In my stats class at uni, we learned about factor analysis, and at the end of the chapter I saw the word eigenvalues, and I am glad because once again I will see their real world impact (after my dive into multicollinearity). 
+In my statistics class at uni, we learned about factor analysis, and at the end of the chapter I saw the word eigenvalues, and I am glad because once again I will see their real world impact (after my dive into multicollinearity). 
 
 Firstly, about factor analysis, here are the results after using unstandardized variables
 
diff --git a/_posts/2024-05-20-day-140.md b/_posts/2024-05-20-day-140.md
index 3038f2e0a..09b3b4caa 100644
--- a/_posts/2024-05-20-day-140.md
+++ b/_posts/2024-05-20-day-140.md
@@ -10,7 +10,7 @@ A quick summary of today:
 * Ch 2 - improper integrals, numerical integration, interest rates and bonds
 * Ch 3 - probability, Black-Scholes formula, Greeks and hedging 
 
-From the books I found yesterday I thought that starting from this one is the best because of its pure math + real application format. Each chapter consists of some math/stats and also how they are used in the real world.
+From the books I found yesterday I thought that starting from this one is the best because of its pure math + real application format. Each chapter consists of some math/statistics and also how they are used in the real world.
 
 
 
@@ -38,7 +38,7 @@ From the books I found yesterday I thought that starting from this one is the be
 ![image](https://github.com/user-attachments/assets/818e4274-bb27-49fd-93d4-42eefc119f70)
 ![image](https://github.com/user-attachments/assets/30ed878d-dd36-454a-a062-150a33ef11af)
 
-On another note ~ today I realised that calling this whole blog project self-study into AI might not be the best. Because math, probability and stats are at the root of AI. So maybe some kind of renaming would be appropriate. 
+On another note ~ today I realised that calling this whole blog project self-study into AI might not be the best. Because math, probability and statistics are at the root of AI. So maybe some kind of renaming would be appropriate. 
 
 
 
diff --git a/_posts/2024-05-21-day-141.md b/_posts/2024-05-21-day-141.md
index f2b690edb..2d7005a60 100644
--- a/_posts/2024-05-21-day-141.md
+++ b/_posts/2024-05-21-day-141.md
@@ -7,7 +7,7 @@ categories: [theory,math]
 ## Hello :) Today is Day 141!
 A quick summary of today:
 * read a bit more of A Primer For The Mathematics Of Financial Engineering by Dan Stefanica
-* applied (sent an email) about becoming a TA for one of the undergraduate big data/ML/stats courses in my uni
+* applied (sent an email) about becoming a TA for one of the undergraduate big data/ML/statistics courses in my uni
 
 I like the Primer book because it is pure math - just Lemmas and Theorems and proving them (+ application in finance).
 
@@ -23,7 +23,7 @@ Also I saw that the official 2nd homework on model management and experiment tra
 
 
 
-As for the 'application' to be a TA. I believe teaching is the highest form of knowing something. Being able to explain a concept to someone who does not know it and them understanding it is one of the biggest certifiers that someone really know's something. So I wrote an email in Korean about that I know python, some math, some stats, have been studying and blogging here about ML/DL, so let's see if I can atleast meet with a professor even just for a chat.
+As for the 'application' to be a TA. I believe teaching is the highest form of knowing something. Being able to explain a concept to someone who does not know it and them understanding it is one of the biggest certifiers that someone really know's something. So I wrote an email in Korean about that I know python, some math, some statistics, have been studying and blogging here about ML/DL, so let's see if I can atleast meet with a professor even just for a chat.
 
 
 
diff --git a/_posts/2024-05-22-day-142.md b/_posts/2024-05-22-day-142.md
index 21b57578e..b722e3606 100644
--- a/_posts/2024-05-22-day-142.md
+++ b/_posts/2024-05-22-day-142.md
@@ -13,7 +13,7 @@ Unfortunately, as with the previous assignments, I am not allowed to share anyth
 
 ![image](https://github.com/user-attachments/assets/6aa469ab-2530-4ca2-8c4c-51bd5a992673)
 
-As for the interview ~ yesterday I sent an email to the head of the department of the Big Data department at my uni - Professor Minju Park, last night he responded and invited me today at 12 at his office. We had a chat, answered questions, ambitions about research, projects, and at the end ... he said there is 1 space left in his lab and that I can join. The lab is all undergrads and myself (the only non-korean) and I am excited to study, learn, share about anything related to AI - from math to probability to stats to ML to DL. I said I can start from tomorrow, and I will start tomorrow. 
+As for the interview ~ yesterday I sent an email to the head of the department of the Big Data department at my uni - Professor Minju Park, last night he responded and invited me today at 12 at his office. We had a chat, answered questions, ambitions about research, projects, and at the end ... he said there is 1 space left in his lab and that I can join. The lab is all undergrads and myself (the only non-korean) and I am excited to study, learn, share about anything related to AI - from math to probability to statistics to ML to DL. I said I can start from tomorrow, and I will start tomorrow. 
 
 
 That is all for today!
diff --git a/_posts/2024-06-15-day-166.md b/_posts/2024-06-15-day-166.md
index 5de8ea11e..d50956553 100644
--- a/_posts/2024-06-15-day-166.md
+++ b/_posts/2024-06-15-day-166.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 166) Buying a new book + pseudocon in Seoul + using Gradio for a quick demo app
-categories: [applying-knowledge,stats]
+categories: [applying-knowledge,statistics]
 ---
 
 ## Hello :) Today is Day 166!
@@ -26,7 +26,7 @@ Some interesting bits from the sessions which covered a wide range - open source
 
 ![image](https://github.com/user-attachments/assets/a2d4f097-bee5-431b-854e-8f1290e661f2)
 
-Next to the microsoft office is a big book store where I know they have imported English books. So I had a look at the probability/stats section for something interesting. And this one was just 10,000 won (~10$) which was very surprising for an imported book. I read a bit on my ride back home, and it includes topics like: multivariate analysis, matrix algebra, displaying multivariate data, tests of significance with multivariate data, measuring and testing multivariate distances, PCA, factor analysis, discriminant function analysis, cluster analysis, canonical correlation analysis, multidim scaling, and ordination. Not new topics, and not topics that I have not read about in other books. But I like having books in my personal library, and importing foreign books in Korea is very expensive for me. So I will happily read this one. 
+Next to the microsoft office is a big book store where I know they have imported English books. So I had a look at the probability/statistics section for something interesting. And this one was just 10,000 won (~10$) which was very surprising for an imported book. I read a bit on my ride back home, and it includes topics like: multivariate analysis, matrix algebra, displaying multivariate data, tests of significance with multivariate data, measuring and testing multivariate distances, PCA, factor analysis, discriminant function analysis, cluster analysis, canonical correlation analysis, multidim scaling, and ordination. Not new topics, and not topics that I have not read about in other books. But I like having books in my personal library, and importing foreign books in Korea is very expensive for me. So I will happily read this one. 
 
 ### Thirdly, a demo webapp for the Glaswegian dataset
 
diff --git a/_posts/2024-06-18-day-169.md b/_posts/2024-06-18-day-169.md
index 1fb3629d3..4c06f1a0b 100644
--- a/_posts/2024-06-18-day-169.md
+++ b/_posts/2024-06-18-day-169.md
@@ -46,7 +46,7 @@ What types of data scientists are in the Bank?
 
 data scientist modeller
 * builds ML models which find patterns in data to make predictions
-* key skills: python/coding, stats, ML, business knowledge
+* key skills: python/coding, statistics, ML, business knowledge
 
 ML engineer
 * productionises ML models
@@ -54,7 +54,7 @@ ML engineer
 
 data science leader
 * uses DS experience and knowledge to lead teams and translate business problems into DS solutions
-* key skills: business knowledge, communication, ML, stats
+* key skills: business knowledge, communication, ML, statistics
 
 AI ethics expert
 * creates the tools, frameworks and guardrails which ensures DS and AI are developed responsibly
diff --git a/_posts/2024-06-25-day-176.md b/_posts/2024-06-25-day-176.md
index 6106d281b..f24e466b8 100644
--- a/_posts/2024-06-25-day-176.md
+++ b/_posts/2024-06-25-day-176.md
@@ -80,7 +80,7 @@ dbt provides a way to generate documentation for our dbt project and render it a
 - Descriptions (from `.yml` file) and tests
 - Information about our data warehouse (`information_schema`)
 - Column names and data types
-- Table stats like size and rows
+- Table statistics like size and rows
 
 dbt docs can also be hosted in dbt Cloud.
 
diff --git a/_posts/2024-07-04-day-185.md b/_posts/2024-07-04-day-185.md
index a6c80ce54..149741fdd 100644
--- a/_posts/2024-07-04-day-185.md
+++ b/_posts/2024-07-04-day-185.md
@@ -43,7 +43,7 @@ Deploy instructions:
 
 ![image](https://github.com/user-attachments/assets/3ff03a7b-dc56-44dc-aafb-b1a4c9867ad0)
 
-Instead of going to mlflow to get info, I tried to make it a bit easier and clearer on how to get a model, and also showing basic performance stats. 
+Instead of going to mlflow to get info, I tried to make it a bit easier and clearer on how to get a model, and also showing basic performance statistics. 
 
 I am using Prefect's latest (and recently) released version 3.x. So to deploy a flow - make it available to be ran from the UI there is a .serve method, and I spent some time figuring that out, but finally it works fine. 
 
diff --git a/_posts/2024-2-28-day-58.md b/_posts/2024-2-28-day-58.md
index 15100c27e..18648e105 100644
--- a/_posts/2024-2-28-day-58.md
+++ b/_posts/2024-2-28-day-58.md
@@ -35,7 +35,7 @@ Next, are the short summaries of each paper. These were definitely interesting p
 
 Evaluation methods for word vector models often focus on the distance or angle between word vectors. Recent approaches focus on word analogies and the two main model families for learning word vectors are global matrix factorization methods like latent semantic analysis (LSA) and local context window methods like the skip-gram model. The authors propose a global log-bilinear regression model trained on global word-word co-occurrence counts to efficiently capture the multi-clustering idea of distributed representations.
 
-Count and prediction based methods both try to do word representations. But with glove, the authours argue that the two methods and not that different since they both operate on the co-occurance stats of the data. So the paper proposes GloVe which utilizes the benefits of the count-based (capturing global statistics), and also the benefits of prediction-based models (adding the linear substructures popular in models like word2vec).
+Count and prediction based methods both try to do word representations. But with glove, the authours argue that the two methods and not that different since they both operate on the co-occurance statistics of the data. So the paper proposes GloVe which utilizes the benefits of the count-based (capturing global statistics), and also the benefits of prediction-based models (adding the linear substructures popular in models like word2vec).
 
 2. Related work
 

From 536e1e8d5c1dcd081279d1830708f457c6136bc4 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 00:56:27 +0900
Subject: [PATCH 347/638] replace research with reading-research

---
 _posts/2024-2-28-day-58.md  | 2 +-
 _posts/2024-3-12-day-71.md  | 2 +-
 _posts/2024-3-14-day-73.md  | 2 +-
 _posts/2024-3-26-day-85.md  | 2 +-
 _posts/2024-4-10-day-100.md | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/_posts/2024-2-28-day-58.md b/_posts/2024-2-28-day-58.md
index 18648e105..4f884633d 100644
--- a/_posts/2024-2-28-day-58.md
+++ b/_posts/2024-2-28-day-58.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 58) Stanford CS224N (NLP with DL) Lecture 2 - Neural classifiers (diving deeper into word embeddings)
-categories: [applying-knowledge,theory,research]
+categories: [applying-knowledge,theory,reading-research]
 ---
 
 ## Hello :) Today is Day 58!
diff --git a/_posts/2024-3-12-day-71.md b/_posts/2024-3-12-day-71.md
index 80f5a958a..cf9dbbb90 100644
--- a/_posts/2024-3-12-day-71.md
+++ b/_posts/2024-3-12-day-71.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 71) Backprop, GELU, Tricking ChatGPT, and Stealing part of an LLM
-categories: [applying-knowledge,theory,nlp,research]
+categories: [applying-knowledge,theory,nlp,reading-research]
 ---
 
 ## Hello :) Today is Day 71!
diff --git a/_posts/2024-3-14-day-73.md b/_posts/2024-3-14-day-73.md
index 171d7942d..d5c353d17 100644
--- a/_posts/2024-3-14-day-73.md
+++ b/_posts/2024-3-14-day-73.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 73) MBR and FUDGE - decoding mechanisms; pre vs post layer normalization
-categories: [nlp,theory,research]
+categories: [nlp,theory,reading-research]
 ---
 
 ## Hello :) Today is Day 73!
diff --git a/_posts/2024-3-26-day-85.md b/_posts/2024-3-26-day-85.md
index 8de69f900..de6061080 100644
--- a/_posts/2024-3-26-day-85.md
+++ b/_posts/2024-3-26-day-85.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 85) How to write a great research paper
-categories: [theory,research]
+categories: [theory,reading-research]
 ---
 
 ## Hello :) Today is Day 85!
diff --git a/_posts/2024-4-10-day-100.md b/_posts/2024-4-10-day-100.md
index cf4399b56..913d961bb 100644
--- a/_posts/2024-4-10-day-100.md
+++ b/_posts/2024-4-10-day-100.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 100) Embeddings in practice + reading a couple of research papers + trying to deploy an LLM in production
-categories: [applying-knowledge,nlp,research]
+categories: [applying-knowledge,nlp,reading-research]
 ---
 
 ## Hello :) Today is Day 100!

From 8f1aa372a210383ec9078fe3f302aa9c446534cc Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 00:57:52 +0900
Subject: [PATCH 348/638] fix tag

---
 _posts/2024-3-19-day-78.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-3-19-day-78.md b/_posts/2024-3-19-day-78.md
index d17242b93..26f3be2a6 100644
--- a/_posts/2024-3-19-day-78.md
+++ b/_posts/2024-3-19-day-78.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 78) NVIDIA GTC talks + accepted to Stanford AI professional certificate + PERL
-categories: [research,theory,nlp]
+categories: [reading-research,theory,nlp]
 ---
 
 ## Hello :) Today is Day 78!

From 172d1f0a2dc18346948e9e6533e8b52c0865116a Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 01:01:09 +0900
Subject: [PATCH 349/638] fix tags

---
 _posts/2024-1-22-day-21.md | 2 +-
 _posts/2024-1-25-day-24.md | 2 +-
 _posts/2024-1-4-day-3.md   | 2 +-
 _posts/2024-1-7-day-6.md   | 2 +-
 _posts/2024-1-8-day-7.md   | 2 +-
 _posts/2024-1-9-day-8.md   | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/_posts/2024-1-22-day-21.md b/_posts/2024-1-22-day-21.md
index d840406c9..bacdeb3bb 100644
--- a/_posts/2024-1-22-day-21.md
+++ b/_posts/2024-1-22-day-21.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 21) (Almost) finished prep for TensorFlow developer certificate
-categories: [deep-learning, time-series]
+categories: [deep-learning,applying-knowledge]
 ---
 
 ## Hello :) Today is Day 21!
diff --git a/_posts/2024-1-25-day-24.md b/_posts/2024-1-25-day-24.md
index d2df8d7d6..f0cd4e110 100644
--- a/_posts/2024-1-25-day-24.md
+++ b/_posts/2024-1-25-day-24.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 24) Using neural nets for time series predictions
-categories: [deep-learning, time-series, applying-knowledge]
+categories: [deep-learning, applying-knowledge]
 ---
 
 ## Hello :) Today is Day 24!
diff --git a/_posts/2024-1-4-day-3.md b/_posts/2024-1-4-day-3.md
index 679da2a05..8afc72c0a 100644
--- a/_posts/2024-1-4-day-3.md
+++ b/_posts/2024-1-4-day-3.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 3) Intro to time series
-categories: [traditional-machine-learning, time-series]
+categories: [traditional-machine-learning]
 ---
 
 ## Hello :) Today is Day 3!
diff --git a/_posts/2024-1-7-day-6.md b/_posts/2024-1-7-day-6.md
index 7d15e9bfc..9e6fc6e0b 100644
--- a/_posts/2024-1-7-day-6.md
+++ b/_posts/2024-1-7-day-6.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 6) Time series course - Kaggle
-categories: [time-series]
+categories: [applying-knowledge,traditional-machine-learning]
 ---
 
 ## Hello :) Today is Day 6!
diff --git a/_posts/2024-1-8-day-7.md b/_posts/2024-1-8-day-7.md
index 53ab7a374..43bbebddb 100644
--- a/_posts/2024-1-8-day-7.md
+++ b/_posts/2024-1-8-day-7.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 7) Applying the time series knowledge to practice
-categories: [time-series, applying-knowledge]
+categories: [traditional-machine-learning,applying-knowledge]
 ---
 
 ## Hello :) Today is Day 7!
diff --git a/_posts/2024-1-9-day-8.md b/_posts/2024-1-9-day-8.md
index 4b6f34455..5c5067ecc 100644
--- a/_posts/2024-1-9-day-8.md
+++ b/_posts/2024-1-9-day-8.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 8) Time series - finding trends
-categories: [time-series, applying-knowledge]
+categories: [traditional-machine-learning, applying-knowledge]
 ---
 
 ## Hello :) Today is Day 8!

From 7f8e5cc6db0601f9ea550c36745ac35e5acd357f Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 01:04:04 +0900
Subject: [PATCH 350/638] fix tags

---
 _posts/2024-04-17-day-107.md | 2 +-
 _posts/2024-04-22-day-112.md | 2 +-
 _posts/2024-05-18-day-138.md | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/_posts/2024-04-17-day-107.md b/_posts/2024-04-17-day-107.md
index bb355060f..ef160b5c0 100644
--- a/_posts/2024-04-17-day-107.md
+++ b/_posts/2024-04-17-day-107.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 107) Transforming natural language to charts
-categories: [applying-knowledge,nlp,project]
+categories: [applying-knowledge,nlp]
 ---
 
 ## Hello :) Today is Day 107!
diff --git a/_posts/2024-04-22-day-112.md b/_posts/2024-04-22-day-112.md
index 9615649dc..1158f2e47 100644
--- a/_posts/2024-04-22-day-112.md
+++ b/_posts/2024-04-22-day-112.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 112) db2chat - Talk with your (sqlite3) database
-categories: [nlp,applying-knowledge,project]
+categories: [nlp,applying-knowledge]
 ---
 
 ## Hello :) Today is Day 112!
diff --git a/_posts/2024-05-18-day-138.md b/_posts/2024-05-18-day-138.md
index ddc30480e..25e384644 100644
--- a/_posts/2024-05-18-day-138.md
+++ b/_posts/2024-05-18-day-138.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 138) Fine-tuning Speech T5 using a very small Glaswegian dataset
-categories: [applying-knowledge,glaswegian,project]
+categories: [applying-knowledge,glaswegian]
 ---
 
 ## Hello :) Today is Day 138!

From 87cef2e262800e254cfc49d6a2959623e22d9d01 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 01:06:58 +0900
Subject: [PATCH 351/638] remove categories tab

---
 _layouts/default.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 87181434b..186a26cbb 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -32,7 +32,6 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
               <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/search">Search</a>
               <a href="{{ site.baseurl }}/about">About</a>
-              <a href="{{ site.baseurl }}/categories">Categories</a>
             </nav>
           </header>
         </div>

From 34ba471acd78cb74c158fc90f7106aef014c3fca Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 01:08:21 +0900
Subject: [PATCH 352/638] remove share

---
 _layouts/post.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_layouts/post.html b/_layouts/post.html
index 1f2192032..06b186f52 100644
--- a/_layouts/post.html
+++ b/_layouts/post.html
@@ -27,9 +27,9 @@ <h1>{{ page.title }}</h1>
     {{ content }}
   </div>
 
-  <div>
+  <!-- <div>
     <p><span class="share-box">Share:</span> <a href="http://twitter.com/share?text={{ page.title }}&url={{site.url}}{{page.url}}" target="_blank">Twitter</a>, <a href="https://www.facebook.com/sharer.php?u={{site.url}}{{page.url}}" target="_blank">Facebook</a></p>
-  </div>
+  </div> -->
 
   <!--<div class="date">
     Written on {{ page.date | date: "%B %e, %Y" }}

From f274488800308bab371c43df389aa56055c70231 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 01:11:23 +0900
Subject: [PATCH 353/638] update about me

---
 _pages/about.md | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/_pages/about.md b/_pages/about.md
index 5f1024b32..2981e78f0 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -8,20 +8,6 @@ Hello!
 
 On the 2nd of January 2024, I embarked on a, what I thought would be, 50 day journey to learn about Machine Learning. It is way past the 50 day mark, and I keep learning and blogging. 
 
-Topics covered so far:
-* Calculus
-* Probability and statistics
-* Classical ML methods
-* Deep learning
-* Computer vision
-* Natural Language Processing
-* Graph Neural Networks
-
-The list is not in order, I keep going between high and low-level concepts, so that I do not fall victim to the <a href="https://www.joelonsoftware.com/2002/11/11/the-law-of-leaky-abstractions/" target="_blank">law of leaky abstractions</a>
-
-Courses covered:
-* aaa
-
-Books read:
-* aaa
+On Day 100 I knew it was not over, so I continued.
 
+After a few days I decided to go for the full year.
\ No newline at end of file

From c615e000044cb9df9b17f93523c9e2e122fc2dab Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 09:52:54 +0900
Subject: [PATCH 354/638] add top categories chart

---
 _pages/categories.md | 59 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/_pages/categories.md b/_pages/categories.md
index e47150437..fc452ac57 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -49,6 +49,21 @@ title: Categories
   .archive-item h4 a:hover {
     text-decoration: underline;
   }
+
+  /* Styling the chart */
+  #category-chart-container {
+    margin-top: 30px;
+    padding: 15px;
+    border: 1px solid #e1e1e1;
+    border-radius: 5px;
+    background-color: #ffffff;
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05);
+  }
+
+  #category-chart {
+    width: 100%;
+    height: 400px;
+  }
 </style>
 
 <div id="archives">
@@ -77,8 +92,14 @@ title: Categories
       </div>
     {% endfor %}
   </div>
+
+  <!-- Bar Chart Container -->
+  <div id="category-chart-container">
+    <canvas id="category-chart"></canvas>
+  </div>
 </div>
 
+<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 <script>
   // JavaScript to show selected category
   document.getElementById('category-select').addEventListener('change', function() {
@@ -91,4 +112,42 @@ title: Categories
       document.querySelector(selectedCategory).style.display = 'block';
     }
   });
+
+  // JavaScript to create the bar chart
+  var ctx = document.getElementById('category-chart').getContext('2d');
+  var categoryData = {
+    labels: [
+      {% for category in site.categories %}
+        {% capture category_name %}{{ category | first }}{% endcapture %}
+        "{{ category_name }}",
+      {% endfor %}
+    ],
+    datasets: [{
+      label: 'Number of Posts',
+      data: [
+        {% for category in site.categories %}
+          {{ site.categories[category_name].size }},
+        {% endfor %}
+      ],
+      backgroundColor: 'rgba(0, 123, 255, 0.5)',
+      borderColor: 'rgba(0, 123, 255, 1)',
+      borderWidth: 1
+    }]
+  };
+
+  var categoryChart = new Chart(ctx, {
+    type: 'bar',
+    data: categoryData,
+    options: {
+      responsive: true,
+      scales: {
+        y: {
+          beginAtZero: true,
+          ticks: {
+            precision: 0
+          }
+        }
+      }
+    }
+  });
 </script>

From 85ae38f766494d31297bf484597bfc74dad0e7f0 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 09:56:17 +0900
Subject: [PATCH 355/638] add graph

---
 _pages/categories.md | 75 ++++++++++++++------------------------------
 1 file changed, 23 insertions(+), 52 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index fc452ac57..b7786adc2 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -49,23 +49,11 @@ title: Categories
   .archive-item h4 a:hover {
     text-decoration: underline;
   }
-
-  /* Styling the chart */
-  #category-chart-container {
-    margin-top: 30px;
-    padding: 15px;
-    border: 1px solid #e1e1e1;
-    border-radius: 5px;
-    background-color: #ffffff;
-    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05);
-  }
-
-  #category-chart {
-    width: 100%;
-    height: 400px;
-  }
 </style>
 
+<!-- Chart Container -->
+<canvas id="categoryChart" width="400" height="200"></canvas>
+
 <div id="archives">
   <label for="category-select">Choose a category:</label>
   <select id="category-select" onchange="location = this.value;">
@@ -92,62 +80,45 @@ title: Categories
       </div>
     {% endfor %}
   </div>
-
-  <!-- Bar Chart Container -->
-  <div id="category-chart-container">
-    <canvas id="category-chart"></canvas>
-  </div>
 </div>
 
 <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 <script>
-  // JavaScript to show selected category
-  document.getElementById('category-select').addEventListener('change', function() {
-    var selectedCategory = this.value;
-    var contents = document.querySelectorAll('.category-content');
-    contents.forEach(function(content) {
-      content.style.display = 'none';
-    });
-    if (selectedCategory) {
-      document.querySelector(selectedCategory).style.display = 'block';
-    }
-  });
-
-  // JavaScript to create the bar chart
-  var ctx = document.getElementById('category-chart').getContext('2d');
+  // Prepare the category data
   var categoryData = {
-    labels: [
-      {% for category in site.categories %}
-        {% capture category_name %}{{ category | first }}{% endcapture %}
-        "{{ category_name }}",
-      {% endfor %}
-    ],
+    labels: [{% for category in site.categories %}"{{ category | first }}",{% endfor %}],
     datasets: [{
       label: 'Number of Posts',
-      data: [
-        {% for category in site.categories %}
-          {{ site.categories[category_name].size }},
-        {% endfor %}
-      ],
-      backgroundColor: 'rgba(0, 123, 255, 0.5)',
-      borderColor: 'rgba(0, 123, 255, 1)',
+      data: [{% for category in site.categories %}{{ site.categories[category | first].size }},{% endfor %}],
+      backgroundColor: 'rgba(75, 192, 192, 0.2)',
+      borderColor: 'rgba(75, 192, 192, 1)',
       borderWidth: 1
     }]
   };
 
+  // Create the chart
+  var ctx = document.getElementById('categoryChart').getContext('2d');
   var categoryChart = new Chart(ctx, {
     type: 'bar',
     data: categoryData,
     options: {
-      responsive: true,
       scales: {
         y: {
-          beginAtZero: true,
-          ticks: {
-            precision: 0
-          }
+          beginAtZero: true
         }
       }
     }
   });
+
+  // JavaScript to show selected category
+  document.getElementById('category-select').addEventListener('change', function() {
+    var selectedCategory = this.value;
+    var contents = document.querySelectorAll('.category-content');
+    contents.forEach(function(content) {
+      content.style.display = 'none';
+    });
+    if (selectedCategory) {
+      document.querySelector(selectedCategory).style.display = 'block';
+    }
+  });
 </script>

From 7028c5087da4d559973aadef28db4d1f9f92b5d9 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 10:02:22 +0900
Subject: [PATCH 356/638] remove cat graph

---
 _pages/categories.md | 30 ------------------------------
 1 file changed, 30 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index b7786adc2..e47150437 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -51,9 +51,6 @@ title: Categories
   }
 </style>
 
-<!-- Chart Container -->
-<canvas id="categoryChart" width="400" height="200"></canvas>
-
 <div id="archives">
   <label for="category-select">Choose a category:</label>
   <select id="category-select" onchange="location = this.value;">
@@ -82,34 +79,7 @@ title: Categories
   </div>
 </div>
 
-<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 <script>
-  // Prepare the category data
-  var categoryData = {
-    labels: [{% for category in site.categories %}"{{ category | first }}",{% endfor %}],
-    datasets: [{
-      label: 'Number of Posts',
-      data: [{% for category in site.categories %}{{ site.categories[category | first].size }},{% endfor %}],
-      backgroundColor: 'rgba(75, 192, 192, 0.2)',
-      borderColor: 'rgba(75, 192, 192, 1)',
-      borderWidth: 1
-    }]
-  };
-
-  // Create the chart
-  var ctx = document.getElementById('categoryChart').getContext('2d');
-  var categoryChart = new Chart(ctx, {
-    type: 'bar',
-    data: categoryData,
-    options: {
-      scales: {
-        y: {
-          beginAtZero: true
-        }
-      }
-    }
-  });
-
   // JavaScript to show selected category
   document.getElementById('category-select').addEventListener('change', function() {
     var selectedCategory = this.value;

From c4c46e2b47a5d7781eb9f4f6fdd9decb221e6511 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 15:33:21 +0900
Subject: [PATCH 357/638] update projects page

---
 _pages/projects.md | 66 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 15 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index d4efbc620..c3ec9a5cb 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -4,20 +4,56 @@ permalink: /projects/
 title: Projects
 ---
 
-<div id="archives">
-{% for category in site.categories %}
-  <div class="archive-group">
-    {% capture category_name %}{{ category | first }}{% endcapture %}
-    {% if category_name=='HTML' %}
-    <div id="#{{ category_name | slugize }}"></div>
-    <p></p>
-    <a name="{{ category_name | slugize }}"></a>
-    {% for post in site.categories[category_name] %}
-    <article class="archive-item">
-      <h3><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h3>
-    </article>
-    {% endfor %}
-    {% endif %}
+<div id="projects">
+  <div class="project-item">
+    <h3>Project Title 1</h3>
+    <p>Short description of Project 1. Highlight key features, technologies used, or anything unique about the project.</p>
+    <a href="http://example.com/project1" target="_blank">View Project</a>
   </div>
-{% endfor %}
+  
+  <div class="project-item">
+    <h3>Project Title 2</h3>
+    <p>Short description of Project 2. Highlight key features, technologies used, or anything unique about the project.</p>
+    <a href="http://example.com/project2" target="_blank">View Project</a>
+  </div>
+
+  <div class="project-item">
+    <h3>Project Title 3</h3>
+    <p>Short description of Project 3. Highlight key features, technologies used, or anything unique about the project.</p>
+    <a href="http://example.com/project3" target="_blank">View Project</a>
+  </div>
+
+  <!-- Add more project items as needed -->
 </div>
+
+<style>
+  #projects {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 20px;
+  }
+
+  .project-item {
+    border: 1px solid #ccc;
+    padding: 20px;
+    border-radius: 8px;
+    width: 100%;
+    max-width: 300px;
+    box-shadow: 0 4px 8px rgba(0,0,0,0.1);
+  }
+
+  .project-item h3 {
+    margin-top: 0;
+  }
+
+  .project-item a {
+    display: inline-block;
+    margin-top: 10px;
+    color: #007BFF;
+    text-decoration: none;
+  }
+
+  .project-item a:hover {
+    text-decoration: underline;
+  }
+</style>

From 4cbc151d192b51c4eb921a83e7bd60a49a3b2f80 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 15:36:23 +0900
Subject: [PATCH 358/638] update projects page

---
 _pages/projects.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index c3ec9a5cb..5a5df1289 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -28,9 +28,7 @@ title: Projects
 
 <style>
   #projects {
-    display: flex;
-    flex-wrap: wrap;
-    gap: 20px;
+    display: block;
   }
 
   .project-item {
@@ -38,8 +36,8 @@ title: Projects
     padding: 20px;
     border-radius: 8px;
     width: 100%;
-    max-width: 300px;
     box-shadow: 0 4px 8px rgba(0,0,0,0.1);
+    margin-bottom: 20px;
   }
 
   .project-item h3 {

From 44bf1cf910927ec960946530c030ebbfe21f31ce Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 19:41:54 +0900
Subject: [PATCH 359/638] add projects to page

---
 _pages/projects.md | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 5a5df1289..72e8aafd9 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -6,24 +6,36 @@ title: Projects
 
 <div id="projects">
   <div class="project-item">
-    <h3>Project Title 1</h3>
-    <p>Short description of Project 1. Highlight key features, technologies used, or anything unique about the project.</p>
-    <a href="http://example.com/project1" target="_blank">View Project</a>
+    <h3>MLOps Architecture for Real-Time Graph Database Fraud Detection</h3>
+    <p>An AI-driven solution for real-time credit card fraud detection using MLOps techniques. It involves fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring. The system focuses on high fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models.
+    </p>
+    <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
+    <a href="https://github.com/divakaivan/kb_project" target="_blank">View on GitHub</a>
   </div>
   
   <div class="project-item">
-    <h3>Project Title 2</h3>
-    <p>Short description of Project 2. Highlight key features, technologies used, or anything unique about the project.</p>
-    <a href="http://example.com/project2" target="_blank">View Project</a>
+    <h3>MLOps Architecture for Insurance Fraud Detection</h3>
+    <p>This project is a capstone for DataTalksClub's MLOps Zoomcamp. It involves building an end-to-end MLOps pipeline to detect car insurance fraud. The pipeline integrates tools like Terraform, Prefect, MLflow, Docker, Grafana, and Evidently, covering aspects from data storage, data preprocessing (using IV and WoE), model training, deployment, and monitoring. The project emphasizes reproducibility, orchestration, and monitoring with a focus on achieving high recall in fraud detection using a Balanced Random Forest Classifier.
+    </p>
+    <p><b>Tech: PostgreSQL DB, Terraform, Google Cloud Platform, Mlflow, Prefect, Grafana, Evidently, Docker, FastAPI</b></p>
+    <a href="https://github.com/divakaivan/insurance-fraud-mlops-pipeline" target="_blank">View Project</a>
   </div>
 
   <div class="project-item">
-    <h3>Project Title 3</h3>
-    <p>Short description of Project 3. Highlight key features, technologies used, or anything unique about the project.</p>
-    <a href="http://example.com/project3" target="_blank">View Project</a>
+    <h3>Transaction Stream Data Engineering Pipeline</h3>
+    <p>This project starts by generating transaction data via Stripe's API, streams it using Apache Kafka, and processes it in real-time with PySpark Structured Streaming. The processed data is stored in PostgreSQL, managed by dbt for data transformations and modelling, and visualized using Grafana.
+    </p>
+    <p><b>Tech: PostgreSQL DB, Kafka, PySpark, dbt, Grafana</b></p>
+    <a href="https://github.com/divakaivan/transaction-stream-data-pipeline" target="_blank">View Project</a>
   </div>
 
-  <!-- Add more project items as needed -->
+  <div class="project-item">
+    <h3>Lending Club Data Engineering Pipeline</h3>
+    <p>This project involves building a data pipeline using various tools and technologies to process and visualize Lending Club data. The pipeline extracts raw data from Kaggle, loads it into Google Cloud Storage, processes it with dbt in BigQuery, and creates visualizations using Looker. Terraform is used for infrastructure management, and Mage orchestrates the entire process.
+    </p>
+    <p><b>Tech: Docker, Mage orchestration, Google Cloud Platform, Terraform, dbt, Looker</b></p>
+    <a href="https://github.com/divakaivan/lending-club-data-pipeline" target="_blank">View Project</a>
+  </div>
 </div>
 
 <style>

From 45634a51a618159f1816fd03acb64a81da1a0771 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 19:45:04 +0900
Subject: [PATCH 360/638] pr

---
 _pages/projects.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 72e8aafd9..3d88ef15d 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -6,7 +6,7 @@ title: Projects
 
 <div id="projects">
   <div class="project-item">
-    <h3>MLOps Architecture for Real-Time Graph Database Fraud Detection</h3>
+    <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
     <p>An AI-driven solution for real-time credit card fraud detection using MLOps techniques. It involves fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring. The system focuses on high fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models.
     </p>
     <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>

From 5494c4ec791e5dce74aaad456ae314ff3e816f03 Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 19:51:44 +0900
Subject: [PATCH 361/638] css change

---
 _pages/projects.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 3d88ef15d..4d6947273 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -45,7 +45,7 @@ title: Projects
 
   .project-item {
     border: 1px solid #ccc;
-    padding: 20px;
+    padding: 10px;
     border-radius: 8px;
     width: 100%;
     box-shadow: 0 4px 8px rgba(0,0,0,0.1);

From 32d71eca9ffecc51d42ce0365a26fc4d2d45647e Mon Sep 17 00:00:00 2001
From: divakaivan <timee98@outlook.com>
Date: Fri, 9 Aug 2024 20:21:57 +0900
Subject: [PATCH 362/638] day221

---
 _posts/2019-12-11-pullquotes.md               | 14 ---
 ...s-post-demonstrates-post-content-styles.md | 95 -------------------
 _posts/2024-08-09-day-221.md                  | 36 +++++++
 3 files changed, 36 insertions(+), 109 deletions(-)
 delete mode 100644 _posts/2019-12-11-pullquotes.md
 delete mode 100644 _posts/2019-7-27-this-post-demonstrates-post-content-styles.md
 create mode 100644 _posts/2024-08-09-day-221.md

diff --git a/_posts/2019-12-11-pullquotes.md b/_posts/2019-12-11-pullquotes.md
deleted file mode 100644
index 647f4edd1..000000000
--- a/_posts/2019-12-11-pullquotes.md
+++ /dev/null
@@ -1,14 +0,0 @@
----
-layout: post
-title:  Pullquotes
-categories: [HTML,Code]
-excerpt: In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic.
----
-
-In graphic design, a pull quote (also known as a lift-out pull quote) is a key phrase, quotation, or excerpt that has been pulled from an article and used as a page layout graphic element, serving to entice readers into the article or to highlight a key topic. {% include pullquote.html quote="It is typically placed in a larger or distinctive typeface and on the same page." %} Pull quotes are often used in magazine and newspaper articles, annual reports, and brochures, as well as on the web. They can add visual interest to text-heavy pages with few images or illustrations.
-
-Placement of a pull quote on a page may be defined in a publication's or website's style guide. Such a typographic device may or may not be aligned with a column on the page. Some designers, for example, choose not to align the quote. In that case, the quotation cuts into two or more columns, as in the example shown. Because the pull quote invites the reader to read about the highlighted material, the pull quote should appear before the text it cites and, generally, fairly close to it.
-
-Pull quotes need not be a verbatim copy of the text being quoted; depending on a publication's house style, pull quotes may be abbreviated for space or paraphrased for clarity, with or without indication.
-
-A disadvantage of pull quotes as a design element is that they can disrupt the reading process of readers invested in reading the text sequentially by drawing attention to ghost fragments out of context. At the other extreme, when pull quotes are used to break up what would otherwise be a formless wall of text, pull quote can serve as visual landmarks to help the reader maintain a sense of sequence and place.
\ No newline at end of file
diff --git a/_posts/2019-7-27-this-post-demonstrates-post-content-styles.md b/_posts/2019-7-27-this-post-demonstrates-post-content-styles.md
deleted file mode 100644
index 99e68c5fe..000000000
--- a/_posts/2019-7-27-this-post-demonstrates-post-content-styles.md
+++ /dev/null
@@ -1,95 +0,0 @@
----
-layout: post
-title: This post demonstrates post content styles
-categories: Miscellaneous
----
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce bibendum neque eget nunc mattis eu sollicitudin enim tincidunt. Vestibulum lacus tortor, ultricies id dignissim ac, bibendum in velit. Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-## Some great heading (h2)
-
-Proin convallis mi ac felis pharetra aliquam. Curabitur dignissim accumsan rutrum. In arcu magna, aliquet vel pretium et, molestie et arcu.
-
-Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris. Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc. Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-## Another great heading (h2)
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce bibendum neque eget nunc mattis eu sollicitudin enim tincidunt. Vestibulum lacus tortor, ultricies id dignissim ac, bibendum in velit.
-
-### Some great subheading (h3)
-
-Proin convallis mi ac felis pharetra aliquam. Curabitur dignissim accumsan rutrum. In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum.
-
-Phasellus et hendrerit mauris. Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc.
-
-### Some great subheading (h3)
-
-Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-> This quote will change your life. It will reveal the secrets of the universe, and all the wonders of humanity. Don't misuse it.
-
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce bibendum neque eget nunc mattis eu sollicitudin enim tincidunt.
-
-### Some great subheading (h3)
-
-Vestibulum lacus tortor, ultricies id dignissim ac, bibendum in velit. Proin convallis mi ac felis pharetra aliquam. Curabitur dignissim accumsan rutrum.
-
-```html
-<html>
-  <head>
-  </head>
-  <body>
-    <p>Hello, World!</p>
-  </body>
-</html>
-```
-
-
-In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris.
-
-#### You might want a sub-subheading (h4)
-
-In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris.
-
-In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris.
-
-#### But it's probably overkill (h4)
-
-In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris.
-
-### Oh hai, an unordered list!!
-
-In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris.
-
-- First item, yo
-- Second item, dawg
-- Third item, what what?!
-- Fourth item, fo sheezy my neezy
-
-### Oh hai, an ordered list!!
-
-In arcu magna, aliquet vel pretium et, molestie et arcu. Mauris lobortis nulla et felis ullamcorper bibendum. Phasellus et hendrerit mauris.
-
-1. First item, yo
-2. Second item, dawg
-3. Third item, what what?!
-4. Fourth item, fo sheezy my neezy
-
-
-
-## Headings are cool! (h2)
-
-Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc. Praesent varius interdum vehicula. Aenean risus libero, placerat at vestibulum eget, ultricies eu enim. Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-Praesent nulla tortor, malesuada adipiscing adipiscing sollicitudin, adipiscing eget est.
-
-Proin eget nibh a massa vestibulum pretium. Suspendisse eu nisl a ante aliquet bibendum quis a nunc.
-
-### Tables
-
-|Title 1               | Title 2               | Title 3               | Title 4              |
-|--------------------- | --------------------- | --------------------- | ---------------------|
-|lorem                 | lorem ipsum           | lorem ipsum dolor     | lorem ipsum dolor sit|
-|lorem ipsum dolor sit | lorem ipsum dolor sit | lorem ipsum dolor sit | lorem ipsum dolor sit|
-|lorem ipsum dolor sit | lorem ipsum dolor sit | lorem ipsum dolor sit | lorem ipsum dolor sit|
-|lorem ipsum dolor sit | lorem ipsum dolor sit | lorem ipsum dolor sit | lorem ipsum dolor sit|
diff --git a/_posts/2024-08-09-day-221.md b/_posts/2024-08-09-day-221.md
new file mode 100644
index 000000000..ad50f365a
--- /dev/null
+++ b/_posts/2024-08-09-day-221.md
@@ -0,0 +1,36 @@
+---
+layout: post
+title: (Day 221) Translating the KB Project info to Korean + New Blog!!!
+categories: []
+---
+
+## Hello :) Today is Day 221!
+A quick summary of today:
+* getting the project closer to submission
+* welcome to the new blog
+
+### Firstly - the new blog is ready
+
+This is the 1st post in the new blog. I cannot believe that I transferred ~100 posts yesterday. I like this new blog as it is in markdown, all on github, and the UI highly customisable. 
+
+I added some of my projects in the Projects tab at the top as well. There is also a 'hidden' categories page where the below stats can be seen:
+
+ADD IMAGE HERE
+
+This is a summary of the tags I used on my posts. I have not beed completely diligent on properly using them all the time, but it is a nice overview. 
+
+I will not delete the old blog, as it is still valuable, I am just going to create a post that points people to here.
+
+### As for the KB project
+
+I put the current `main` branch with full project info to a new branch called `main-en`, and on the [latest main branch](https://github.com/divakaivan/kb_project) I translated everything to Korean. 
+
+It is the same text, just in Korean so that it is better accepted by the competition assessors.
+
+### Final note - got my MLx certificate
+
+The final Oxford summer school course is 22-24 Aug and it is about [GenAI](https://www.oxfordml.school/genai) (Theory, Agents, Products) and I am really looking forward to that. 
+
+That is all for today!
+
+See you tomorrow :) 

From 74206ab60d21c4fdbda6f1af20903f3fefaac2a0 Mon Sep 17 00:00:00 2001
From: ivanstudyblog <167014511+ivanstudyblog@users.noreply.github.com>
Date: Fri, 9 Aug 2024 20:25:12 +0900
Subject: [PATCH 363/638] Update 2024-08-09-day-221.md

---
 _posts/2024-08-09-day-221.md | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/_posts/2024-08-09-day-221.md b/_posts/2024-08-09-day-221.md
index ad50f365a..fd13164c1 100644
--- a/_posts/2024-08-09-day-221.md
+++ b/_posts/2024-08-09-day-221.md
@@ -15,7 +15,7 @@ This is the 1st post in the new blog. I cannot believe that I transferred ~100 p
 
 I added some of my projects in the Projects tab at the top as well. There is also a 'hidden' categories page where the below stats can be seen:
 
-ADD IMAGE HERE
+<img width="272" alt="image" src="https://github.com/user-attachments/assets/6d607924-018d-4963-9fb1-0963dcfcb038">
 
 This is a summary of the tags I used on my posts. I have not beed completely diligent on properly using them all the time, but it is a nice overview. 
 
@@ -27,9 +27,21 @@ I put the current `main` branch with full project info to a new branch called `m
 
 It is the same text, just in Korean so that it is better accepted by the competition assessors.
 
-### Final note - got my MLx certificate
+### Final note - got my 2nd MLx certificate
 
-The final Oxford summer school course is 22-24 Aug and it is about [GenAI](https://www.oxfordml.school/genai) (Theory, Agents, Products) and I am really looking forward to that. 
+![image](https://github.com/user-attachments/assets/8f52b1bd-0336-47c4-ba06-bf0d5c96d7e0)
+
+The final Oxford summer school course is 22-24 Aug and it is about [GenAI](https://www.oxfordml.school/genai) (Theory, Agents, Products) and I am really looking forward to that. Here are the speakers:
+
+<img width="1455" alt="image" src="https://github.com/user-attachments/assets/32f22e0c-e647-419e-9aca-6b5ac947eae2">
+
+And topics:
+* Advanced theoretical topics in representation learning (e.g., vision, language, multi-modal, …)
+* Agentic AI (e.g., agentic reasoning and design patterns)
+* Human+AI alignment
+* Building Gen. AI products — from model.fit() to market.fit()
+* Using SOTA foundation models (e.g., fine tuning, RAG, RLHF, prompt engineering, …)
+* Application of large frontier models in applied domains (e.g., medicine, finance, education, …)
 
 That is all for today!
 

From 7a87c3a9820079e3fb0bbd01d3fa744954eddb7f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 9 Aug 2024 20:32:06 +0900
Subject: [PATCH 364/638] Update 2024-08-09-day-221.md

---
 _posts/2024-08-09-day-221.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/_posts/2024-08-09-day-221.md b/_posts/2024-08-09-day-221.md
index fd13164c1..ed9dabe0e 100644
--- a/_posts/2024-08-09-day-221.md
+++ b/_posts/2024-08-09-day-221.md
@@ -27,7 +27,7 @@ I put the current `main` branch with full project info to a new branch called `m
 
 It is the same text, just in Korean so that it is better accepted by the competition assessors.
 
-### Final note - got my 2nd MLx certificate
+### Got my 2nd MLx certificate
 
 ![image](https://github.com/user-attachments/assets/8f52b1bd-0336-47c4-ba06-bf0d5c96d7e0)
 
@@ -43,6 +43,11 @@ And topics:
 * Using SOTA foundation models (e.g., fine tuning, RAG, RLHF, prompt engineering, …)
 * Application of large frontier models in applied domains (e.g., medicine, finance, education, …)
 
+### Found OmdenaAI
+
+This company has [open projects](https://www.omdena.com/search?query=projects&focus=projects) where people can apply and (if selected) work as volunteers. I will keep an eye and apply if something interesting comes up.
+
+
 That is all for today!
 
 See you tomorrow :) 

From 294d34d462cf4e20112d319fc8ccfe33a3113f67 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 10 Aug 2024 11:40:27 +0900
Subject: [PATCH 365/638] add google analytics tag

---
 _layouts/default.html | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 186a26cbb..ebe1a1ef3 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -13,7 +13,15 @@
     <link rel="icon" type="image/png" sizes="32x32" href="{{ site.baseurl }}/images/favicon-32x32.png">
     {% include analytics_head.html %}
   </head>
-
+  <!-- Google tag (gtag.js) -->
+  <script async src="https://www.googletagmanager.com/gtag/js?id=G-WWES1B0RFM"></script>
+  <script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+  
+    gtag('config', 'G-WWES1B0RFM');
+  </script>
   <body>
     <div id="bar"></div>
     <div class="wrapper-container">

From 654cae5da1b5f24d1ad8a53509a3992543966248 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 10 Aug 2024 11:42:26 +0900
Subject: [PATCH 366/638] Update _config.yml

---
 _config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_config.yml b/_config.yml
index 3c07e4e1c..d9087f17a 100644
--- a/_config.yml
+++ b/_config.yml
@@ -41,10 +41,10 @@ footer-links:
 disqus: 
 
 # Enter your Google Analytics web tracking code (e.g. UA-2110908-2) to activate tracking
-google_analytics: UA-43339302-11
+# google_analytics: UA-43339302-11
 
 # For newer "GA4" analytics, use the following instead of the "UA" entry above
-#google_analytics_ga4: G-GABC1DEFG
+google_analytics_ga4: G-WWES1B0RFM
 
 # Your website URL (e.g. http://amitmerchant1990.github.io or http://www.amitmerchant.com)
 # Used for Sitemap.xml and your RSS feed

From 60528a2013f9ef91454ad17d45a2589fee05099d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 10 Aug 2024 15:33:23 +0900
Subject: [PATCH 367/638] Create 2024-08-10-day-222.md

---
 _posts/2024-08-10-day-222.md | 75 ++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 _posts/2024-08-10-day-222.md

diff --git a/_posts/2024-08-10-day-222.md b/_posts/2024-08-10-day-222.md
new file mode 100644
index 000000000..2d689035a
--- /dev/null
+++ b/_posts/2024-08-10-day-222.md
@@ -0,0 +1,75 @@
+---
+layout: post
+title: (Day 222) Chapter 3 of Fundamentals of Data Engineering
+categories: [data-eng,theory]
+---
+
+## Hello :) Today is Day 222!
+A quick summary of today:
+* read chapter 3: Designing Good Data Architecture of the Data Eng book
+
+
+### What is Data Architecture ?
+
+A definition given by the book: _Data architecture is the design of systems to support the evolving data needs of an
+enterprise, achieved by flexible and reversible decisions reached through a careful
+evaluation of trade-offs._
+
+#### Operational vs Technical data architecture
+
+<img width="967" alt="image" src="https://github.com/user-attachments/assets/50032d39-3a18-44e6-b137-25cb4efdf4bb">
+
+_Operational architecture encompasses the functional require‐
+ments of what needs to happen related to people, processes, and technology. For
+example, what business processes does the data serve? How does the organization
+manage data quality? What is the latency requirement from when the data is pro‐
+duced to when it becomes available to query? Technical architecture outlines how data
+is ingested, stored, transformed, and served along the data engineering lifecycle. For
+instance, how will you move 10 TB of data every hour from a source database to your
+data lake? In short, operational architecture describes what needs to be done, and
+technical architecture details how it will happen._
+
+**Good Data Architecture**
+
+Good data architecture serves business requirements with
+a common, widely reusable set of building blocks while maintaining flexibility and
+making appropriate trade-offs. Bad architecture is authoritarian and tries to cram a
+bunch of one-size-fits-all decisions into a big ball of mud.
+
+### Principles of Good Data Architecture
+
+Following the [AWS Well-Architected Framework](https://docs.aws.amazon.com/wellarchitected/latest/framework/welcome.html) consists of six pillars:
+* Operational excellence
+* Security
+* Reliability
+* Performance efficiency
+* Cost optimization
+* Sustainability
+
+Or [Google Cloud’s Five Principles for Cloud-Native Architecture](https://cloud.google.com/blog/products/application-development/5-principles-for-cloud-native-architecture-what-it-is-and-how-to-master-it) are as follows:
+* Design for automation.
+* Be smart with state.
+* Favor managed services.
+* Practice defense in depth.
+* Always be architecting.
+
+The authors expand on these principles by adding:
+1. Choose common components wisely.
+2. Plan for failure.
+3. Architect for scalability.
+4. Architecture is leadership.
+5. Always be architecting.
+6. Build loosely coupled systems.
+7. Make reversible decisions.
+8. Prioritize security.
+9. Embrace FinOps.
+
+
+
+
+
+
+
+
+
+

From 721df8924e01ff0f2e9f00c051b30c287ddf5439 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 10 Aug 2024 20:59:43 +0900
Subject: [PATCH 368/638] Update 2024-08-10-day-222.md

---
 _posts/2024-08-10-day-222.md | 140 ++++++++++++++++++++++++++++++++++-
 1 file changed, 138 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-10-day-222.md b/_posts/2024-08-10-day-222.md
index 2d689035a..30a144fd7 100644
--- a/_posts/2024-08-10-day-222.md
+++ b/_posts/2024-08-10-day-222.md
@@ -1,14 +1,17 @@
 ---
 layout: post
-title: (Day 222) Chapter 3 of Fundamentals of Data Engineering
-categories: [data-eng,theory]
+title: (Day 222) Fundamentals of Data Engineering and Introducing MLOps in O'Reilly
+categories: [data-eng,theory,mlops]
 ---
 
 ## Hello :) Today is Day 222!
 A quick summary of today:
 * read chapter 3: Designing Good Data Architecture of the Data Eng book
+* started Introducing MLOps
 
 
+## Fundamentals of Data Engineering Chapter 3
+
 ### What is Data Architecture ?
 
 A definition given by the book: _Data architecture is the design of systems to support the evolving data needs of an
@@ -64,12 +67,145 @@ The authors expand on these principles by adding:
 8. Prioritize security.
 9. Embrace FinOps.
 
+### Major Architecture Concepts
+
+Pre-reqs: domain and service
+
+* A domain is the real-world subject area for which you’re architecting.
+* A service is a set of functionality whose goal is to accomplish a task
+
+As data engineers, we’re interested in four closely related characteristics of data systems:
+
+1. Scalability: Allows us to increase the capacity of a system to improve performance and
+handle the demand. For example, we might want to scale a system to handle a
+high rate of queries or process a huge data set.
+
+2. Elasticity: The ability of a scalable system to scale dynamically; a highly elastic system can
+automatically scale up and down based on the current workload. Scaling up is
+critical as demand increases, while scaling down saves money in a cloud environ‐
+ment. Modern systems sometimes scale to zero, meaning they can automatically
+shut down when idle.
+
+3. Availability: The percentage of time an IT service or component is in an operable state.
+
+4. Reliability: The system’s probability of meeting defined standards in performing its intended
+function during a specified interval.
+
+#### Tight vs Loose Coupling
+
+When designing data architecture, the choice between tight and loose coupling impacts the interdependence of domains, services, and resources. Tight coupling centralizes dependencies, creating highly interconnected systems, while loose coupling decentralizes components, reducing dependencies but requiring clear standards and accountability. Architectures can be single-tier (tightly coupled, prone to failures) or multitier (decoupled, more reliable). Monolithic architectures bundle services tightly, making changes difficult, while microservices promote decentralized, loosely coupled services, allowing independent functionality. Monoliths may be simpler initially but often require gradual decomposition into microservices. Adopting loose coupling and modularity, with an awareness of technological limitations, is ideal.
+
+### Brownfield Versus Greenfield Projects
+
+When designing data architecture, the approach depends on whether you're working on a brownfield or greenfield project. 
+
+**Brownfield projects** involve refactoring existing architecture, requiring an understanding of legacy systems and careful change management. The "strangler pattern" is often preferred, gradually replacing old components to avoid risks and allow reversible decisions.
+
+**Greenfield projects** offer a fresh start, allowing the use of new technologies without legacy constraints. However, it's crucial to avoid "shiny object syndrome" and focus on meeting project goals rather than just using the latest trends.
+
+In both cases, prioritize good architecture principles, flexible decisions, and a positive ROI.
+
+### Examples and Types of Data Architecture
+
+**Data Warehouse**
+
+A data warehouse is a central data hub used for reporting and analysis. Data in a data warehouse is typically highly formatted and structured for analytics use cases. It’s among the oldest and most well-established data architectures.
+
+<img width="924" alt="image" src="https://github.com/user-attachments/assets/47a25037-dc76-4300-9572-62b6cda5f11c">
+
+<img width="969" alt="image" src="https://github.com/user-attachments/assets/c05d9746-33e8-4dd1-b5d9-8803e49f0e97">
+
+
+**First-gen Data Lake**
+
+The data lake emerged during the big data era as a popular architecture, allowing the storage of vast amounts of structured and unstructured data in a central location without tight structural constraints. The first-generation data lake, known as "data lake 1.0," began with HDFS and later moved to cloud-based storage, offering cheap, limitless capacity and flexible processing options. However, it failed to deliver on its promise, becoming a "data swamp" due to poor schema management, data cataloging, and discovery tools. Processing data was cumbersome, and managing Hadoop clusters became expensive, requiring large teams of specialized engineers. While data lakes provided value for tech giants like Netflix and Facebook, many organizations faced spiraling costs and unmanageable data, leading to disappointment.
+
+**Convergence, Next-Generation Data Lakes, and the Data Platform**
+
+In response to the shortcomings of first-generation data lakes, new approaches like the data lakehouse have emerged. Databricks pioneered the lakehouse concept, which integrates the robust data management and ACID transaction support of data warehouses with the flexible storage of data lakes. This marks a convergence between data lakes and data warehouses. Cloud data warehouses have also evolved to resemble data lakes, supporting large-scale queries, diverse data types, and advanced processing technologies. The trend is moving towards unified data platforms that blend the capabilities of both architectures, offered by leading vendors like AWS, Azure, Google Cloud, Snowflake, and Databricks. Future data engineers will likely choose these converged platforms based on specific needs rather than between distinct architectures.
+
+### Modern Data Stack
+
+<img width="971" alt="image" src="https://github.com/user-attachments/assets/ad899b29-d332-4087-b146-7aba79d5fb72">
+
+Key outcomes of the modern data stack are self-service (analytics and pipelines), agile data management, and using open source tools or simple proprietary tools with clear pricing structures
+
+### Lambda Architecture
+
+<img width="977" alt="image" src="https://github.com/user-attachments/assets/8df72fc2-5b35-4380-9791-601603bbf516">
+
+In a Lambda architecture, you have systems operating independently of each other—batch, streaming, and serving. The source system is ideally immutable and append-only, sending data to two destinations for processing: stream and batch. In-stream processing intends to serve the data with the lowest possible latency in a “speed” layer, usually a NoSQL database. In the batch layer, data is processed and transformed in a system such as a data warehouse, creating precomputed and aggre‐ gated views of the data. The serving layer provides a combined view by aggregating query results from the two layers.
+
+### Kappa Architecture
+
+<img width="976" alt="image" src="https://github.com/user-attachments/assets/df5d3cdb-3def-48d6-b83c-e631ce37ec8f">
+
+The central thesis is this: why not just use a stream-processing platform as the backbone for all data handling—inges‐ tion, storage, and serving? This facilitates a true event-based architecture. Real-time and batch processing can be applied seamlessly to the same data by reading the live event stream directly and replaying large chunks of data for batch processing.
+
+### The Dataflow Model and Unified Batch and Streaming
+
+Lambda and Kappa architectures attempted to unify batch and streaming data processing by integrating various tools within the Hadoop ecosystem, though not seamlessly. A key challenge was managing different code paths for batch and stream processing. Kappa used a unified queuing and storage layer but still required different tools for real-time and batch tasks. Modern approaches, like Google's Dataflow model and Apache Beam framework, view all data as events, treating batches as bounded streams. This philosophy, where "batch is a special case of streaming," is now widespread, with frameworks like Flink and Spark adopting similar strategies.
+
+
+## [Introducing MLOps](https://learning.oreilly.com/library/view/introducing-mlops/9781492083283/) Chapter 1: MLOps What and Why
+
+###### I had my eyes on a few books in O'Reilly for a bit (this, Deep Learning at Scale, and Machine Learning Production Systems), but I did not want to sub to them. Today I found that my university supplies a free sub to their materials so now I have access to all books on the website 🥳
+
+MLOps interest over time. (fyi the book is released in 2020)
+
+![image](https://github.com/user-attachments/assets/8d97dfaf-ec59-40f6-aa3c-f43d24a38e1d)
+
+At its core, MLOps is the standardization and streamlining of machine learning life cycle management.
+
+![image](https://github.com/user-attachments/assets/33ce2be0-3811-4808-829c-7125fa9b95af)
+
+Managing machine learning models is challenging due to constantly changing data, shifting business needs, and the need to ensure alignment between model performance and business goals. Communication gaps exist between business, data science, and IT teams, as they use different tools and lack common skills, complicating collaboration. Additionally, data scientists often lack software engineering expertise, leading to difficulties in managing models at scale, especially as team turnover increases and they are required to handle models they didn't originally develop.
+
+![image](https://github.com/user-attachments/assets/d15fae38-abd5-4d0b-b502-14c90a4f5cff)
+
+MLOps pulls knowledge from DevOps:
+
+* Robust automation and trust between teams
+* The idea of collaboration and increased communication between teams
+* The end-to-end service life cycle (build, test, release)
+* Prioritizing continuous delivery and high quality
+
+### MLOps to Mitigate Risk
 
+Deploying machine learning models without MLOps infrastructure is risky because true model performance can only be accurately assessed in the production environment, where data may differ from the training environment. Model performance is sensitive to the production environment, including software and operating system versions, making it crucial to match these with the verified setup. Deploying models is just the beginning; continuous monitoring is essential to ensure models behave as expected, making MLOps critical to mitigate risks and maintain business stability.
 
+### MLOps for Scale
 
+MLOps isn’t just important because it helps mitigate the risk of machine learning models in production; it is also an essential component to massively deploying machine learning efforts (and in turn benefiting from the corresponding economies of scale). Going from one or a handful of models in production to tens, hundreds, or thousands that have a positive business impact requires MLOps discipline.
 
+Good MLOps practices will help teams at a minimum:
+* Keep track of versioning, especially with experiments in the design phase
+* Understand whether retrained models are better than the previous versions (and promoting models to production that are performing better)
+* Ensure (at defined periods—daily, monthly, etc.) that model performance is not degrading in production
 
+## Introducing MLOps Chapter 2: People of MLOps
 
+| Role                        | Role in Machine Learning Model Life Cycle                                                                                                                                                    | MLOps Requirements                                                                                                                                    |
+|-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Subject Matter Experts**  | Provide business questions, goals, or KPIs around which ML models should be framed.                                                                                                         | Easy way to understand deployed model performance in business terms.                                                                                  |
+|                             | Continually evaluate and ensure that model performance aligns with or resolves the initial need.                                                                                            | Mechanism or feedback loop for flagging model results that don’t align with business expectations.                                                    |
+| **Data Scientists**         | Build models that address the business question or needs brought by subject matter experts.                                                                                                 | Automated model packaging and delivery for quick and easy (yet safe) deployment to production.                                                        |
+|                             | Deliver operationalizable models so that they can be properly used in the production environment and with production data.                                                                  | Ability to develop tests to determine the quality of deployed models and to make continual improvements.                                              |
+|                             | Assess model quality (of both original and tests) in tandem with subject matter experts to ensure they answer initial business questions or needs.                                           | Visibility into the performance of all deployed models (including side-by-side for tests) from one central location.                                   |
+|                             |                                                                                                                                                                                             | Ability to investigate data pipelines of each model to make quick assessments and adjustments regardless of who originally built the model.            |
+| **Data Engineers**          | Optimize the retrieval and use of data to power ML models.                                                                                                                                  | Visibility into performance of all deployed models.                                                                                                   |
+|                             |                                                                                                                                                                                             | Ability to see the full details of individual data pipelines to address underlying data plumbing issues.                                               |
+| **Software Engineers**      | Integrate ML models in the company’s applications and systems.                                                                                                                              | Versioning and automatic tests.                                                                                                                       |
+|                             | Ensure that ML models work seamlessly with other non-machine-learning-based applications.                                                                                                   | The ability to work in parallel on the same application.                                                                                               |
+| **DevOps**                  | Conduct and build operational systems and test for security, performance, availability.                                                                                                     | Continuous Integration/Continuous Delivery (CI/CD) pipeline management.                                                                               |
+|                             |                                                                                                                                                                                             | Seamless integration of MLOps into the larger DevOps strategy of the enterprise.                                                                      |
+|                             |                                                                                                                                                                                             | Seamless deployment pipeline.                                                                                                                         |
+| **Model Risk Managers/Auditors** | Minimize overall risk to the company as a result of ML models in production.                                                                                                              | Robust, likely automated, reporting tools on all models (currently or ever in production), including data lineage.                                     |
+|                             | Ensure compliance with internal and external requirements before pushing ML models to production.                                                                                           |                                                                                                                                                       |
+| **Machine Learning Architects** | Ensure a scalable and flexible environment for ML model pipelines, from design to development and monitoring.                                                                              | High-level overview of models and their resources consumed.                                                                                           |
+|                             | Introduce new technologies when appropriate that improve ML model performance in production.                                                                                                | Ability to drill down into data pipelines to assess and adjust infrastructure needs.                                                                   |
 
 
+That is all for today!
 
+See you tomorrow :)

From a55efe1bf7427a86bda28b921bc812fa7e73ce0c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 10 Aug 2024 22:15:05 +0900
Subject: [PATCH 369/638] Update 2024-08-10-day-222.md

---
 _posts/2024-08-10-day-222.md | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/_posts/2024-08-10-day-222.md b/_posts/2024-08-10-day-222.md
index 30a144fd7..bf631bc02 100644
--- a/_posts/2024-08-10-day-222.md
+++ b/_posts/2024-08-10-day-222.md
@@ -206,6 +206,41 @@ Good MLOps practices will help teams at a minimum:
 |                             | Introduce new technologies when appropriate that improve ML model performance in production.                                                                                                | Ability to drill down into data pipelines to assess and adjust infrastructure needs.                                                                   |
 
 
+## Introducing MLOps Chapter 3: Key MLOps Features
+
+1. Model Development
+  * establishing business objectives
+  * data sourcing and EDA
+  * feature engineering and selection
+  * training and evaluation
+  * reproducability
+  * responsible AI
+2. Productionalization and Deployment
+  * Model-as-a-service, or live-scoring model
+  * Embedded model
+3. Monitoring
+  * Is the model getting the job done quickly enough?
+  * Is it using a sensible amount of memory and processing time?
+  * Is the model's performance degrading?
+  * Is the model delivering value to the enterprise?
+  * Do the benefits of the model outweigh the cost of developing and deploying it? (And how can we measure this?)
+4. Iteration and Life Cycle: Developing and deploying improved versions of a model is an essential part of the MLOps life cycle, and one of the more challenging. There are various reasons to develop a new model version, one of which is model performance degradation due to model drift, as discussed in the prior section. Sometimes there is a need to reflect refined business objectives and KPIs, and other times, it’s just that the data scientists have come up with a better way to design the model.
+5. Governance: Governance is the set of controls placed on a business to ensure that it delivers on its responsibilities to all stakeholders, from shareholders and employees to the public and national governments. These responsibilities include financial, legal, and ethical obligations. Underpinning all three of these is the fundamental principle of fairness.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 That is all for today!
 
 See you tomorrow :)

From 55270cdf469c609b20126f801a5907dc02f429bb Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 11 Aug 2024 23:07:55 +0900
Subject: [PATCH 370/638] Create 2024-08-11-day-223.md

---
 _posts/2024-08-11-day-223.md | 433 +++++++++++++++++++++++++++++++++++
 1 file changed, 433 insertions(+)
 create mode 100644 _posts/2024-08-11-day-223.md

diff --git a/_posts/2024-08-11-day-223.md b/_posts/2024-08-11-day-223.md
new file mode 100644
index 000000000..d7859a57c
--- /dev/null
+++ b/_posts/2024-08-11-day-223.md
@@ -0,0 +1,433 @@
+---
+layout: post
+title: (Day 223) Finishing up Introducing MLOps
+categories: [mlops,theory,data-eng]
+---
+
+# Hello :) Today is Day 223!
+A quick summary of today:
+* read the last few chapters of Introducing MLOps
+* read chapter 4 of the fundamentals of data eng book
+* created a short demo video and submitted the KB AI competition project 
+
+## Introducing MLOps Chapter 4: Developing Models
+
+The first step in the MLOps cycle
+
+<img width="799" alt="image" src="https://github.com/user-attachments/assets/f04ea2f3-e305-4c97-b9f8-9ed11ecd4c43">
+
+### In Theory
+- **Representation**: A machine learning model is an approximation of reality, representing certain aspects of a real-world process.
+- **Foundation**: Models are built using statistical theory and are crafted by algorithms from training data.
+- **Generalization**: The ability to predict unseen cases based on patterns learned from training data is known as generalization capacity.
+
+### In Practice
+- **Model Components**: A model comprises parameters that transform input data into predictions. This includes mathematical functions and derived data.
+- **Application Example**: Predicting house prices based on factors like area, bedrooms, and market conditions.
+- **Outputs**: Outputs can vary from single numbers to probabilities with confidence intervals, or even recommendations.
+
+### Required Components
+- **Training Data**: Labeled data used to train the model.
+- **Performance Metric**: The metric the model optimizes (e.g., accuracy, precision).
+- **ML Algorithm**: The method used to build the model, with different algorithms suited to different tasks.
+- **Hyperparameters**: Configurations for algorithms that influence how models learn.
+- **Evaluation Dataset**: Data separate from the training set to test the model's generalization.
+
+### MLOps Considerations by Algorithm Type
+- **Linear Models**: Susceptible to overfitting.
+- **Tree-based Models**: Can be unstable; some, like Random Forests, are hard to interpret.
+- **Deep Learning**: Highly complex and resource-intensive, with low interpretability.
+
+### Computing Power
+- **Role in ML**: The evolution of ML has been tied to the availability of computing power, impacting algorithm complexity and usage.
+
+### Data Exploration
+- **Initial Analysis**: Involves understanding data through documentation, statistics, distribution analysis, and comparison to other models or datasets.
+
+### Feature Engineering and Selection
+- **Purpose**: Transform raw data into features that better represent the problem to the model.
+- **Techniques**: Includes derivatives, enrichment, encoding, and combinations of features.
+- **Impact**: The number and quality of features directly affect model performance and MLOps strategy.
+
+### Experimentation
+- **Process**: Iterative testing and adjustment of model components, hyperparameters, and features to optimize performance.
+- **Bias/Variance Trade-off**: Balancing between underfitting (high bias) and overfitting (high variance).
+
+### Evaluating and Comparing Models
+- **Usefulness Over Perfection**: Models should be judged on their practical utility rather than aiming for perfection.
+- **Evaluation Metrics**: Metrics should be chosen carefully to reflect the problem at hand, with methods like cross-validation ensuring robustness.
+- **Model Behavior**: Beyond metrics, understanding how models react to different inputs and across subpopulations is critical.
+
+
+## Introducint MLOps Chapter 5 Preparing for Production
+
+![image](https://github.com/user-attachments/assets/4aeae079-71c0-4523-b52f-cff8767a38d2)
+
+### Runtime Environments in MLOps
+
+- **Diverse Production Environments**: Models need to be adaptable to various environments like custom services, data platforms, or Kubernetes. Ideally, a model runs in production as it did in development, but this is often challenging.
+
+- **Adaptation Challenges**: Moving from development to production may require anything from minimal changes to complete reimplementation, potentially delaying deployment by months or years.
+
+- **Tooling**: Early consideration of production formats (e.g., PMML, ONNX) is crucial. Tools should be integrated during development to avoid blocking the validation process.
+
+- **Performance Optimization**: Conversion for performance (e.g., Python to C++) might be necessary, especially for models running on low-power devices. Techniques like quantization, pruning, and distillation can reduce model size and improve speed.
+
+- **Data Access**: Models may need real-time data access in production. Proper setup of data connections and validation in a production-like environment is critical to avoid malfunctions.
+
+- **Final Thoughts**: While training is computationally intensive, inference is more resource-demanding over time. Simplifying models for production can reduce costs and environmental impact, offering a viable alternative to complex models.
+
+### Model Risk Evaluation
+
+#### Purpose of Model Validation
+- Models are imperfect and can have bugs, leading to real-world consequences.
+- Validation is essential to anticipate and minimize risks in production.
+- Organizations must consider risks like malfunctions, attacks, financial, legal, and reputational damage.
+
+#### Key Questions Before Production
+- Worst-case scenario for model behavior?
+- Potential for data or logic extraction?
+- Financial, business, legal, safety, and reputational risks?
+
+#### Importance of Risk Awareness
+- High-risk applications require rigorous validation.
+- Machine learning introduces unique risks, needing careful management.
+
+#### Origins of ML Model Risk
+- Risks arise from:
+  - Design, training, or evaluation bugs.
+  - Runtime framework bugs or incompatibilities.
+  - Low-quality training data.
+  - Differences between training and production data.
+  - Misuse or misinterpretation.
+  - Adversarial attacks.
+  - Legal and reputational issues.
+- Risks are amplified by:
+  - Broad model use.
+  - Rapidly changing environments.
+  - Complex model interactions.
+
+### Quality Assurance for Machine Learning
+
+- **Maturity**: QA in traditional software is mature; for ML, it's still evolving and challenging to scale in MLOps.
+- **Continuous QA**: QA should occur throughout the model development process, not just at the final stage.
+- **Validation**: Those responsible for validation need ML expertise and authority to address risks and halt production if necessary.
+- **Documentation**: QA also involves documenting and validating models against organizational guidelines, including data origin and security.
+  
+### Key Testing Considerations
+
+- **Test Data**: Test scenarios should include both realistic and problematic data (e.g., extreme values).
+- **Metrics**: Collect both statistical and computational metrics; tests should fail if key assumptions are not met.
+- **Stability**: Model stability (small input changes should cause small output changes) is crucial; simpler models often exhibit better stability.
+
+### Subpopulation Analysis and Model Fairness
+
+- **Fairness**: Assess fairness by analyzing subpopulations, particularly for sensitive variables.
+- **Impact**: Lack of fairness evaluation can lead to business, regulatory, and reputational risks.
+
+### Reproducibility and Auditability
+
+- **Reproducibility**: In MLOps, reproducibility involves rerunning experiments with the same data, code, and environment.
+- **Auditability**: Requires access to full model history, including documentation, artifacts, test results, and logs.
+
+### Machine Learning Security
+
+- **Adversarial Attacks**: Small input modifications can cause drastic prediction changes in models, especially in deep learning.
+- **Training Data Poisoning**: Manipulating training data can control model behavior (e.g., Microsoft’s 2016 Twitter bot incident).
+- **Governance**: Security must be embedded from the start; feedback mechanisms and governance structures are crucial.
+
+### Model Risk Mitigation
+
+- **Deployment**: Control deployment with progressive rollouts; monitor for behavior changes.
+- **Changing Environments**: Fast-changing inputs can lead to significant risks; models may need to operate in a degraded mode during remediation.
+- **Interactions Between Models**: Complexity grows with the number of models and their interactions; design independent processing chains to reduce risk.
+
+### Model Misbehavior
+
+- **Input/Output Monitoring**: Implement real-time checks on feature values and outputs to detect misbehavior.
+- **Anomaly Detection**: Use anomaly detection and certainty scores to validate predictions, especially in high-risk models.
+
+## Introducing MLOps Chapter 6: Deploying to Production
+
+![image](https://github.com/user-attachments/assets/41b236e2-6151-46d8-b76a-b60bd2c9b614)
+
+### CI/CD Pipelines Overview
+
+**CI/CD (Continuous Integration/Delivery):**
+- **CI:** Integrate and test code frequently, catching bugs early.
+- **CD:** Automate deployment of tested code to production, reducing manual intervention.
+
+### Key Components:
+1. **Centralized Version Control:** Git for code; additional tools for data/versioning.
+2. **Artifacts:** Bundled code, model, data, environment, and tests ready for deployment.
+3. **Testing Pipeline:** Automated tests to ensure model performance and compliance.
+
+### Deployment Strategies
+- **Integration:** Merging code changes and testing.
+- **Delivery:** Ready-to-deploy, validated models.
+- **Deployment:** Running models on target infrastructure.
+- **Release:** Controlling when the new model handles production workload.
+
+### Deployment Approaches
+- **Batch Scoring:** Process entire datasets (e.g., daily jobs).
+- **Real-time Scoring:** Handle individual requests (e.g., user interactions).
+
+### Model Deployment Strategies
+- **Blue-Green:** Run new and old versions in parallel, switch to new once stable.
+- **Canary Release:** Gradually shift traffic to a new version, monitoring performance.
+
+### Maintenance
+- **Resource Monitoring:** Track CPU, memory, etc.
+- **Health Checks:** Ensure the model is online and responsive.
+- **ML Metrics Monitoring:** Track model accuracy and relevance over time.
+
+### Containerization
+- **Docker:** Packages applications with all dependencies, ensuring consistency.
+- **Kubernetes:** Orchestrates containers, manages scaling, updates, and failures.
+
+### Scaling Challenges 
+- **Horizontal Scaling:** Add more machines to handle increased loads.
+- **Elastic Systems:** Automatically adjust resources based on demand.
+
+### Batch Processing
+- **Spark with Kubernetes:** Distributes computation across multiple nodes.
+
+### Managing Multiple Models
+- **Standardized CI/CD Pipelines:** Ensures consistency and efficiency across deployments.
+- **Tools:** E.g., Spinnaker for managing complex deployment scenarios.
+
+### Logging Requirements
+- Centralized logging for tracking model performance and identifying issues across multiple servers and model versions.
+
+## Introducing MLOps Chapter 7: Monitoring and Feedback Loop
+
+![image](https://github.com/user-attachments/assets/d14b1e8d-3e85-49cd-98fa-3867654e3d43)
+
+### Key Factors:
+- **Domain**: Fields like cybersecurity need frequent updates; stable domains, less so.
+- **Cost**: Evaluate if retraining costs justify performance gains.
+- **Model Performance**: Retrain based on data sufficiency and performance monitoring.
+
+### Organizational Boundaries:
+- **Upper Bound**: At least once a year for skill retention and toolchain readiness.
+- **Lower Bound**: Likely not more than once a day, depending on feedback speed.
+
+### Online Learning:
+- Costly but necessary for streaming data scenarios.
+- Requires rigorous testing due to iterative learning.
+
+### Drift Detection:
+- **Input Drift**: Monitor feature values to detect performance changes.
+- **Techniques**: Use univariate statistical tests or domain classifiers to detect drift.
+
+### Feedback Loop:
+- **Components**: Logging system, model evaluation store, and online model comparison.
+- **Methods**:
+  - **Champion/Challenger**: Shadow testing for model comparison.
+  - **A/B Testing**: Used when direct comparison isn't possible, e.g., ad engines.
+
+### Conclusion:
+- Retraining is essential, with frequency determined by the domain, cost, and model performance. Continuous monitoring and a robust feedback loop are critical.
+
+## Introducing MLOps Chapter 8: Model Governance
+
+Model governance goals aim to ensure that the business delivers on its responsibilities to all stakeholders, from shareholders and employees to the public and national governments. The responsibilities include financial, legal, and ethical, and are all underpinned by the desire for fairness.
+
+### Financial Model Risk Management Regulation
+
+In finance, model risk is the risk of incurring losses when the models used for making decisions about tradable assets prove to be inaccurate. These models, such as the Black–Scholes model, existed long before the arrival of ML.
+
+The UK Prudential Regulation Authority’s (PRA) regulation, for example, defines four principles for good MRM:
+
+Model definition
+* Define a model and record such models in inventory.
+
+Risk governance
+* Establish model risk governance framework, policies, procedures, and controls.
+
+Life cycle management
+* Create robust model development, implementation, and usage processes.
+
+Effective challenge
+* Undertake appropriate model validation and independent review.
+
+### Key elements of responsible AI
+
+1. Data
+2. Bias
+3. Inclusiveness
+4. Model Management at Scale
+5. Governance
+
+### A template for MLOps governance
+
+1. Understand and Classify the Analytics Use Cases
+2. Establish an Ethical Position
+3. Establish Responsibilities
+
+![image](https://github.com/user-attachments/assets/ef7ff8c1-d28a-47e6-9a53-12118a031c65)
+
+4. Determine Governance Policies
+
+| **Governance Consideration**          | **Example Measures**                                                                                                                                                     |
+|---------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Reproducibility and Traceability**  | Full VM and data snapshot for rapid model re-instantiation, ability to re-create environment and retrain with a data sample, or record only metrics of deployed models?   |
+| **Audit and Documentation**           | Full log of all changes during development including experiments and reasons for choices, automated documentation of deployed model only, or no documentation at all?     |
+| **Human-in-the-Loop Sign-Off**        | Multiple sign-offs for every environment move (development, QA, preproduction, production).                                                                               |
+| **Preproduction Verification**        | Hand-code model for verification, automated test pipeline with extensive unit and end-to-end test cases, or automated checks on database, software version, and standards?|
+| **Transparency and Explainability**   | Manually-coded decision tree for maximum explainability, use of regression algorithms’ explainability tools like Shapley values, or acceptance of opaque algorithms?      |
+| **Bias and Harm Testing**             | “Red team” adversarial manual testing with multiple tools, or automated bias checking on specific subpopulations.                                                         |
+| **Production Deployment Modes**       | Containerized deployment to scalable, high-availability multi-node configuration with automated stress/load testing, or a single production server?                      |
+| **Production Monitoring**             | Real-time alerting, dynamic model balancing, automated nightly retraining and redeployment, weekly input drift monitoring with manual retraining, or basic infrastructure alerts? |
+| **Data Quality and Compliance**       | PII considerations including anonymization, documented and reviewed column-level lineage, and automated data quality checks for anomalies.                               |
+
+5. Integrate Policies into the MLOps Process
+
+| Process Step          | Example Activities and Governance Considerations                                          |
+|-----------------------|--------------------------------------------------------------------------------------------|
+| **Business Scoping**  | Record objectives, define KPIs, and record sign-off: for internal governance considerations |
+| **Ideation**          | Data discovery: data quality and regulatory compliance constraints<br>Algorithm choice: impacted by explainability requirements |
+| **Development**       | Data preparation: consider PII compliance, separation of legal regional scopes, avoid input bias<br>Model development: consider model reproducibility and auditability<br>Model testing and verification: bias and harm testing, explainability |
+| **Preproduction**     | Verify performance/bias with production data<br>Production-ready testing: verify scalability |
+| **Deployment**        | Deployment strategy: driven by the level of operationalization<br>Deployment verification tests<br>Use of shadow challenger or A/B test techniques for in-production verification |
+| **Monitoring and Feedback** | Performance metrics and alerting<br>Prediction log analysis for input drift with alerting |
+
+6. Select the Tools for Centralized Governance Management
+7. Engage and Educate
+8. Monitor and Refine
+
+
+## Introducing MLOps Chapter 9: MLOps in Practice: Consumer Credit Risk Management
+
+### Background: The Business Use Case
+
+When a consumer applies for a loan, the credit institution must decide whether to grant it, often using scores to estimate the likelihood of repayment. The process involves:
+
+- **Prescreen Stage**: A preliminary score with minimal features to quickly filter out some applications.
+- **Underwriting Stage**: A detailed score based on comprehensive information for a more accurate decision.
+- **Post-Underwriting**: Scores to assess the risk of loans in the portfolio.
+
+**Historical Context**:
+- **FICO Score**: Used since 1995 in the U.S. for credit risk assessment.
+- **Regulations**: Models, whether rule-based, statistical, or machine learning, must comply with strict regulations.
+
+**Credit Decision Information**:
+- **Data Availability**: Includes customer’s credit history and repayment behavior.
+- **Outcome Definition**: “Bad” outcomes (non-repayment) are often determined before the final repayment data is available.
+
+### Model Development
+
+- **Traditional Methods**: Relied on expert rules and logistic regression.
+- **Modern Techniques**: Gradient boosting algorithms like XGBoost offer better model-building but come with challenges like the black-box effect.
+- **Validation**: Methods like Shapley values are used to build trust and validate models.
+
+### Model Bias Considerations
+
+- **Selection Bias**: Models used for loan rejection must avoid cherry-picking and ensure accurate predictions across all applicant segments.
+- **Probability Calibration**: Models must produce accurate probabilities, not just binary outcomes, which often requires calibration techniques.
+
+### Validation and Monitoring
+
+- **Validation**: Includes performance testing on out-of-sample data and analysis of subpopulations.
+- **Monitoring**: Both data and performance drift must be assessed. Data drift involves metrics like Kolmogorov-Smirnov and stability indices, while performance drift is monitored with metrics like AUC.
+
+### Prepare for Production
+
+- **Documentation**: Detailed records of data, model assumptions, validation methods, and monitoring approaches are essential.
+- **Validation Process**: Independent reviews and potential model rebuilds ensure accuracy and understanding.
+
+### Deploy to Production
+
+- **Environment**: Production environments are typically separate and may use different technical stacks.
+- **Deployment**: Ranges from simple operations to complex integrations. Automation of monitoring depends on the frequency and criticality of the model.
+
+## Fundamentals of Data Engineering Chapter 4: Choosing Technologies Across the Data Engineering Lifecycle
+
+### Team Size and Capabilities
+
+The first thing you need to assess is your team’s size and its capabilities with technol‐ ogy. Are you on a small team (perhaps a team of one) of people who are expected to wear many hats, or is the team large enough that people work in specialized roles? Will a handful of people be responsible for multiple stages of the data engineering lifecycle, or do people cover particular niches? Your team’s size will influence the types of technologies you adopt.
+
+### Speed to Market
+
+In technology, speed to market wins. This means choosing the right technologies that help you deliver features and data faster while maintaining high-quality standards and security. It also means working in a tight feedback loop of launching, learning, iterating, and making improvements.
+
+### Interoperability
+
+Rarely will you use only one technology or system. When choosing a technology or system, you’ll need to ensure that it interacts and operates with other technologies. Interoperability describes how various technologies or systems connect, exchange information, and interact.
+
+### Today Versus the Future: Immutable Versus Transitory Technologies
+
+<img width="733" alt="image" src="https://github.com/user-attachments/assets/59538076-dccd-4696-878f-ece27fc3e149">
+
+Given the rapid pace of tooling and best-practice changes, we suggest evaluating tools every two years. Whenever possible, find the immutable technologies along the data engineering lifecycle, and use those as your base. Build transitory tools around the immutables.
+
+### Location
+
+- on-premise
+- cloud
+- hybrid
+- multi-cloud
+
+### Build Versus Buy
+
+Build versus buy comes back to knowing your competitive advantage and where it makes sense to invest resources toward customization. In general, we favor open-source (OSS) software and Commercial OSS by default, which frees you to focus on improving those areas where these options are insufficient. Focus on a few areas where building something will add significant value or reduce friction substantially.
+
+### Monolith Versus Modular
+
+While monoliths are attractive because of ease of understanding and reduced com‐ plexity, this comes at a high cost. The cost is the potential loss of flexibility, opportu‐ nity cost, and high-friction development cycles
+
+### Serverless Versus Servers
+
+Key factors to consider:
+* workload size and complexity
+* execution frequency and duration
+* requests and networking
+* language
+* runtime limitations
+* cost
+
+
+## [Kubernetes for the Absolute Beginners - Hands-On](https://www.oreilly.com/videos/kubernetes-for-the/9781838555962/)
+
+This is a video course full with short lectures. Tbh I clicked on it by mistake but decided to watch a bit. Turns out is is quite interesting. Below is some of the interesting bits I learned
+
+![image](https://github.com/user-attachments/assets/2e664447-c0c6-4181-b21a-0ab763fbb100)
+
+### What is a Pod?
+- **Pod**: The smallest Kubernetes object that encapsulates one or more containers.
+- **Function**: Encapsulates a container to deploy an application on a set of worker nodes.
+
+### Scaling
+- **Scaling Up**: Create new pods for additional instances of the application, not more containers within the same pod.
+- **Scaling Down**: Delete existing pods to reduce the number of instances.
+- **Capacity**: If more capacity is needed, add new nodes to the cluster and deploy additional pods there.
+
+### Multiple Containers in a Pod
+- **Single Container**: A pod usually has a 1:1 relationship with containers.
+- **Multiple Containers**: Possible within a single pod for specific use cases, such as having a helper container alongside the main application container.
+  - **Communication**: Containers in the same pod share the same network and storage.
+  - **Lifecycle**: Helper containers are created and destroyed with the main container.
+
+### Comparing with Docker Containers
+- **Without Kubernetes**: Manual management of networking, volumes, and state for multiple containers.
+- **With Kubernetes**: Kubernetes manages these aspects automatically within pods, simplifying deployment and management.
+
+### Pod Deployment
+- **Command**: `kubectl run` deploys a Docker container by creating a pod.
+  - **Image**: Specify the image using the `--image` parameter (e.g., Nginx from Docker Hub).
+- **Viewing Pods**: Use `kubectl get pods` to see the list and status of pods.
+
+### Replicas. 
+
+The ReplicaSet controller manages the desired number of pod replicas, ensuring that the specified number of identical pods are running at all times. This helps balance load and provides redundancy in case of pod failures.
+
+## Finally, we submitted the KB project
+
+In the last second I added a small video demo of the project. Instead of speaking I put subtitles. [Here](https://youtu.be/uHJ3EclH43A) is the youtube link <- it is in Korean only. 
+
+They do not mention when they will announce people inveted for the finals, which are in-person on the 28th of Aug, so we have to just wait and see ~
+
+
+That is all for today! 
+
+See you tomorrow :) 

From 30eba8292d84630feef7e4d38037ba0079bbb500 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 12 Aug 2024 18:53:52 +0900
Subject: [PATCH 371/638] Create 2024-08-12-day-224.md

---
 _posts/2024-08-12-day-224.md | 414 +++++++++++++++++++++++++++++++++++
 1 file changed, 414 insertions(+)
 create mode 100644 _posts/2024-08-12-day-224.md

diff --git a/_posts/2024-08-12-day-224.md b/_posts/2024-08-12-day-224.md
new file mode 100644
index 000000000..e5b28fef5
--- /dev/null
+++ b/_posts/2024-08-12-day-224.md
@@ -0,0 +1,414 @@
+---
+layout: post
+title: (Day 224) Learning about Snowflake and starting the book - Deep Learning at Scale
+categories: [deep-learning,mlops,theory,data-eng]
+---
+
+# Hello :) Today is Day 224!
+A quick summary of today:
+* a bit more about k8s
+* read chapter 1 from Deep Learning at Scale
+* learn about Snowflake
+
+
+## Kubernetes for the Absolute Beginners - Hands-On
+
+### Deployments in K8s
+
+In Kubernetes, deployments manage the deployment and scaling of applications. They ensure multiple instances of an application are running, allow seamless updates via rolling updates (updating instances one by one to avoid downtime), and support rollback if an update causes issues. Deployments also allow pausing and resuming changes to batch multiple updates together. A deployment creates a ReplicaSet, which in turn creates pods that run the application. Deployment configuration is defined in a YAML file similar to a ReplicaSet, with the key difference being the kind set to "Deployment." Once applied, the deployment manages ReplicaSets and pods, ensuring the application runs as specified.
+
+![image](https://github.com/user-attachments/assets/d7c661a9-b91a-45f4-9341-fd69c415f1d5)
+
+### Deployment Strategies
+
+When a deployment is first created, it triggers a rollout, generating a deployment revision (e.g., revision one). Updating the application, such as changing the container version, triggers a new rollout and creates a subsequent revision (e.g., revision two). This versioning allows for easy tracking of changes and the ability to roll back if needed.
+
+There are two deployment strategies:
+
+1. **Recreate Strategy**: This involves shutting down all instances of the application before deploying the new version, causing temporary downtime.
+2. **Rolling Update Strategy** (default): Instances are updated one by one, ensuring the application remains accessible during the upgrade.
+
+Commands like `kubectl rollout status` and `kubectl rollout history` are used to monitor rollout status and history.
+
+![image](https://github.com/user-attachments/assets/35540d30-9ab2-4341-9f2f-93c41a488359)
+
+## [Deep Learning at Scale](https://learning.oreilly.com/library/view/deep-learning-at/9781098145279/)
+
+This is one of the books on my `to-read` list in O'reilly, so today I decided to start it. Below is a summary of chapter 1 which included lots of nature references - trying to explain scalability in nature.
+
+**Scaling in deep learning** is a multidimensional challenge, generally consisting of three aspects:
+
+1. **Generalizability**: Enhancing the ability of models to generalize across diverse tasks or demographics.
+2. **Training and Development**: Improving model training techniques to reduce development time while meeting resource needs.
+3. **Inference**: Boosting the serviceability of the model during deployment and serving.
+
+This book focuses primarily on the first two aspects, with inference being beyond its scope.
+
+### Six Development Considerations
+
+When developing a deep learning solution, six key considerations are critical for scaling:
+
+#### 1. Well-Defined Problem
+- **Understanding the Problem**: Clearly define the problem you are solving and ensure you have the right data.
+- **Example**: Scaling a roof detection model from Sydney to global use requires defining if the task is just detecting roofs or also identifying roof styles.
+
+#### 2. Domain Knowledge (Constraints)
+- **Domain-Specific Constraints**: Understand how the problem space, like geographic differences in roof materials, impacts model development.
+
+#### 3. Ground Truth
+- **Dataset Representation**: Ensure the dataset represents global variability, balancing quality and generalization.
+- **Example**: Consider unique roofs, such as Scandinavian sod roofs, when curating data.
+
+#### 4. Model Development
+- **Scaling Techniques**: Develop models that can handle increased data and complexity as the task scales.
+- **Questions**: Plan experiments, adopt efficient training strategies, and consider distributed training when necessary.
+
+#### 5. Deployment
+- **Deployment Strategy**: Though not covered in this book, it's essential to consider deployment constraints and performance from the early stages.
+
+#### 6. Feedback
+- **Real-Time Feedback**: Implement systems for collecting user feedback to continuously improve the model.
+- **Example**: Automate feedback collection to create a data engine that iteratively improves the system.
+
+### Scaling Considerations
+
+Scaling is complex and requires careful planning and strategy. Key considerations include:
+
+### Questions to Ask Before Scaling
+- **What are we scaling?**
+- **How will we measure success?**
+- **Do we really need to scale?**
+- **What are the ripple effects?**
+- **What are the constraints?**
+- **How will we scale?**
+- **Is our scaling technique optimal?**
+
+### Characteristics of Scalable Systems
+
+1. **Reliability**: The system should recover gracefully from failures.
+2. **Availability**: Strive for high availability, ideally 99.999% uptime.
+3. **Adaptability**: Ensure the system can handle growing demands.
+4. **Performance**: Monitor and optimize resource consumption as the system scales.
+
+### Design Considerations for Scalable Systems
+
+- **Avoid Single Points of Failure (SPOF)**: Minimize SPOFs to prevent complete system halts.
+- **Design for High Availability**: Implement reliability, resilience, and redundancy.
+- **Scaling Paradigms**: Utilize horizontal, vertical, or hybrid scaling based on the needs.
+- **Communication**: Choose between synchronous and asynchronous communication depending on system requirements.
+- **Caching and Storage**: Use caching to minimize expensive information retrieval processes.
+- **Process State**: Opt for stateless processes where possible to simplify scaling.
+- **Graceful Recovery and Checkpointing**: Implement checkpointing for efficient recovery from failures.
+- **Maintainability and Observability**: Monitor systems effectively to simplify maintenance and reduce downtime.
+
+### Scaling Effectively
+
+- **Measure Twice, Cut Once**: Apply careful planning, benchmarking, and iterative improvement to scale efficiently.
+- **Efficiency**: Efficient use of resources is critical, considering environmental and societal impacts of deep learning practices.
+
+## Then I found Snowflake has a 1 month free trial and also a 0-to-Snowflake in 90 mintues free course
+
+I saw that Snowflake sales engineers offer a 90-min intro to snowflake, and that the next course is on the 14th of Aug. I registered, and then made an account in Snowflake and tried out some of the functionality. 
+
+This is the homepage UI:
+
+![image](https://github.com/user-attachments/assets/8af0be19-3f01-462f-a323-c8587b40889c)
+
+
+Seems that SQL and python can be used for data manipulation. I chose SQL for the default offered quick intro to snowflake, and below are the queries.
+
+1. We create a table
+
+```sql
+CREATE OR REPLACE TABLE tasty_bytes_sample_data.raw_pos.menu
+(
+    menu_id NUMBER(19,0),
+    menu_type_id NUMBER(38,0),
+    menu_type VARCHAR(16777216),
+    truck_brand_name VARCHAR(16777216),
+    menu_item_id NUMBER(38,0),
+    menu_item_name VARCHAR(16777216),
+    item_category VARCHAR(16777216),
+    item_subcategory VARCHAR(16777216),
+    cost_of_goods_usd NUMBER(38,4),
+    sale_price_usd NUMBER(38,4),
+    menu_item_health_metrics_obj VARIANT
+);
+```
+
+2. Load data from an S3 bucket
+
+```sql
+---> create the Stage referencing the Blob location and CSV File Format
+CREATE OR REPLACE STAGE tasty_bytes_sample_data.public.blob_stage
+url = 's3://sfquickstarts/tastybytes/'
+file_format = (type = csv);
+
+---> query the Stage to find the Menu CSV file
+LIST @tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/;
+```
+
+Basically it is just normal sql. And we can query and manipulate data and get basic visualisations
+
+```sql
+SELECT TOP 10 * FROM tasty_bytes_sample_data.raw_pos.menu;
+```
+![image](https://github.com/user-attachments/assets/18354ba8-e6a3-4584-ac32-0ef88aa89db1)
+
+### Next, I tried another tutorial: Create a Notebook to visualize data and insights
+
+_Snowflake Notebooks enable you to write and execute code in SQL, Python, or Markdown to create iterative flows, visualize results, and more._
+
+Here is what I saw first:
+
+![image](https://github.com/user-attachments/assets/5328a6d2-0be0-42fc-9689-c50c85c64b23)
+
+On the left side, next to the notebook there is an environment.yml which seems to serve like requirements.txt
+
+```yaml
+name: app_environment
+channels:
+  - snowflake
+dependencies:
+  - matplotlib=3.7.2
+  - scipy=1.10.1
+```
+
+We can install packages directly as well using the `Package` button
+
+![image](https://github.com/user-attachments/assets/21e1366a-60d3-40da-8c03-ba2b39c9aa4a)
+
+We can also execute Python or SQL in the notebook cells. Up to us what we do and how we do it.
+
+A cool thing we can do is:
+
+1. Run this in some cell (say `cell4`)
+
+```sql
+-- Generating a synthetic dataset of Snowboard products, along with their price and rating
+SELECT CONCAT('SNOW-',UNIFORM(1000,9999, RANDOM())) AS PRODUCT_ID, 
+       ABS(NORMAL(5, 3, RANDOM())) AS RATING, 
+       ABS(NORMAL(750, 200::FLOAT, RANDOM())) AS PRICE
+FROM TABLE(GENERATOR(ROWCOUNT => 100));
+```
+
+2. Then in `cell5`, we can do:
+
+```python
+df = cell5.to_pandas()
+```
+
+And we have the data from the SQL query as a pandas df now 🤯
+
+We can also use Jinja syntax to use python vars in sql queries
+
+```python
+threshold = 5
+```
+
+```sql
+-- Reference Python variable in SQL
+SELECT * FROM SNOW_CATALOG where RATING > {{threshold}}
+```
+
+We can also refer to other cell's output using Jinja
+
+```sql
+SELECT AVG(RATING) FROM {{cell23}}
+WHERE RATING > 5
+```
+
+Where cell23 returns a table.
+
+### There is also an AI & ML Studio section
+
+![image](https://github.com/user-attachments/assets/c0a919ea-b50b-41be-807d-b1bd58c46ec7)
+
+I am looking forward to the 0-to-Snowflake 90 minute course on Wednesday ^^ 
+
+## [Build an ELT Pipeline](https://youtu.be/OLXkGB7krGo) (dbt, Snowflake, Airflow)
+
+I wanted to watch some tutorial related to snowflake and found this video that uses dbt (which I know) and Airflow (which I have not used), so I decided to give it a go.
+
+It covered:
+
+1. Setting up Snowflake (warehouse, roles, schemas, databases, users)
+2. Configure dbt
+3. Set up staging and source models
+4. Dbt macros
+5. Dbt generic and singular tests
+6. Orchestration using Airflow
+
+
+### Setting up snowflake
+
+```sql
+-- Switch to the ACCOUNTADMIN role, which has full access to the Snowflake account
+use role accountadmin;
+
+-- Create a new warehouse named 'dbt_wh' with a size of 'x-small'
+create warehouse dbt_wh with warehouse_size='x-small';
+
+-- Create a database named 'dbt_db' if it does not already exist
+create database if not exists dbt_db;
+
+-- Create a role named 'dbt_role' if it does not already exist
+create role if not exists dbt_role;
+
+-- Display the grants (privileges) on the 'dbt_wh' warehouse
+show grants on warehouse dbt_wh;
+
+-- Grant the 'usage' privilege on the 'dbt_wh' warehouse to the 'dbt_role' role
+grant usage on warehouse dbt_wh to role dbt_role;
+
+-- Grant the 'dbt_role' role to the user 'divakaivan12'
+grant role dbt_role to user divakaivan12;
+
+-- Grant all privileges on the 'dbt_db' database to the 'dbt_role' role
+grant all on database dbt_db to role dbt_role;
+
+-- Switch to the 'dbt_role' role
+use role dbt_role;
+
+-- Create a schema named 'dbt_schema' within the 'dbt_db' database
+create schema dbt_db.dbt_schema;
+```
+
+### Set up dbt
+
+The data used is some example dataset provided by snowflake
+
+* stg_tpch_line_items.sql
+
+```sql
+select
+    {{
+        dbt_utils.generate_surrogate_key([
+            'l_orderkey',
+            'l_linenumber'
+        ])
+    }} as order_item_key,
+    l_orderkey as order_key,
+    l_partkey as part_key,
+    l_linenumber as line_number,
+    l_quantity as quantity,
+    l_extendedprice as extended_price,
+    l_discount as discount_percentage,
+    l_tax as tax_rate
+from
+    {{ source('tpch', 'lineitem') }}
+```
+
+* stg_tpch_orders.sql
+
+```sql
+select
+    o_orderkey as order_key,
+    o_custkey as customer_key,
+    o_orderstatus as status_code,
+    o_totalprice as total_price,
+    o_orderdate as order_date
+from 
+    {{ source('tpch', 'orders') }}
+```
+
+* int_order_items.sql
+
+```sql
+select
+    line_item.order_item_key,
+    line_item.part_key,
+    line_item.line_number,
+    line_item.extended_price,
+    orders.order_key,
+    orders.customer_key,
+    orders.order_date,
+    {{ discounted_amount('line_item.extended_price', 'line_item.discount_percentage') }} as item_discount_amount
+from
+    {{ ref('stg_tpch_orders') }} as orders
+join 
+    {{ ref('stg_tpch_line_items') }} as line_item
+        on orders.order_key = line_item.order_key 
+order by 
+    orders.order_date
+```
+
+* int_order_items_summary.sql
+
+```sql
+select 
+    order_key,
+    sum(extended_price) as gross_item_sales_amount,
+    sum(item_discount_amount) as item_discount_amount
+from 
+    {{ ref('int_order_items') }}
+group by    
+    order_key
+```
+
+* fct_orders.sql
+
+```sql
+select 
+    orders.*,
+    order_item_summary.gross_item_sales_amount,
+    order_item_summary.item_discount_amount    
+from
+    {{ ref('stg_tpch_orders') }} as orders
+join 
+    {{ ref('int_order_items_summary') }} as order_item_summary
+        on orders.order_key = order_item_summary.order_key
+order by
+    order_date
+```
+
+* pricing macro
+
+```sql
+{% macro discounted_amount(extended_price, discount_percentage, scale=2) %}
+    (-1 * {{extended_price}} * {{discount_percentage}})::decimal(16, {{ scale }})
+{% endmacro %}
+```
+
+* singular tests on specific fields
+
+```sql
+select 
+    *
+from
+    {{ ref('fct_orders') }}
+where
+    date(order_date) > CURRENT_DATE()
+    or date(order_date) < date('1990-01-01')
+```
+
+and
+
+```sql
+select
+    *
+from 
+    {{ ref('fct_orders') }}
+where
+    item_discount_amount > 0
+```
+
+### Running the pipeline in Airflow
+
+Airflow was set up using astronomer-cosmos which requires `astro dev init`, `astro dev start` to start airflow.
+
+To include the created dbt pipeline, all I needed to do was just put the dbt directory I was working in into astronomer-cosmos' dag folder, and I could see it in Airflow
+
+<img width="908" alt="image" src="https://github.com/user-attachments/assets/b3c044fb-33d8-4639-be84-ccbcc941f342">
+
+Here is the data lineage from Airflow:
+
+<img width="995" alt="image" src="https://github.com/user-attachments/assets/15e23066-cbd2-4aa9-a74e-8b2b04742ecb">
+
+And after a failed run due to some access errors, it ran
+
+<img width="1485" alt="image" src="https://github.com/user-attachments/assets/bcd3e27b-7fd5-438a-b5b2-4c5a734bf741">
+
+That is all for today!
+
+See you tomorrow :) 

From 898d06622488e46c3ea2eb86d1df091283ea3b0a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 12 Aug 2024 19:06:55 +0900
Subject: [PATCH 372/638] Update 2024-08-12-day-224.md

---
 _posts/2024-08-12-day-224.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_posts/2024-08-12-day-224.md b/_posts/2024-08-12-day-224.md
index e5b28fef5..edf40d51a 100644
--- a/_posts/2024-08-12-day-224.md
+++ b/_posts/2024-08-12-day-224.md
@@ -409,6 +409,8 @@ And after a failed run due to some access errors, it ran
 
 <img width="1485" alt="image" src="https://github.com/user-attachments/assets/bcd3e27b-7fd5-438a-b5b2-4c5a734bf741">
 
+All the code I wrote is on [my repo](https://github.com/divakaivan/learn_etl_dbt_snowflake_airflow).
+
 That is all for today!
 
 See you tomorrow :) 

From f519fa9485f13da2732d595f9b146ed05fbeca7c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 12 Aug 2024 20:10:26 +0900
Subject: [PATCH 373/638] Update 2024-08-12-day-224.md

---
 _posts/2024-08-12-day-224.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_posts/2024-08-12-day-224.md b/_posts/2024-08-12-day-224.md
index edf40d51a..f4f7d2dc5 100644
--- a/_posts/2024-08-12-day-224.md
+++ b/_posts/2024-08-12-day-224.md
@@ -393,6 +393,7 @@ where
     item_discount_amount > 0
 ```
 
+
 ### Running the pipeline in Airflow
 
 Airflow was set up using astronomer-cosmos which requires `astro dev init`, `astro dev start` to start airflow.

From a56238e2922db86c6dbc8331c665f8340f6392b6 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 12 Aug 2024 20:11:26 +0900
Subject: [PATCH 374/638] Update 2024-08-12-day-224.md

---
 _posts/2024-08-12-day-224.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-08-12-day-224.md b/_posts/2024-08-12-day-224.md
index f4f7d2dc5..d331bb5e1 100644
--- a/_posts/2024-08-12-day-224.md
+++ b/_posts/2024-08-12-day-224.md
@@ -362,7 +362,7 @@ order by
     order_date
 ```
 
-* pricing macro
+* pricing macro*
 
 ```sql
 {% macro discounted_amount(extended_price, discount_percentage, scale=2) %}

From 99af95f2522b4ce9371091e47112fbfd4d984be9 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 12 Aug 2024 20:14:36 +0900
Subject: [PATCH 375/638] Update 2024-08-12-day-224.md

---
 _posts/2024-08-12-day-224.md | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/_posts/2024-08-12-day-224.md b/_posts/2024-08-12-day-224.md
index d331bb5e1..4e2f71cde 100644
--- a/_posts/2024-08-12-day-224.md
+++ b/_posts/2024-08-12-day-224.md
@@ -364,11 +364,9 @@ order by
 
 * pricing macro*
 
-```sql
-{% macro discounted_amount(extended_price, discount_percentage, scale=2) %}
-    (-1 * {{extended_price}} * {{discount_percentage}})::decimal(16, {{ scale }})
-{% endmacro %}
-```
+<img width="739" alt="image" src="https://github.com/user-attachments/assets/6323c6f7-bde1-42dc-baac-d2653748f9d0">
+
+(has to be as a pic because of a jinja setup in my blog that tries to execute the macro)
 
 * singular tests on specific fields
 

From 9d198899c48d87529806064a82968462b57ac051 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 13 Aug 2024 23:55:12 +0900
Subject: [PATCH 376/638] Create 2024-08-13-day-225.md

---
 _posts/2024-08-13-day-225.md | 324 +++++++++++++++++++++++++++++++++++
 1 file changed, 324 insertions(+)
 create mode 100644 _posts/2024-08-13-day-225.md

diff --git a/_posts/2024-08-13-day-225.md b/_posts/2024-08-13-day-225.md
new file mode 100644
index 000000000..baed2da1b
--- /dev/null
+++ b/_posts/2024-08-13-day-225.md
@@ -0,0 +1,324 @@
+---
+layout: post
+title: (Day 225) Starting the Finance Voice Assistant project
+categories: [applying-knowledge,nlp,data-eng]
+---
+
+# Hello :) Today is Day 225!
+A quick summary of today:
+* read a bit more of the Fundamentals of Data Eng book
+* started the finance voice assistant app
+
+
+
+## Fundamentals of Data Engineering Chapter 5: Data Generation in Source Systems
+
+![image](https://github.com/user-attachments/assets/7cdb0dd2-2cc6-46d5-9ae0-037c8ad9e0b8)
+
+### Source Systems: Main Ideas
+
+#### Files and Unstructured Data
+- **Files**: Sequences of bytes stored on disk, used to store parameters, events, logs, images, and audio.
+- **Data Exchange**: Files are a universal medium; common formats include Excel, CSV, TXT, JSON, and XML.
+- **Structure**: Files can be structured (Excel, CSV), semistructured (JSON, XML), or unstructured (TXT, CSV).
+
+#### APIs
+- **APIs**: Standard method for data exchange between systems. Simplifies but also adds complexity to data ingestion.
+- **Maintenance**: Data engineers often need to manage custom API connections.
+
+#### Application Databases (OLTP Systems)
+- **OLTP Systems**: High-rate read/write databases, supporting low latency and high concurrency, used in applications like banking.
+- **ACID Properties**: Ensure atomicity, consistency, isolation, and durability, simplifying application development but not always necessary for performance.
+- **Atomic Transactions**: A set of operations committed as a unit, crucial in applications like banking.
+
+#### OLTP and Analytics
+- **Challenges**: Running analytics on OLTP systems can degrade performance; hybrid systems combining transactions and analytics are emerging.
+
+#### Online Analytical Processing Systems (OLAP)
+- **OLAP Systems**: Designed for large-scale analytics queries, not efficient for individual lookups.
+- **Usage**: Often source systems for analytics or integrated in workflows like reverse ETL.
+
+#### Change Data Capture (CDC)
+- **CDC**: Method for extracting database change events (insert, update, delete) for real-time replication or event streaming.
+
+#### Logs
+- **Logs**: Capture event information in systems, valuable for data analysis, ML, and automation.
+- **Sources**: Include operating systems, applications, servers, containers, networks, and IoT devices. Logs track the "who," "what," and "when" of events.
+
+### CRUD
+
+CRUD stands for Create, Read, Update, and Delete, and represents the four essential operations used in managing persistent storage within programming. This pattern is foundational for storing application state in a database and is ubiquitous in APIs and databases. The CRUD operations ensure that data can be created, read, updated, and eventually deleted as needed, providing a basic framework for data interaction.
+
+A common use of CRUD can be seen in web applications, where CRUD operations underpin RESTful HTTP requests, facilitating the storage and retrieval of data from databases. For example, when an application requires the creation of a new record, a corresponding Create operation is performed, followed by Read operations to retrieve the data, Update operations to modify it, and finally, Delete operations to remove the data if necessary.
+
+### Insert-Only
+
+The Insert-Only pattern is a database operation strategy where instead of updating records, new records are inserted with a timestamp indicating when they were created. This pattern is beneficial for maintaining historical data directly in a table. For example, if a customer changes their address, rather than updating the existing record, a new record is inserted with the new address and a timestamp. To find the current address, the latest record for that customer ID would be retrieved.
+
+This pattern effectively maintains a log within the table, making it ideal for applications that require access to historical data, such as banking systems that need to track changes over time. However, Insert-Only can lead to large table sizes, especially when data changes frequently. To mitigate this, records may be purged based on specific criteria, such as a sunset date, or limited to a maximum number of versions.
+
+### Messages and Streams
+
+In event-driven architecture, two key concepts are Message Queues and Streaming Platforms. Although these terms are sometimes used interchangeably, they represent different mechanisms for handling data. A **message** is a unit of data sent from one system to another, often through a message queue. Once delivered, the message is typically removed from the queue. This approach is common in systems that require discrete, one-time communication between components.
+
+For example, in an IoT system, a device might send a temperature reading to a message queue, which is then processed to determine whether to adjust a thermostat. Once the action is taken, the message is removed.
+
+In contrast, a **stream** is an append-only log of event records. Events in a stream are accumulated in a sequence, often ordered by time or ID. Streams are used when tracking a series of events is important, and data is retained over an extended period. Streaming platforms enable complex operations on this data, such as aggregating multiple events or replaying data from a specific point in time.
+
+### Types of Time
+
+When dealing with streaming data, understanding the different types of time is crucial. These include:
+
+- **Event Time**: The time when the event is generated within the source system.
+- **Ingestion Time**: The time when the event is ingested into a message queue, database, or another storage system.
+- **Process Time**: The time when the event is processed, usually after ingestion.
+- **Processing Time**: The duration taken to process the event.
+
+Tracking these times is essential for accurate data pipeline monitoring and performance analysis. Each stage of data movement should be logged with timestamps to ensure transparency and facilitate debugging or optimization efforts.
+
+### Source System Practical Details
+
+Interacting with modern source systems requires a thorough understanding of various technologies like databases and APIs. This section provides critical knowledge for data engineers, focusing on the types of databases you might encounter and the major considerations for working with them.
+
+#### Databases
+
+Databases are categorized into **Relational** and **Nonrelational** types, each with its own use cases and architectural considerations.
+
+- **Relational Databases (RDBMS)**: These are widely used for storing structured data with relationships between different data entities. They support ACID properties (Atomicity, Consistency, Isolation, Durability) and are ideal for applications requiring transaction consistency and data integrity. Relational databases are typically normalized to reduce data redundancy and support complex queries through SQL.
+
+- **Nonrelational Databases (NoSQL)**: These databases are designed to handle a wide variety of data models, including key-value, document, wide-column, graph, and others. NoSQL databases often trade off some of the ACID properties to achieve greater scalability and performance. They are ideal for applications with large-scale, unstructured, or semi-structured data, where flexibility and speed are prioritized.
+
+### Key-Value Stores
+
+Key-value databases are a type of NoSQL database optimized for quick retrieval of values using a unique key. These databases are akin to hash maps or dictionaries in programming languages and are used in scenarios requiring fast access to large volumes of data, such as caching sessions for web applications.
+
+### Document Stores
+
+Document stores are another type of key-value database where the value is a document, often in JSON format. These databases are highly flexible, allowing for the storage of complex nested data structures. They do not enforce a strict schema, which makes them adaptable but also potentially more challenging to manage.
+
+### Wide-Column Databases
+
+Wide-column databases are designed for high transaction rates and massive scalability. They store data in columns rather than rows, making them suitable for use cases like real-time analytics and handling large datasets with low latency.
+
+### Graph Databases
+
+Graph databases store data in a graph structure, with nodes representing entities and edges representing relationships. These databases are particularly effective for applications that need to analyze connections between data points, such as social networks or recommendation engines.
+
+### Third-Party Data Sources
+
+With the widespread adoption of technology, companies across industries, including government agencies, are increasingly offering their data to customers and users. This data can be a part of their service offerings or available as a separate subscription. For example, the US Bureau of Labor Statistics provides labor market data, NASA shares research data, and Facebook offers data to businesses advertising on its platform.
+
+The strategic reason behind making data available is that data is inherently "sticky," meaning it encourages greater user engagement and integration. This creates a flywheel effect—users who integrate third-party data into their systems tend to generate more data, which in turn can be further integrated and utilized, leading to increased user adoption and engagement. This cycle leads to a near-infinite supply of third-party data sources.
+
+Access to third-party data is typically facilitated through APIs, cloud platforms, or direct data downloads. APIs are particularly popular for enabling deep integrations, allowing customers to push and pull data seamlessly. For instance, Customer Relationship Management (CRM) systems often offer APIs that allow users to integrate CRM data into their own applications. A common workflow involves retrieving data from a CRM, processing it through a customer scoring model, and then using reverse ETL to feed the refined data back into the CRM, thereby helping sales teams target more qualified leads.
+
+### Message Queues and Event-Streaming Platforms
+
+Event-driven architectures are becoming more prevalent in software development, driven by the ease of managing message queues and event-streaming platforms in cloud environments. These architectures are particularly well-suited for data applications that require real-time analytics because events can trigger application workflows and support near real-time data processing.
+
+Streaming data, like messages and streams, often crosses multiple stages of the data engineering lifecycle. Unlike relational databases directly tied to applications, streaming data can blur the lines between different lifecycle stages due to its transient nature. For example, an event-streaming platform can serve as both a source system for message passing in event-driven applications and as a tool for real-time data processing during ingestion and transformation.
+
+#### Message Queues
+
+Message queues are systems that enable asynchronous data transmission between discrete systems using a publish-subscribe model. In this setup, data is published to a message queue and delivered to one or more subscribers, who acknowledge receipt, thereby removing the message from the queue.
+
+Message queues are vital for decoupling applications in microservices architectures. They buffer messages to handle transient load spikes and ensure message durability through distributed architectures with replication. Key considerations when working with message queues include:
+
+- **Message Ordering and Delivery**: Message queues typically use a First-In, First-Out (FIFO) model, but in distributed systems, strict FIFO ordering can be challenging to maintain. For instance, Amazon SQS offers both standard and FIFO queues, with the latter providing stronger ordering guarantees at the cost of additional overhead.
+- **Delivery Frequency**: Messages can be delivered "exactly once" or "at least once." Exactly once delivery ensures the message is not duplicated, while at least once delivery allows for potential redundancy, which can be useful in certain scenarios. Idempotent systems, which produce the same result regardless of how many times a message is processed, are ideal for handling delivery complexities.
+- **Scalability**: Modern message queues are horizontally scalable, operating across multiple servers. This scalability allows them to dynamically adjust to load variations, but it also introduces challenges such as message duplication and fuzzy ordering.
+
+#### Event-Streaming Platforms
+
+Event-streaming platforms extend the functionality of message queues by providing a mechanism to ingest and process data in an ordered log of records. Unlike message queues, which primarily route messages, event-streaming platforms retain data for a set period, enabling the replay of past events.
+
+In an event-streaming platform, events are typically composed of a key, value, and timestamp. For instance, an e-commerce order event might include a unique order ID (key), product details (value), and the time of purchase (timestamp).
+
+Key features of event-streaming platforms include:
+
+- **Topics**: Producers stream events to topics, which are collections of related events. For example, a topic might contain customer orders, and different consumers, such as fulfillment and marketing departments, can subscribe to this topic to trigger actions or analyze data.
+- **Stream Partitions**: Streams can be divided into multiple partitions, allowing for parallel processing and higher throughput. The partition key determines how messages are distributed across partitions. For example, using a device ID as a partition key ensures that all messages from a particular device are processed together. However, it's crucial to avoid "hotspotting," where some partitions receive a disproportionate number of messages.
+- **Fault Tolerance and Resilience**: Event-streaming platforms are distributed, meaning that if one node fails, another can take over, ensuring data continuity. This resilience makes them suitable for applications requiring reliable data production, storage, and ingestion.
+
+## Fundamentals of Data Engineering Chapter 6: Storage
+
+![image](https://github.com/user-attachments/assets/33511338-5dec-41b1-9f95-f5c517d63643)
+
+![image](https://github.com/user-attachments/assets/b90c3d14-8c0f-41cd-8db7-5e8316ca7d4f)
+
+### Storage Abstractions
+
+- **Data Lake**: A centralized repository that allows you to store all your structured and unstructured data at any scale. It can handle raw data in its native format until it’s needed.
+  
+- **Data Lakehouse**: A hybrid architectural model that combines the capabilities of data lakes and data warehouses. It supports both large-scale data storage and advanced analytics.
+
+- **Data Platform**: An integrated suite of tools and technologies that manage the data lifecycle, from ingestion to processing and analysis. It includes the infrastructure to handle data storage, governance, and access.
+
+- **Cloud Data Warehouse**: A type of data warehouse that is hosted in the cloud. It allows for scalable, on-demand data storage and computing power to analyze large volumes of data.
+
+### Storage Systems
+
+- **HDFS (Hadoop Distributed File System)**: A distributed file system that stores data across multiple machines in a large cluster. It is designed to handle large volumes of data with fault tolerance and scalability.
+
+- **RDBMS (Relational Database Management System)**: A database system that organizes data into tables and uses SQL (Structured Query Language) for database management and querying.
+
+- **Cache/Memory-based Storage**: Storage that resides in memory (RAM) to provide fast access to frequently accessed data, reducing latency compared to disk-based storage.
+
+- **Object Storage**: A storage architecture that manages data as objects rather than as a file hierarchy. It is commonly used for storing unstructured data like images, videos, and backups.
+
+- **Streaming Storage**: A storage system designed to handle real-time data streams, enabling immediate processing and analysis of data as it is generated.
+
+### Raw Ingredients
+
+- **HDD (Hard Disk Drive)**: A traditional storage device that uses spinning disks to read/write data. It is known for its large storage capacity and relatively lower cost but slower speed compared to SSDs.
+
+- **SSD (Solid State Drive)**: A storage device that uses flash memory to store data, providing faster read/write speeds than HDDs. SSDs have no moving parts, making them more reliable and quicker.
+
+- **RAM (Random Access Memory)**: A form of volatile memory used to store working data and machine code currently in use. It offers high-speed data access but is temporary, losing its contents when power is off.
+
+- **Networking**: The interconnection of various hardware devices and systems to allow data transfer and communication. It is essential for enabling distributed computing and storage systems to function cohesively.
+
+- **Serialization**: The process of converting data into a format that can be easily stored or transmitted and then reconstructed. It is crucial for data storage, transfer, and communication across systems.
+
+- **Compression**: The technique used to reduce the size of data for storage or transmission. It can be lossless or lossy, depending on whether data integrity is preserved.
+
+- **CPU (Central Processing Unit)**: The primary component of a computer that performs most of the processing inside a system. It executes instructions and processes data, playing a critical role in data handling and storage operations.
+
+
+#### Data Storage Lifecycle and Data Retention
+
+![image](https://github.com/user-attachments/assets/f78db91f-8a25-402d-ab5c-980f7bd50ab9)
+
+## Fundamentals of Data Engineering Chapter 7: Ingestion
+
+![image](https://github.com/user-attachments/assets/97055ab0-b639-48aa-b57a-c041dc29e44e)
+
+### What Is Data Ingestion?
+
+The process of moving data from one place to another. Data ingestion implies data movement from source systems into storage in the data engineering lifecycle, with ingestion as an intermediate step
+
+![image](https://github.com/user-attachments/assets/d0c7abff-5eb2-4de1-b45d-3aaec2ebd472)
+
+### Key Engineering Considerations for the Ingestion Phase
+
+* What’s the use case for the data I’m ingesting?
+* Can I reuse this data and avoid ingesting multiple versions of the same dataset?
+* Where is the data going? What’s the destination?
+* How often should the data be updated from the source?
+* What is the expected data volume?
+* What format is the data in? Can downstream storage and transformation accept this format?
+* Is the source data in good shape for immediate downstream use? That is, is the data of good quality? What post-processing is required to serve it?
+* What are data-quality risks (e.g., could bot traffic to a website contaminate the data)?
+* Does the data require in-flight processing for downstream ingestion if the data is from a streaming source?
+
+#### Regardless of how often the data is ingested, you’ll want to consider these factors when designing your ingestion architecture:
+
+* bounded versus unbounded
+* frequency
+* sync vs async
+* serialisation and deserialisation
+* throughput and scalability
+* reliability and durability
+* payload
+* push vs pull vs poll patterns
+
+### Bounded vs Unbounded data
+
+![image](https://github.com/user-attachments/assets/90825694-4615-4337-83b0-94923aecffb6)
+
+Unbounded data is data as it exists in reality, as events happen, either sporadically or continuously, ongoing and flowing. Bounded data is a conve‐ nient way of bucketing data across some sort of boundary, such as time.
+
+### Frequency
+
+One of the critical decisions that data engineers must make in designing dataingestion processes is the data-ingestion frequency. Ingestion processes can be batch, micro-batch, or real-time.
+
+![image](https://github.com/user-attachments/assets/0050c131-5fe5-4086-821b-be8419a25e12)
+
+### Synchronous Versus Asynchronous Ingestion
+
+![image](https://github.com/user-attachments/assets/a602a5fc-1007-4c7e-82e9-f243220c88ab)
+
+![image](https://github.com/user-attachments/assets/fba9dea6-7f05-46c2-9370-d74c4b7e4736)
+
+With asynchronous ingestion, dependencies can now operate at the level of individ‐ ual events, much as they would in a software backend built from microservices. Individual events become available in storage as soon as they are inges‐ ted individually
+
+### Serialization and Deserialization
+
+Moving data from source to destination involves serialization and deserialization. As a reminder, serialization means encoding the data from a source and preparing data structures for transmission and intermediate storage stages. When ingesting data, ensure that your destination can deserialize the data it receives. We’ve seen data ingested from a source but then sitting inert and unusable in the destination because the data cannot be properly deserialized.
+
+### Throughput and Scalability
+
+In theory, your ingestion should never be a bottleneck. In practice, ingestion bottle‐ necks are pretty standard. Data throughput and system scalability become critical as your data volumes grow and requirements change. 
+
+Another thing to consider is your ability to handle bursty data ingestion. Data gener‐ ation rarely happens at a constant rate and often ebbs and flows. Built-in buffering is required to collect events during rate spikes to prevent data from getting lost.
+
+### Reliability and Durability
+
+Reliability and durability are vital in the ingestion stages of data pipelines. Reliability entails high uptime and proper failover for ingestion systems. Durability entails making sure that data isn’t lost or corrupted.
+
+### Payload
+
+A payload is the dataset you’re ingesting and has characteristics such as kind, shape, size, schema and data types, and metadata
+
+### Push vs. Pull vs. Poll Patterns
+
+A push strategy involves a source system sending data to a target
+
+![image](https://github.com/user-attachments/assets/6a455e57-db1e-48a3-bb76-1584fd36ec91)
+
+A pull strategy (Figure 7-8) entails a target reading data directly from a source
+
+![image](https://github.com/user-attachments/assets/9dcb314a-94c1-4b1d-b155-c6741ec1da47)
+
+Another pattern related to pulling is polling for data. Polling involves periodically checking a data source for any changes. When changes are detected, the destination pulls the data as it would in a regular pull situation.
+
+![image](https://github.com/user-attachments/assets/34e65292-54aa-426b-b1de-09c0b3f9641c)
+
+### Batch Ingestion Considerations
+
+![image](https://github.com/user-attachments/assets/d08e24c5-4830-4bdb-9c79-a969e706048f)
+![image](https://github.com/user-attachments/assets/0953e004-da3d-4c8a-abd6-df7a1d3ee676)
+
+* snapshot or differential extraction
+* file-based export and ingestion
+* ETL vs ELT
+* inserts, updates, and batch size
+* data migration
+
+### Message and Stream Ingestion Considerations
+
+* schema evaluation
+* late-arriving data
+* ordering and multiple delivery
+* replay
+* time to live
+* message size
+* error handling and dead-letter queues
+* consumer pull and push
+
+## Finance Voice Assistant
+
+After reading today, I thought about going back to complete one of my project ideas - creating a finance voice assistant. 
+
+I can share that a working version is available on [my github](https://github.com/divakaivan/finance_voice_assistant/). I ended up using llama index for the pipeline, inspired by [this](https://youtu.be/L1o1VPVfbb0) amazin tutorial that showed up how to use llama index to query a db using natural language. 
+
+It is a bit late now, but I can share some basic pictures:
+
+RAG DAG:
+
+<img width="952" alt="image" src="https://github.com/user-attachments/assets/69bd8d52-5994-464e-9e9d-bded782b1062">
+
+Streamlit UI:
+
+<img width="1390" alt="image" src="https://github.com/user-attachments/assets/1d83ba61-c3cb-4556-bd37-55ca54658726">
+
+A very cool thing is this tracing app called Pheonix, where we see the RAP pipeline's executaion and what the LLM sees throughout
+
+<img width="1495" alt="image" src="https://github.com/user-attachments/assets/bac167ba-8c81-4700-86d0-5fe86aab80a1">
+
+I will document the whole thing tomorrow and maybe make some more improvements.
+
+
+That is all for today!
+
+See you tomorrow :)

From 3542b0d46c614e961f6a83f96bf0b74c7e13d2a0 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 14 Aug 2024 21:22:05 +0900
Subject: [PATCH 377/638] Create 2024-08-14-day-226.md

---
 _posts/2024-08-14-day-226.md | 100 +++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 _posts/2024-08-14-day-226.md

diff --git a/_posts/2024-08-14-day-226.md b/_posts/2024-08-14-day-226.md
new file mode 100644
index 000000000..e6cc6bb04
--- /dev/null
+++ b/_posts/2024-08-14-day-226.md
@@ -0,0 +1,100 @@
+---
+layout: post
+title: (Day 226) Your Personal Finance Assistant
+categories: [applying-knowledge,nlp]
+---
+
+# Hello :) Today is Day 226!
+A quick summary of today:
+* finishing and documenting the personal finance assistant [project](https://github.com/divakaivan/personal_finance_assistant)
+
+The full information and code is on [my github](https://github.com/divakaivan/personal_finance_assistant), and below I will give some info about today's improvements.
+
+## I added a `prepare_data.py` script
+
+For reproducability, I prepared a script that takes the raw dataset, preprocesses it, indexes, creates description - all files needed for the UI to work, and outputs it to the folder where they can be directly read by the app.
+
+## Added a Makefile for easy setup
+
+```
+Usage: make [option]
+
+Options:
+  help                 Show this help message
+  install              Install dependencies
+  data                 Prepare data for the app (will overwrite any existing data)
+  start                Start app
+  del-audio            Delete any generated audio files
+```
+
+The last option is because the google TTS creates 3 mp3 files (for the welcome msg, user audio input and AI output), so for easy cleanup - all are deleted. 
+
+## Switched to a different streamlit mic recorder
+
+I was using another 3rd party mic input module, but it did not look very well in the UI and it did not work sometimes. 
+
+Old:
+
+<img width="188" alt="image" src="https://github.com/user-attachments/assets/16506e4a-1194-4aca-9140-4cbf547ca5be">
+
+It had all these buttons and while recording, a big audio box appears. 
+
+**The new one**
+
+<img width="141" alt="image" src="https://github.com/user-attachments/assets/96316833-8909-48eb-95f7-2c7f70bd135a">
+
+A simple button that even changes text from - Start Audio to Stop Audio (and I can edit it). Bottom line - it provides better user experience. 
+
+## Added prompt validity check
+
+If the user inputs something that does not seem logical, or related to transactions, or even '...a u h' (random letters) or if the transcribed audio is random letters because of some noise, the input first goes through gpt4o-mini to check whether:
+* it is related to transactions
+* related to the chat history
+* not random letters
+
+Maybe there is a better way to do it, but at the moment the code is:
+
+```python
+def check_valid_prompt(user_text_query: str, chat_history: List[Dict[str, str]]) -> str:
+    """
+    Check if the user query is valid and related to finance transactions.
+    """
+    check_prompt = f"You are a personal finance assistant. Here is the conversation history so far: {chat_history}\n\nDoes the below user query look correct and related to finance transactions and the chat history? Answer with 'Yes' or 'No':\n\n{user_text_query}"
+    gpt4omini = gpt_client()
+    response = gpt4omini.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": check_prompt}]
+    )
+    llm_check_response = response.choices[0].message.content
+    return llm_check_response
+```
+
+## Added project info
+
+* Used Dataset - Kaggle
+* Database - SQLite
+* LLM - OpenAI's GPT
+* Audio Transcription Model - openai/whisper-small from HuggingFace
+* RAG pipeline - LlamaIndex
+* Monitoring/Tracing - Azire Pheonix
+* User Interface - Streamlit
+
+UI example usage:
+
+![image](https://github.com/user-attachments/assets/c0ccfd9c-8a12-498f-89da-72014f4bd0fd)
+
+Monitoring example:
+
+![image](https://github.com/user-attachments/assets/e9df9419-6743-434c-81a4-6899eea2e3e2)
+
+Created a project diagram:
+
+![image](https://github.com/user-attachments/assets/e92642b7-3122-4717-9d8a-81d37a94905a)
+
+## Recorder a short demo
+
+Unfortunately, the demo is best seen in the repo itself.
+
+That is all for today!
+
+See you tomorrow :) 

From f4ccb94618ec82039a0713b1568db3e1d88c6647 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 15 Aug 2024 00:16:25 +0900
Subject: [PATCH 378/638] Update projects.md

---
 _pages/projects.md | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 4d6947273..144590ab9 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -5,9 +5,24 @@ title: Projects
 ---
 
 <div id="projects">
+  <div class="project-item">
+    <h3>Your Personal Finance Voice Assistant</h3>
+    <p>
+      <li>Chat, Talk, Learn and Analyse your spending habits with your Personal Finance Assistant</li>
+      <li>Communicate through text or speech</li>
+      <li>Ask follow-up questions (the language model can see the chat history)</li>
+      <li>On the dev side, see how the RAG is performing by analysing prompts and retrieved information</li>
+    </p>
+    <p><b>Tech: SQLite, OpenAI, HuggingFace, LlamaIndex, Azire Pheonix, Streamlit</b></p>
+    <a href="https://github.com/divakaivan/personal_finance_assistant" target="_blank">View on GitHub</a>
+  </div>
+  
   <div class="project-item">
     <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
-    <p>An AI-driven solution for real-time credit card fraud detection using MLOps techniques. It involves fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring. The system focuses on high fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models.
+    <p>
+      <li>An AI-driven solution for real-time credit card fraud detection using MLOps techniques</li>
+      <li>Fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring</li>
+      <li>High fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models</li>
     </p>
     <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
     <a href="https://github.com/divakaivan/kb_project" target="_blank">View on GitHub</a>
@@ -15,7 +30,10 @@ title: Projects
   
   <div class="project-item">
     <h3>MLOps Architecture for Insurance Fraud Detection</h3>
-    <p>This project is a capstone for DataTalksClub's MLOps Zoomcamp. It involves building an end-to-end MLOps pipeline to detect car insurance fraud. The pipeline integrates tools like Terraform, Prefect, MLflow, Docker, Grafana, and Evidently, covering aspects from data storage, data preprocessing (using IV and WoE), model training, deployment, and monitoring. The project emphasizes reproducibility, orchestration, and monitoring with a focus on achieving high recall in fraud detection using a Balanced Random Forest Classifier.
+    <p>
+      <li>building an end-to-end MLOps pipeline to detect car insurance fraud</li>
+      <li>Pipeline orchestration covering data storage, data preprocessing (using IV and WoE), model training, deployment, and monitoring</li>
+      <li>Focus on achieving high recall in fraud detection using a Balanced Random Forest Classifier</li>
     </p>
     <p><b>Tech: PostgreSQL DB, Terraform, Google Cloud Platform, Mlflow, Prefect, Grafana, Evidently, Docker, FastAPI</b></p>
     <a href="https://github.com/divakaivan/insurance-fraud-mlops-pipeline" target="_blank">View Project</a>
@@ -23,7 +41,12 @@ title: Projects
 
   <div class="project-item">
     <h3>Transaction Stream Data Engineering Pipeline</h3>
-    <p>This project starts by generating transaction data via Stripe's API, streams it using Apache Kafka, and processes it in real-time with PySpark Structured Streaming. The processed data is stored in PostgreSQL, managed by dbt for data transformations and modelling, and visualized using Grafana.
+    <p>
+      <li>Generate transaction data via Stripe's API</li>
+      <li>Stream data using Apache Kafka and process it in real-time with PySpark Structured Streaming</li>
+      <li>Store processed data in PostgreSQL</li>
+      <li>Manage data transformations and modeling using dbt</li>
+      <li>Visualize data using Grafana</li>
     </p>
     <p><b>Tech: PostgreSQL DB, Kafka, PySpark, dbt, Grafana</b></p>
     <a href="https://github.com/divakaivan/transaction-stream-data-pipeline" target="_blank">View Project</a>
@@ -31,7 +54,13 @@ title: Projects
 
   <div class="project-item">
     <h3>Lending Club Data Engineering Pipeline</h3>
-    <p>This project involves building a data pipeline using various tools and technologies to process and visualize Lending Club data. The pipeline extracts raw data from Kaggle, loads it into Google Cloud Storage, processes it with dbt in BigQuery, and creates visualizations using Looker. Terraform is used for infrastructure management, and Mage orchestrates the entire process.
+    <p>
+      <li>Build a data pipeline to process and visualize Lending Club data</li>
+      <li>Extract raw data from Kaggle and load it into Google Cloud Storage</li>
+      <li>Process data with dbt in BigQuery</li>
+      <li>Create visualizations using Looker</li>
+      <li>Manage infrastructure with Terraform</li>
+      <li>Orchestrate the entire process with Mage</li>
     </p>
     <p><b>Tech: Docker, Mage orchestration, Google Cloud Platform, Terraform, dbt, Looker</b></p>
     <a href="https://github.com/divakaivan/lending-club-data-pipeline" target="_blank">View Project</a>

From 35936bda27b0a0aa3d20fff3168a7b5673cd9674 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 15 Aug 2024 00:20:45 +0900
Subject: [PATCH 379/638] Update projects.md

---
 _pages/projects.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index 144590ab9..711c83785 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -13,7 +13,7 @@ title: Projects
       <li>Ask follow-up questions (the language model can see the chat history)</li>
       <li>On the dev side, see how the RAG is performing by analysing prompts and retrieved information</li>
     </p>
-    <p><b>Tech: SQLite, OpenAI, HuggingFace, LlamaIndex, Azire Pheonix, Streamlit</b></p>
+    <p><b>Tech: SQLite, OpenAI, HuggingFace, LlamaIndex, Azire Pheonix monitoring, Streamlit</b></p>
     <a href="https://github.com/divakaivan/personal_finance_assistant" target="_blank">View on GitHub</a>
   </div>
   
@@ -31,7 +31,7 @@ title: Projects
   <div class="project-item">
     <h3>MLOps Architecture for Insurance Fraud Detection</h3>
     <p>
-      <li>building an end-to-end MLOps pipeline to detect car insurance fraud</li>
+      <li>Building an end-to-end MLOps pipeline to detect car insurance fraud</li>
       <li>Pipeline orchestration covering data storage, data preprocessing (using IV and WoE), model training, deployment, and monitoring</li>
       <li>Focus on achieving high recall in fraud detection using a Balanced Random Forest Classifier</li>
     </p>

From 4b4c10c5b5f3d28cf1129cd3b1287df02cfa50d3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 15 Aug 2024 00:37:15 +0900
Subject: [PATCH 380/638] Update projects.md

---
 _pages/projects.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/_pages/projects.md b/_pages/projects.md
index 711c83785..aa82f1533 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -27,6 +27,17 @@ title: Projects
     <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
     <a href="https://github.com/divakaivan/kb_project" target="_blank">View on GitHub</a>
   </div>
+
+  <div class="project-item">
+    <h3>Glaswegian Audio Dataset and ASR model</h3>
+    <p>
+      <li>Co-create a 120 minute open-sourced Glaswegian dataset</li>
+      <li>Preprocess raw audio and transcriptions and upload to HuggingFace</li>
+      <li>Research into audio AI models and fine-tune ASR and TTS models</li>
+    </p>
+    <p><b>Tech: HuggingFace, Python, Fine-Tuning, Audio AI</b></p>
+    <a href="https://huggingface.co/datasets/divakaivan/glaswegian_audio" target="_blank">View on HuggingFace</a>
+  </div>
   
   <div class="project-item">
     <h3>MLOps Architecture for Insurance Fraud Detection</h3>

From bc3e8408b811750dd5e352a8f16cd2f0fae06413 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 15 Aug 2024 21:35:48 +0900
Subject: [PATCH 381/638] Create 2024-08-15-day-227.md

---
 _posts/2024-08-15-day-227.md | 312 +++++++++++++++++++++++++++++++++++
 1 file changed, 312 insertions(+)
 create mode 100644 _posts/2024-08-15-day-227.md

diff --git a/_posts/2024-08-15-day-227.md b/_posts/2024-08-15-day-227.md
new file mode 100644
index 000000000..9d38a3385
--- /dev/null
+++ b/_posts/2024-08-15-day-227.md
@@ -0,0 +1,312 @@
+---
+layout: post
+title: (Day 227) Reading more about DL at scale
+categories: [statistics,theory,deep-learning]
+---
+
+# Hello :) Today is Day 227!
+A quick summary of today:
+* reading a bit more of the Deep Learning at Scale book on O'Reilly
+* started a new book by a Microsoft data scientist and ex-MIT researcher
+* found some new courses to try out
+
+
+## Deep Learning at Scale Chapter 2. Deep Learning
+
+![image](https://github.com/user-attachments/assets/d15a7ca7-efc5-43a0-9255-610ef98c09be)
+
+### Data Flow in Deep Learning
+
+![image](https://github.com/user-attachments/assets/9c0880eb-7456-46e1-9d37-3989811805bf)
+
+1. **Incoming Data & Data at Rest**:
+   - **Foundation**: Data collection, governance, labeling, and management.
+   - **Human-Centric**: Heavily involves humans in data generation and labeling.
+   - **Efficiency**: A fluid process here enhances the scalability of the model development.
+
+2. **Data Preprocessing**:
+   - **ETL Process**: Involves extracting, transforming, and loading data for model use.
+   - **Pipeline**: Feeds input data into the model.
+
+3. **The Model**:
+   - **Mathematical Operations**: Series of chained operations forming the model.
+   - **Compression Algorithm**: Extracts latent knowledge during learning.
+
+4. **Forward Pass**:
+   - **Prediction**: Applying the chain of operations to produce the model's output.
+   - **Initialization**: Parameters are initialized randomly or via transfer learning.
+
+5. **Error Calculation**:
+   - **Feedback**: Measures how wrong the model’s predictions are.
+   - **Loss Function**: Guides learning by defining error through a specific mathematical function.
+
+6. **Backward Pass**:
+   - **Parameter Adjustment**: Modifies model parameters based on error feedback.
+   - **Gradient Descent**: Uses derivatives to adjust parameters, minimizing error iteratively.
+
+7. **Output**:
+   - **Final Model**: Represents "compressed" knowledge, ready for inference.
+   - **Inference**: Applying the trained model to new data to make predictions.
+
+8. **Feedback**:
+   - **Continual Learning**: Updates the model post-inference to maintain relevance.
+   - **Monitoring**: Includes model monitoring, drift analysis, and user feedback to improve the model.
+
+Besides this, the other bit was using the [book code](https://github.com/suneeta-mall/deep_learning_at_scale/tree/main) and creating a simple tensorflow and then pytorch lightning model and using it for inference on MNIST. 
+
+## Deep Learning at Scale Chapter 3. The Computational Side of Deep Learning
+
+This chapter talked about how computations are performed on the hardware and how acceleration is achieved through hardware advancements. It concentrated more on the hardware side, and below is a quick overview:
+
+### Computer Architecture
+- **The Birth of the Electromechanical Engine**: Discusses the origins of computing machines, focusing on the development of the first electromechanical engines.
+- **Memory and Persistence**: Explores the concepts of memory in computing, particularly how data is stored and retained over time.
+- **Computation and Memory Combined**: Looks at the integration of computation and memory within computing systems, highlighting their interdependence.
+
+### The Scaling Laws of Electronics
+- Covers the principles that govern the scaling of electronic components, likely referring to Moore's Law and related concepts.
+
+### Scaling Out Computation with Parallelization
+- **Threads Versus Processes: The Unit of Parallelization**: Compares and contrasts threads and processes, examining their roles in parallel computing.
+- **Hardware-Optimized Libraries for Acceleration**: Discusses specialized libraries designed to enhance computational performance through hardware optimization.
+- **Parallel Computer Architectures: Flynn’s and Duncan’s Taxonomies**: Explores the classification of parallel computer architectures using Flynn's and Duncan's taxonomies.
+
+### Accelerated Computing
+- **Popular Accelerated Devices for Deep Learning**: Reviews commonly used hardware accelerators that boost the performance of deep learning tasks.
+- **CUDA**: Focuses on CUDA, a parallel computing platform and programming model developed by NVIDIA, which is widely used for high-performance computing and deep learning.
+
+## Deep Learning at Scale Chapter 4. Putting It All Together: Efficient Deep Learning
+
+### GPT-2
+
+#### Key contributors to scale
+
+* Transformer attention block
+* Unsupervised training
+* Zero-shot learning
+* Parameter scale
+
+provided GPT2 model code:
+
+```python
+datamodule = WikiDataModule(name=model_name, batch_size=batch_size, 
+                            num_workers=num_workers)
+model = GPT2Module(name=model_name)
+
+trainer = PLTrainer(
+    accelerator="auto",
+    devices=="auto",
+    max_epochs=max_epochs,
+    callbacks=[
+        TQDMProgressBar(refresh_rate=refresh_rate),
+        ckpt_cb,
+        DeviceStatsMonitor(cpu_stats=True),
+        EarlyStopping(monitor="val/loss", mode="min"),
+    ],
+    logger=[
+        exp_logger,
+        TensorBoardLogger(save_dir=result_dir / "logs"),
+    ],
+    profiler=torch_profiler,
+)
+trainer.fit(model, datamodule)
+```
+
+Everything is provided so to run anything, the book gives a simple command like:
+
+```
+deep-learning-at-scale chapter_4 train_gpt2
+```
+
+The book introduces Aim for experiment tracking. It is an alternative to MLflow and I will keep it in mind to check later and try it myself. 
+
+```python
+exp_logger = AimLogger(
+    experiment=exp_name,
+    train_metric_prefix="train/",
+    val_metric_prefix="val/",
+    test_metric_prefix="test/",
+)
+```
+
+### Modeling Techniques to Scale Training on a Single Device
+
+#### Graph Compilation
+
+- torch.compile: Converts `torch.nn.Module` to `OptimizedModule`, optimizing graph modules for performance gains (30% to 200%).
+- fullgraph: Generates a static graph without eager mode fallback, more efficient when control flow is absent.
+- Compilation Modes:
+  - default: Balanced approach for large models.
+  - reduce-overhead: Minimizes framework overhead, suitable for smaller models.
+  - max-autotune: Maximizes tuning for the fastest performance but has the longest compilation time.
+- Example Usage:
+
+```python
+torch.compile(self.model, mode="max-autotune")
+```
+
+- Debugging: Use TORCH_LOGS="graph_breaks,recompiles" for debugging compilation issues.
+- Batch Size Optimization: For an A100 SXM 80 GB GPU, a batch size of 24 is optimal for training with single precision (fp32).
+
+#### Reduced- and Mixed-Precision Training
+
+- Memory Consumption: Divided into model parameters, input data, activations, gradients, optimizer states, and metrics/losses.
+- Mixed Precision:
+  - Combines single- and half-precision formats for efficient computation.
+  - Implemented using torch.cuda.amp for CUDA-compatible devices.
+  - Memory Penalty: Requires 1.5x memory compared to single precision.
+- Usage:
+
+```python
+Trainer(precision="16-mixed")
+```
+
+- Gradient Scaling and Clipping: 
+  - Gradient Scaling: Manages precision loss with torch.cuda.amp.GradScaler.
+  - Gradient Clipping: Mitigates exploding gradients by capping gradient values.
+
+- 8-bit Optimizers and Quantization:
+  - Techniques like dynamic tree quantization and block quantization help reduce memory while maintaining accuracy.
+  - Implementations available in bitsandbytes library, e.g., bnb.optim.Adam8bit.
+
+#### Memory Tricks for Efficiency
+
+- Memory Layout:
+  - Channels-first (NCHW) vs. Channels-last (NHWC) formats.
+  - Switching to channels-last can provide up to 1.8x performance gain on CPUs.
+- Example
+
+```python
+images = images.to(memory_format=torch.channels_last)
+self.model = self.model.to(memory_format=torch.channels_last)
+```
+
+- Feature Compression: Reduces memory footprint but may introduce performance overhead.
+- Meta and Fake Tensors: Represent tensor shapes and types without actual memory allocation.
+
+#### Optimizer Efficiencies
+
+- Stochastic Gradient Descent (SGD): Propagates gradients in batches, making it effective for large datasets.
+- Gradient Accumulation: Simulates larger batches by accumulating gradients over several steps.
+- Gradient Checkpointing: Balances memory efficiency with computational overhead by saving gradients at checkpoints.
+- Patch Gradient Descent (PatchGD): Scales training of large images by accumulating gradients across image patches.
+
+#### Learning Rate and Weight Decay
+
+- Learning Rate: Crucial for convergence time; needs careful tuning to avoid suboptimal models.
+- Weight Decay: Helps regularize models and mitigate overfitting.
+
+#### Model Input Pipeline Tricks
+
+- Efficient DataLoaders and scalable transformations are crucial as dataset size increases, ensuring that GPUs remain fully utilized.
+
+## Starting [Machine Learning Algorithms in Depth](https://www.manning.com/books/machine-learning-algorithms-in-depth)
+
+<img width="621" alt="image" src="https://github.com/user-attachments/assets/53739b16-761e-4f12-8447-fc2f6e279559">
+
+I saw a Microsoft data scientist - VADIM SMOLYAKOV, published a new book on Manning, after looking at the outline I decided to use one of my free credits on there and purchase it.
+
+### Chapter 1: Machine learning algorithms
+
+- **Algorithm Definition**:
+  - An algorithm is a sequence of steps to achieve a task, taking an input, performing operations, and producing an output.
+  - Example: Sorting a list of integers.
+
+- **Algorithm Performance**:
+  - Two key metrics:
+    - **Runtime Complexity**: Measures the speed of execution.
+    - **Space Complexity**: Measures the amount of memory required.
+
+- **Machine Learning (ML) Algorithms**:
+  - **Supervised Learning**:
+    - Learns a mapping from inputs \(x\) to outputs \(y\) using labeled data.
+    - **Types**:
+      - **Classification**: Predicts discrete labels (e.g., classifying emails as spam or not).
+      - **Regression**: Predicts continuous values (e.g., forecasting stock prices).
+    - **Loss Functions**:
+      - **Classification**: Measures misclassification rate.
+      - **Regression**: Uses Mean Squared Error (MSE) to measure deviation from the true value.
+
+  - **Unsupervised Learning**:
+    - Identifies patterns in data without labeled outputs.
+    - Example: Clustering data points based on similarity.
+
+  - **Deep Learning**:
+    - Involves multiple layers of computation to learn complex patterns.
+    - Uses backpropagation to optimize parameters.
+    - Applications: Natural Language Processing (NLP), Computer Vision (CV), etc.
+
+- **Bayesian Inference**:
+  - Updates beliefs based on new data.
+  - Involves calculating the posterior distribution using prior knowledge and observed data.
+
+- **Types of Bayesian Inference**:
+  - **Markov Chain Monte Carlo (MCMC)**: Samples from high-dimensional spaces to approximate distributions.
+  - **Variational Inference (VI)**: Approximates the posterior by optimizing a simpler distribution.
+
+- **Importance of Learning Algorithms from Scratch**:
+  - Enables better understanding, selection, and troubleshooting of ML algorithms.
+  - Facilitates extending and improving existing models.
+
+- **Mathematical Background**:
+  - Understanding of probability, calculus, and linear algebra is crucial for ML algorithms.
+
+- **Software Implementation**:
+  - Focus on efficient coding practices, using Python, with attention to data structures and problem-solving paradigms like:
+    - **Complete Search**: Traverses the entire search space.
+    - **Greedy Algorithms**: Makes locally optimal choices.
+    - **Divide and Conquer**: Solves problems by breaking them into sub-problems.
+    - **Dynamic Programming**: Solves overlapping subproblems and stores results.
+
+### Chapter 2: Markov chain Monte Carlo
+
+#### 1. Markov chain Monte Carlo (MCMC) is a methodology of sampling from high-dimensional parameter spaces to approximate the posterior distribution \( p(\theta|x) \).
+
+MCMC methods are used in situations where direct sampling from the posterior distribution \( p(\theta|x) \) is challenging, especially in high-dimensional spaces. MCMC constructs a Markov chain, where each state in the chain represents a sample of the parameters \( \theta \), and the chain is designed so that its stationary distribution is the desired posterior distribution. By running the chain for a sufficient number of steps, the samples generated can be used to approximate \( p(\theta|x) \). These methods are particularly useful in Bayesian statistics, where the posterior distribution is often complex and cannot be directly computed.
+
+#### 2. Monte Carlo (MC) integration has an advantage over numerical integration (which evaluates a function at a fixed grid of points), in that the function is only evaluated in places where there is a nonnegligible probability.
+
+Monte Carlo integration is a method of estimating the value of an integral using random sampling. Unlike traditional numerical integration, which evaluates a function at predetermined grid points, MC integration focuses on sampling points in the domain where the function value contributes most to the integral. This is particularly beneficial in high-dimensional spaces where many regions have negligible contributions to the integral. By concentrating computational effort on areas with significant probability, MC integration can be more efficient and accurate, especially when dealing with complex or high-dimensional integrals.
+
+#### 3. In a binomial tree model of a stock price, we assume that at each time step, the stock could be in either up or down states with unequal payoffs characteristic of a risky asset.
+
+The binomial tree model is a discrete-time model for the price evolution of financial assets, such as stocks. At each time step, the model assumes the stock price can move to one of two possible states: "up" or "down." The movement is determined by probabilities assigned to each state, and the payoffs (or returns) are generally unequal, reflecting the inherent risk of the asset. This model captures the uncertainty in stock price movements and is widely used in option pricing and other financial derivatives, providing a flexible framework to approximate continuous-time models like the Black-Scholes model.
+
+#### 4. Self-avoiding random walks are simply random walks that do not cross themselves. We can use Monte Carlo integration to simulate a self-avoiding random walk.
+
+A self-avoiding random walk (SAW) is a type of random walk where the path taken does not revisit any previously occupied position. This restriction makes the problem more complex and is particularly relevant in modeling physical phenomena like polymer chains. Monte Carlo methods can be used to simulate SAWs by generating many random walk paths and then selecting the ones that do not intersect themselves. This simulation can help estimate properties of the walk, such as its average length or the probability of reaching a certain point without crossing itself.
+
+#### 5. Gibbs sampling is based on the idea of sampling one variable at a time from a multidimensional distribution conditioned on the latest samples from all the other variables.
+
+Gibbs sampling is a specific MCMC method used to sample from a joint distribution when direct sampling is difficult. It works by iteratively sampling each variable in a multidimensional distribution while keeping all other variables fixed at their most recent values. The new value for each variable is drawn from its conditional distribution, given the current values of all other variables. Over many iterations, the samples converge to the joint distribution of interest. Gibbs sampling is especially useful when the full conditional distributions are easier to sample from than the joint distribution.
+
+#### 6. The basic idea of the Metropolis-Hastings (MH) algorithm is to propose a move from the current state \( x \) to a new state \( x' \) based on a proposal distribution \( q(x'|x) \) and then either accept or reject the proposed state according to the MH ratio that ensures detailed balance is satisfied.
+ 
+The Metropolis-Hastings algorithm is a versatile MCMC method for sampling from complex distributions. It starts from a current state \( x \) and proposes a new state \( x' \) using a proposal distribution \( q(x'|x) \). The proposed move is then accepted with a probability that depends on the ratio of the target distribution at the new state to that at the current state, adjusted by the proposal distribution. If the move is accepted, the new state becomes the current state; otherwise, the current state remains unchanged. This acceptance rule ensures that the chain will eventually sample from the desired distribution, satisfying the property of detailed balance.
+
+#### 7. The idea of importance sampling is to draw samples in interesting regions (i.e., where both \( p(x) \) and \( |f(x)| \) are large). Importance sampling works by drawing samples from an easier-to-sample proposal distribution \( q(x) \).
+
+Importance sampling is a technique used to estimate properties of a distribution \( p(x) \) by focusing sampling effort on regions that contribute most to the quantity being estimated. Instead of sampling directly from \( p(x) \), which might be difficult, importance sampling draws samples from a proposal distribution \( q(x) \) that is easier to sample from. The key idea is to weight the samples according to the ratio of the target distribution \( p(x) \) to the proposal distribution \( q(x) \). This weighting corrects for the bias introduced by sampling from \( q(x) \) and allows for more efficient estimation by concentrating on "important" regions of the distribution.
+
+
+## I was also on the look for things to learn
+
+Today I found some interesting courses that I can do while reading the above two books:
+
+* [The Complete dbt (Data Build Tool) Bootcamp: Zero to Hero](https://www.udemy.com/course/complete-dbt-data-build-tool-bootcamp-zero-to-hero-learn-dbt)
+* [What is Dagster?](https://youtu.be/Xe8wYYC2gWQ)
+* [Algorithmic trading with Apache Flink](https://datamasterylab.com/home/course/algorithmic-trading-with-apache-flink-end-to-end-data-engineering/33)
+
+On Coursera:
+
+* [Large Language Model Operations (LLMOps) Specialization](https://www.coursera.org/specializations/large-language-model-operations)
+* [Building Cloud Computing Solutions at Scale Specialization](https://www.coursera.org/specializations/building-cloud-computing-solutions-at-scale)
+* [MLOps | Machine Learning Operations Specialization](https://www.coursera.org/specializations/mlops-machine-learning-duke)
+* [Applied Python Data Engineering Specialization](https://www.coursera.org/specializations/python-data-engineering)
+
+Some of them cost a little money, but I need to pick which one to go for. The dbt one was highly recommended by many in the data eng subreddit. And the coursera ones are taught by Duke Uni professors. 
+
+
+That is all for today!
+
+See you tomorrow :)

From 8ee616ef5b36666c767eb5b1c49fd153e2b77a38 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 15 Aug 2024 22:29:50 +0900
Subject: [PATCH 382/638] Update 2024-08-15-day-227.md

---
 _posts/2024-08-15-day-227.md | 45 ++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/_posts/2024-08-15-day-227.md b/_posts/2024-08-15-day-227.md
index 9d38a3385..9d015ec40 100644
--- a/_posts/2024-08-15-day-227.md
+++ b/_posts/2024-08-15-day-227.md
@@ -288,6 +288,51 @@ The Metropolis-Hastings algorithm is a versatile MCMC method for sampling from c
 
 Importance sampling is a technique used to estimate properties of a distribution \( p(x) \) by focusing sampling effort on regions that contribute most to the quantity being estimated. Instead of sampling directly from \( p(x) \), which might be difficult, importance sampling draws samples from a proposal distribution \( q(x) \) that is easier to sample from. The key idea is to weight the samples according to the ratio of the target distribution \( p(x) \) to the proposal distribution \( q(x) \). This weighting corrects for the bias introduced by sampling from \( q(x) \) and allows for more efficient estimation by concentrating on "important" regions of the distribution.
 
+### Chapter 3: Variational inference
+
+#### 1. The main idea of variational inference is to choose an approximate distribution q(x) from a family of tractable distributions and then make this approximation as close as possible to the true posterior distribution p(x).
+
+In Bayesian inference, we often want to compute the posterior distribution \( p(x \mid y) \), which represents the distribution of latent variables \( x \) given observed data \( y \). However, the exact computation of the posterior is often intractable due to the complexity of the model or the high dimensionality of the data. 
+
+**Variational Inference** offers a solution by introducing a simpler, more tractable distribution \( q(x) \) from a chosen family of distributions. Instead of directly working with the complex posterior \( p(x \mid y) \), we approximate it using \( q(x) \). The goal is to make \( q(x) \) as close as possible to \( p(x \mid y) \) by minimizing a divergence measure, typically the Kullback-Leibler (KL) divergence. This approach allows us to perform inference in models where exact inference is computationally prohibitive.
+
+
+
+#### 2. An evidence lower bound is an objective function that we seek to maximize to learn the variational parameters of our model.
+
+The **Evidence Lower Bound (ELBO)** is a key concept in variational inference. It serves as a surrogate objective function that we maximize to find the optimal parameters of the variational distribution \( q(x) \). The ELBO is derived from the marginal likelihood (or evidence) \( p(y) \) of the observed data, which is generally difficult to compute directly.
+
+The ELBO is defined as:
+
+\[
+ELBO = \mathbb{E}_{q(x)}[\log p(y, x)] - \mathbb{E}_{q(x)}[\log q(x)]
+\]
+
+_(need to figure out how to do math notation so github's markdown can read the formulas properly)_
+
+Maximizing the ELBO is equivalent to minimizing the KL divergence between the variational distribution \( q(x) \) and the true posterior \( p(x \mid y) \). By optimizing the ELBO, we adjust the parameters of \( q(x) \) to make it a better approximation of the posterior distribution.
+
+
+
+#### 3. In mean-field approximation, we assume the approximate distribution q(x) is fully factorized.
+
+The **Mean-Field Approximation** is a simplification method used in variational inference to make the problem more tractable. In this approach, we assume that the approximate distribution \( q(x) \) can be factorized into independent distributions over the individual latent variables:
+
+\[
+q(x) = \prod_{i} q_i(x_i)
+\]
+
+This assumption reduces the complexity of the optimization problem because we now optimize each \( q_i(x_i) \) independently, rather than dealing with a joint distribution over all variables. While this assumption can significantly simplify computation, it may lead to suboptimal approximations if the true posterior \( p(x \mid y) \) exhibits strong dependencies among the latent variables.
+
+
+#### 4. Mutual information maximization can be carried out by deriving and maximizing the MI lower bound.
+
+**Mutual Information (MI)** is a measure of the amount of information that one random variable \( X \) contains about another random variable \( Y \). In many machine learning tasks, maximizing MI can be beneficial, for example, in feature selection, clustering, or unsupervised learning.
+
+However, directly computing MI can be challenging, especially in high dimensions. To address this, we can derive a **lower bound** on the MI, similar to the ELBO in variational inference. By maximizing this lower bound, we can effectively maximize the MI without needing to compute it directly.
+
+One common approach to derive such a bound is to use variational methods, where we introduce an auxiliary distribution to approximate parts of the MI. The objective then becomes maximizing this bound with respect to the parameters of the auxiliary distribution, leading to an increase in the true MI.
+
 
 ## I was also on the look for things to learn
 

From 2508ff1da9aa09bb532a8451cae1fb8827a1b981 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 16 Aug 2024 20:49:38 +0900
Subject: [PATCH 383/638] Create 2024-08-16-day-228.md

---
 _posts/2024-08-16-day-228.md | 1072 ++++++++++++++++++++++++++++++++++
 1 file changed, 1072 insertions(+)
 create mode 100644 _posts/2024-08-16-day-228.md

diff --git a/_posts/2024-08-16-day-228.md b/_posts/2024-08-16-day-228.md
new file mode 100644
index 000000000..81f6efa00
--- /dev/null
+++ b/_posts/2024-08-16-day-228.md
@@ -0,0 +1,1072 @@
+---
+layout: post
+title: (Day 227) Making a poaster for the Not Google Devs Society
+categories: [statistics,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 228!
+A quick summary of today:
+* read a bit more of the ML Algorithms in depth book
+* made promotion material and a linkedin page for my potential society - [Not Google Devs](https://www.linkedin.com/company/not-google-devs)
+
+
+As a fan of the 'basics' and testing my knowledge, I decided to continue with the ML Algs in Depth book today.
+
+
+## Machine Learning Algorithms in Depth Chapter 4: Software implementation
+
+Why the chapter name? In the book it gave implementations for some of the algorithms explained below.
+
+### 1. **Data Structures: Storing and Organizing Data**
+A **data structure** is a systematic way of organizing and storing data so that it can be accessed and modified efficiently. Different data structures are suited for different types of applications, and they can be categorized into three main types:
+
+- Linear Data Structures: These store data in a linear or sequential order. Examples include arrays, linked lists, stacks, and queues. In these structures, elements are arranged one after another in memory.
+  
+- Nonlinear Data Structures: These store data in a hierarchical or interconnected way rather than in a straight line. Examples include trees and graphs. Nonlinear structures are used to represent complex relationships among data.
+  
+- Probabilistic Data Structures: These use randomization or probabilistic techniques to store and retrieve data efficiently, often with a trade-off between accuracy and performance. Examples include Bloom filters and skip lists.
+
+### 2. **Algorithmic Paradigms**
+An **algorithmic paradigm** is a general approach or strategy to solve a class of problems. The chapter discusses four key paradigms:
+
+- Complete Search: Also known as brute force, this method explores the entire search space to find a solution. It's exhaustive and guarantees finding the correct solution if one exists. Pruning techniques can be applied to cut off parts of the search space that don't need to be explored, improving efficiency.
+
+- Greedy Algorithm: This approach makes a series of choices, each of which looks best at the moment (locally optimum), with the hope that these choices lead to a globally optimum solution. However, greedy algorithms don't always produce the best solution for every problem.
+
+- Divide and Conquer: This technique divides a problem into smaller, independent subproblems, solves each subproblem, and then combines their solutions to solve the original problem. Classic examples include merge sort and quicksort algorithms.
+
+- Dynamic Programming (DP): This approach is similar to divide and conquer but is used when subproblems overlap. DP solves each subproblem once, stores the solution in a table, and reuses it when needed. This avoids redundant calculations and optimizes performance.
+
+### 3. **Complete Search (Brute Force)**
+**Complete search** or **brute force** involves exploring every possible option in the search space to find the correct solution. For example, if you were trying to crack a password, a brute force approach would involve trying every possible combination until the correct one is found. **Pruning** refers to eliminating parts of the search space that cannot possibly contain the solution, which improves the efficiency of the search.
+
+### 4. **Greedy Algorithm**
+A **greedy algorithm** makes decisions step-by-step, each time choosing the option that looks the best at that moment. The hope is that by making a series of locally optimal choices, you will arrive at a globally optimal solution. However, greedy algorithms do not always guarantee the best solution, and they work best for problems where a locally optimal choice leads to a globally optimal solution.
+
+### 5. **Divide and Conquer**
+**Divide and conquer** is a strategy that breaks down a problem into smaller, more manageable subproblems that are easier to solve. Each subproblem is solved independently, and then their solutions are combined to form a solution to the original problem. This method is effective for problems like sorting (e.g., merge sort) or finding the closest pair of points in a set.
+
+### 6. **Dynamic Programming (DP)**
+**Dynamic programming** is a method used when a problem can be broken down into overlapping subproblems. DP solves each subproblem once, stores the results in a table (often called a DP table), and reuses these results to construct the solution to the original problem. This approach is especially useful for optimization problems, like finding the shortest path in a graph.
+
+### 7. **Mastery Through Competitive Programming**
+Mastering algorithms and software implementation often requires extensive practice, which can be effectively achieved through **competitive programming**. This involves solving algorithmic problems under time constraints, which sharpens problem-solving skills and deepens understanding of various algorithms and data structures. Appendix A provides resources for getting started with competitive programming.
+
+### 8. **Machine Learning Mastery**
+To achieve mastery in **machine learning**, it’s crucial to have a strong grasp of the underlying mathematical and algorithmic principles. This includes knowledge of linear algebra, probability, statistics, and optimization. Appendix A suggests texts that can help deepen your understanding and broaden your expertise in these fundamental areas.
+
+### 9. **Staying Current in Machine Learning**
+The field of **machine learning** is evolving rapidly, with new techniques and algorithms being developed continuously. To stay up-to-date, it’s important to regularly read the latest research papers presented at major conferences like NeurIPS, ICML, and CVPR. These papers provide insights into the cutting-edge developments in the field.
+
+---
+
+## Machine Learning Algorithms in Depth Chapter 5: Classification
+
+This chapter (and I suspect all the next ones will be similar) is pretty cool for implementing each algorithm from scratch
+
+* The goal of a classification algorithm is to learn a mapping from inputs x to outputs y, where y is a discrete quantity.
+
+### Perceptron is a classification algorithm that updates the decision boundary until there are no more classification mistakes.
+
+```python
+import numpy as np 
+import seaborn as sns
+import matplotlib.pyplot as plt 
+
+from scipy.stats import randint
+from sklearn.datasets import load_iris
+from sklearn.metrics import confusion_matrix
+from sklearn.model_selection import train_test_split
+
+class perceptron:
+    def __init__(self, num_epochs, dim):
+        self.num_epochs = num_epochs
+        self.theta0 = 0
+        self.theta = np.zeros(dim)
+
+    def fit(self, X_train, y_train):
+        n = X_train.shape[0]
+        dim = X_train.shape[1]
+
+        k = 1
+        for epoch in range(self.num_epochs):
+            for i in range(n):
+                #sample random point
+                idx = randint.rvs(0, n-1, size=1)[0] 
+                #hinge loss
+                if (y_train[idx] * (np.dot(self.theta, X_train[idx,:]) + self.theta0) <= 0):
+                    #update learning rate
+                    eta = pow(k+1, -1)
+                    k += 1
+                    #print("eta: ", eta)
+
+                    #update theta
+                    self.theta = self.theta + eta * y_train[idx] * X_train[idx, :]
+                    self.theta0 = self.theta0 + eta * y_train[idx]            
+                #end if
+            print("epoch: ", epoch)
+            print("theta: ", self.theta)
+            print("theta0: ", self.theta0)
+            #end for
+        #end for
+
+    def predict(self, X_test):
+        n = X_test.shape[0]
+        dim = X_test.shape[1]
+
+        y_pred = np.zeros(n)
+        for idx in range(n):
+            y_pred[idx] = np.sign(np.dot(self.theta, X_test[idx,:]) + self.theta0)
+        #end for
+        return y_pred
+
+if __name__ == "__main__":
+    
+    #load dataset
+    iris = load_iris()
+    X = iris.data[:100,:]
+    y = 2*iris.target[:100] - 1 #map to {+1,-1} labels
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    #perceptron (binary) classifier
+    clf = perceptron(num_epochs=5, dim=X.shape[1])
+    clf.fit(X_train, y_train)
+    y_pred = clf.predict(X_test)
+
+    cmt = confusion_matrix(y_test, y_pred)
+    acc = np.trace(cmt)/np.sum(np.sum(cmt))
+    print("percepton accuracy: ", acc)
+    
+    #generate plots
+    plt.figure()
+    sns.heatmap(cmt, annot=True, fmt="d")
+    plt.title("Confusion Matrix"); plt.xlabel("predicted"); plt.ylabel("actual")
+    #plt.savefig("./figures/perceptron_acc.png")
+    plt.show()
+```
+
+Output:
+
+![image](https://github.com/user-attachments/assets/814e868f-ee92-431f-ab49-808ba644ab41)
+
+
+### SVM is a max-margin classifier. The training data points that lie on the margin boundaries become support vectors.
+
+```python
+import cvxopt
+import numpy as np
+
+from sklearn.svm import SVC #for comparison only
+from sklearn.datasets import load_iris
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import train_test_split
+
+def rbf_kernel(gamma, **kwargs):
+    def f(x1, x2):
+        distance = np.linalg.norm(x1 - x2) ** 2
+        return np.exp(-gamma * distance)
+    return f
+
+class SupportVectorMachine(object):
+    def __init__(self, C=1, kernel=rbf_kernel, power=4, gamma=None, coef=4):
+        self.C = C
+        self.kernel = kernel
+        self.power = power
+        self.gamma = gamma
+        self.coef = coef
+        self.lagr_multipliers = None
+        self.support_vectors = None
+        self.support_vector_labels = None
+        self.intercept = None
+
+    def fit(self, X, y):
+
+        n_samples, n_features = np.shape(X)
+
+        # Set gamma to 1/n_features by default
+        if not self.gamma:
+            self.gamma = 1 / n_features
+
+        # Initialize kernel method with parameters
+        self.kernel = self.kernel(
+            power=self.power,
+            gamma=self.gamma,
+            coef=self.coef)
+
+        # Calculate kernel matrix
+        kernel_matrix = np.zeros((n_samples, n_samples))
+        for i in range(n_samples):
+            for j in range(n_samples):
+                kernel_matrix[i, j] = self.kernel(X[i], X[j])
+
+        # Define the quadratic optimization problem
+        P = cvxopt.matrix(np.outer(y, y) * kernel_matrix, tc='d')
+        q = cvxopt.matrix(np.ones(n_samples) * -1)
+        A = cvxopt.matrix(y, (1, n_samples), tc='d')
+        b = cvxopt.matrix(0, tc='d')
+
+        if not self.C: #if its empty
+            G = cvxopt.matrix(np.identity(n_samples) * -1)
+            h = cvxopt.matrix(np.zeros(n_samples))
+        else:
+            G_max = np.identity(n_samples) * -1
+            G_min = np.identity(n_samples)
+            G = cvxopt.matrix(np.vstack((G_max, G_min)))
+            h_max = cvxopt.matrix(np.zeros(n_samples))
+            h_min = cvxopt.matrix(np.ones(n_samples) * self.C)
+            h = cvxopt.matrix(np.vstack((h_max, h_min)))
+
+        # Solve the quadratic optimization problem using cvxopt
+        minimization = cvxopt.solvers.qp(P, q, G, h, A, b)
+
+        # Lagrange multipliers
+        lagr_mult = np.ravel(minimization['x'])
+
+        # Extract support vectors
+        # Get indexes of non-zero lagr. multipiers
+        idx = lagr_mult > 1e-11
+        # Get the corresponding lagr. multipliers
+        self.lagr_multipliers = lagr_mult[idx]
+        # Get the samples that will act as support vectors
+        self.support_vectors = X[idx]
+        # Get the corresponding labels
+        self.support_vector_labels = y[idx]
+
+    # Calculate intercept with first support vector
+        self.intercept = self.support_vector_labels[0]
+        for i in range(len(self.lagr_multipliers)):
+          self.intercept -= self.lagr_multipliers[i] * self.support_vector_labels[
+             i] * self.kernel(self.support_vectors[i], self.support_vectors[0])
+
+
+    def predict(self, X):
+        y_pred = []
+    # Iterate through list of samples and make predictions
+        for sample in X:
+            prediction = 0
+            # Determine the label of the sample by the support vectors
+            for i in range(len(self.lagr_multipliers)):
+                prediction += self.lagr_multipliers[i] * self.support_vector_labels[
+                    i] * self.kernel(self.support_vectors[i], sample)
+            prediction += self.intercept
+            y_pred.append(np.sign(prediction))
+        return np.array(y_pred)
+
+
+def main():
+
+    #load dataset
+    iris = load_iris()
+    X = iris.data[:100,:]
+    y = 2*iris.target[:100] - 1 #map to {+1,-1} labels
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
+    clf = SupportVectorMachine(kernel=rbf_kernel, gamma = 1)
+    clf.fit(X_train, y_train)
+    y_pred = clf.predict(X_test)
+    accuracy = accuracy_score(y_test, y_pred)
+    print ("Accuracy (scratch):", accuracy)
+
+    clf_sklearn = SVC(gamma = 'auto')
+    clf_sklearn.fit(X_train, y_train)
+    y_pred2 = clf_sklearn.predict(X_test)
+    accuracy = accuracy_score(y_test, y_pred2)
+    print ("Accuracy :", accuracy)
+
+if __name__ == "__main__":
+    main()
+```
+
+Comparing from scratch and out-of-the-box SVM:
+
+![image](https://github.com/user-attachments/assets/bef3deda-3e0b-4968-81fa-55c81e5b02c2)
+
+
+### Logistic regression is a classification algorithm that computes class conditional density based on a softmax function.
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+
+def generate_data():
+        
+    n = 1000
+    mu1 = np.array([1,1])
+    mu2 = np.array([-1,-1])        
+    pik = np.array([0.4,0.6])
+        
+    X = np.zeros((n,2))
+    y = np.zeros((n,1))
+        
+    for i in range(1,n):
+        u = np.random.rand()
+        idx = np.where(u < np.cumsum(pik))[0]                            
+            
+        if (len(idx)==1):
+            X[i,:] = np.random.randn(1,2) + mu1
+            y[i] = 1
+        else:
+            X[i,:] = np.random.randn(1,2) + mu2
+            y[i] = -1
+              
+    return X, y    
+
+
+class sgdlr:
+    
+    def __init__(self):
+        
+        self.num_iter = 100
+        self.lmbda = 1e-9
+        
+        self.tau0 = 10
+        self.kappa = 1
+        self.eta = np.zeros(self.num_iter)
+        
+        self.batch_size = 200        
+        
+    def fit(self, X, y):
+
+        #random init
+        theta = np.random.randn(X.shape[1],1)
+        
+        #learning rate schedule
+        for i in range(self.num_iter):
+            self.eta[i] = (self.tau0+i)**(-self.kappa)        
+                
+        #divide data in batches        
+        batch_data, batch_labels = self.make_batches(X,y,self.batch_size)                    
+        num_batches = batch_data.shape[0]
+        num_updates = 0
+        
+        J_hist = np.zeros((self.num_iter * num_batches,1))
+        t_hist = np.zeros((self.num_iter * num_batches,1))
+        
+        for itr in range(self.num_iter):
+            for b in range(num_batches):
+                Xb = batch_data[b]
+                yb = batch_labels[b]
+
+                J_cost, J_grad = self.lr_objective(theta, Xb, yb, self.lmbda)
+                theta = theta - self.eta[itr]*(num_batches*J_grad)     
+
+                J_hist[num_updates] = J_cost
+                t_hist[num_updates] = np.linalg.norm(theta,2)
+                num_updates = num_updates + 1
+            print("iteration %d, cost: %f" %(itr, J_cost))
+        
+        y_pred = 2*(self.sigmoid(X.dot(theta)) > 0.5) - 1
+        y_err = np.size(np.where(y_pred - y)[0])/float(y.shape[0])
+        print("classification error:", y_err)
+
+        self.generate_plots(X, J_hist, t_hist, theta)
+        return theta        
+    
+    def make_batches(self, X, y, batch_size):
+        n = X.shape[0]
+        d = X.shape[1]
+        num_batches = int(np.ceil(n/batch_size))
+
+        groups = np.tile(range(num_batches),batch_size)
+        batch_data=np.zeros((num_batches,batch_size,d))
+        batch_labels=np.zeros((num_batches,batch_size,1))
+        
+        for i in range(num_batches):
+            batch_data[i,:,:] = X[groups==i,:]
+            batch_labels[i,:] = y[groups==i]
+            
+        return batch_data, batch_labels
+            
+    def lr_objective(self, theta, X, y, lmbda):
+        
+        n = y.shape[0]
+        y01 = (y+1)/2.0
+        
+        #compute the objective
+        mu = self.sigmoid(X.dot(theta))
+    
+        #bound away from 0 and 1
+        eps = np.finfo(float).eps   
+        mu = np.maximum(mu,eps)
+        mu = np.minimum(mu,1-eps)
+        
+        #compute cost
+        cost = -(1/n)*np.sum(y01*np.log(mu)+(1-y01)*np.log(1-mu))+np.sum(lmbda*theta*theta)
+        
+        #compute the gradient of the lr objective
+        grad = X.T.dot(mu-y01) + 2*lmbda*theta
+        
+        #compute the Hessian of the lr objective
+        #H = X.T.dot(np.diag(np.diag( mu*(1-mu) ))).dot(X) + 2*lmbda*np.eye(np.size(theta))
+        
+        return cost, grad
+                    
+    def sigmoid(self, a):
+        return 1/(1+np.exp(-a))
+    
+    def generate_plots(self, X, J_hist, t_hist, theta):
+                        
+        plt.figure()
+        plt.plot(J_hist)
+        plt.title("logistic regression")
+        plt.xlabel('iterations')
+        plt.ylabel('cost')
+        #plt.savefig('./figures/lrsgd_loss.png')
+        plt.show()
+        
+        plt.figure()
+        plt.plot(t_hist)
+        plt.title("logistic regression")
+        plt.xlabel('iterations')
+        plt.ylabel('theta l2 norm')
+        #plt.savefig('./figures/lrsgd_theta_norm.png')
+        plt.show()   
+        
+        plt.figure()
+        plt.plot(self.eta)
+        plt.title("logistic regression")
+        plt.xlabel('iterations')
+        plt.ylabel('learning rate')
+        #plt.savefig('./figures/lrsgd_learning_rate.png')
+        plt.show()
+        
+        plt.figure()
+        x1 = np.linspace(np.min(X[:,0])-1,np.max(X[:,0])+1,10)
+        plt.scatter(X[:,0], X[:,1])
+        plt.plot(x1, -(theta[0]/theta[1])*x1)
+        plt.title('logistic regression')
+        plt.grid(True)
+        plt.xlabel('X1')
+        plt.ylabel('X2')
+        #plt.savefig('./figures/lrsgd_clf.png')
+        plt.show()
+                    
+if __name__ == "__main__":
+        
+    X, y = generate_data()
+    sgd = sgdlr()
+    theta = sgd.fit(X,y)
+```
+
+![image](https://github.com/user-attachments/assets/b66ba84e-f646-40c1-864b-6a7d85a1b20e)
+![image](https://github.com/user-attachments/assets/787ac842-cb04-4515-b113-24ea8da05c27)
+![image](https://github.com/user-attachments/assets/19c06026-f78a-4f11-8d18-23b9bd8b2dbc)
+![image](https://github.com/user-attachments/assets/cc61f463-55af-4b7e-8ecf-126466c6d012)
+
+
+
+### The naive Bayes algorithm assumes features are conditionally independent, given the class label. It is commonly used in document classification.
+
+```python
+import numpy as np 
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from time import time
+from nltk.corpus import stopwords
+from nltk.tokenize import RegexpTokenizer
+
+from sklearn.metrics import accuracy_score
+from sklearn.datasets import fetch_20newsgroups
+from sklearn.model_selection import train_test_split
+from sklearn.feature_extraction.text import CountVectorizer
+
+sns.set_style("whitegrid")
+tokenizer = RegexpTokenizer(r'\w+')
+stop_words = set(stopwords.words('english'))
+stop_words.update(['s','t','m','1','2'])
+
+class naive_bayes:
+    def __init__(self, K, D):
+        self.K = K #number of classes
+        self.D = D #dictionary size
+
+        self.pi = np.ones(K) #class priors
+        self.theta = np.ones((self.D, self.K)) #bernoulli parameters
+
+    def fit(self, X_train, y_train):
+        
+        num_docs = X_train.shape[0]
+        for doc in range(num_docs):
+            
+            label = y_train[doc]
+            self.pi[label] += 1
+
+            for word in range(self.D):
+                if (X_train[doc][word] > 0):
+                    self.theta[word][label] += 1
+                #end if
+            #end for
+        #end for
+        
+        #normalize pi and theta
+        self.pi = self.pi/np.sum(self.pi)
+        self.theta = self.theta/np.sum(self.theta, axis=0)
+
+    def predict(self, X_test):
+
+        num_docs = X_test.shape[0]
+        logp = np.zeros((num_docs,self.K))
+        for doc in range(num_docs):
+            for kk in range(self.K):
+                logp[doc][kk] = np.log(self.pi[kk])
+                for word in range(self.D):
+                    if (X_test[doc][word] > 0):
+                        logp[doc][kk] += np.log(self.theta[word][kk])
+                    else:
+                        logp[doc][kk] += np.log(1-self.theta[word][kk])
+                    #end if
+                #end for
+            #end for
+        #end for
+        return np.argmax(logp, axis=1)
+        
+if __name__ == "__main__":
+
+    import nltk
+    nltk.download('stopwords')
+
+    #load data
+    print("loading 20 newsgroups dataset...")
+    tic = time()
+    classes = ['sci.space', 'comp.graphics', 'rec.autos', 'rec.sport.hockey']
+    dataset = fetch_20newsgroups(shuffle=True, random_state=0, remove=('headers','footers','quotes'), categories=classes)
+    X_train, X_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.5, random_state=0)
+    toc = time()
+    print("elapsed time: %.4f sec" %(toc - tic))
+    print("number of training docs: ", len(X_train))
+    print("number of test docs: ", len(X_test))
+
+    print("vectorizing input data...")
+    cnt_vec = CountVectorizer(tokenizer=tokenizer.tokenize, analyzer='word', ngram_range=(1,1), max_df=0.8, min_df=2, max_features=1000, stop_words=stop_words)
+    cnt_vec.fit(X_train)
+    toc = time()
+    print("elapsed time: %.2f sec" %(toc - tic))
+    vocab = cnt_vec.vocabulary_
+    idx2word = {val: key for (key, val) in vocab.items()}
+    print("vocab size: ", len(vocab))
+
+    X_train_vec = cnt_vec.transform(X_train).toarray()
+    X_test_vec = cnt_vec.transform(X_test).toarray()
+
+    print("naive bayes model MLE inference...")
+    K = len(set(y_train)) #number of classes
+    D = len(vocab) #dictionary size
+    nb_clf = naive_bayes(K, D)
+    nb_clf.fit(X_train_vec, y_train)
+
+    print("naive bayes prediction...")
+    y_pred = nb_clf.predict(X_test_vec)
+    nb_clf_acc = accuracy_score(y_test, y_pred)
+    print("test set accuracy: ", nb_clf_acc) 
+```
+
+Test acc: ~82%
+
+### The CART decision tree is a greedy, recursive algorithm that finds the optimum feature splits by minimizing an objective function, such as the Gini index.
+
+```python
+import numpy as np 
+import matplotlib.pyplot as plt 
+
+from sklearn.datasets import load_iris
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import train_test_split
+
+class TreeNode():
+    def __init__(self, gini, num_samples, num_samples_class, class_label):
+        self.gini = gini  #gini cost
+        self.num_samples = num_samples #size of node
+        self.num_samples_class = num_samples_class #number of node pts with label k
+        self.class_label = class_label #predicted class label
+        self.feature_idx = 0 #idx of feature to split on
+        self.treshold = 0  #best threshold to split on
+        self.left = None #left subtree pointer
+        self.right = None #right subtree pointer
+
+class DecisionTreeClassifier():
+    def __init__(self, max_depth = None):
+        self.max_depth = max_depth
+    
+    def best_split(self, X_train, y_train):
+        m = y_train.size
+        if (m <= 1):
+            return None, None
+        
+        #number of points of class k
+        mk = [np.sum(y_train == k) for k in range(self.num_classes)]
+
+        #gini of current node
+        best_gini = 1.0 - sum((n / m) ** 2 for n in mk)
+        best_idx, best_thr = None, None
+
+        #iterate over all features
+        for idx in range(self.num_features):
+            # sort data along selected feature
+            thresholds, classes = zip(*sorted(zip(X[:, idx], y)))
+
+            num_left = [0]*self.num_classes
+            num_right = mk.copy()
+
+            #iterate overall possible split positions
+            for i in range(1, m):
+                
+                k = classes[i-1]
+                
+                num_left[k] += 1
+                num_right[k] -= 1
+
+                gini_left = 1.0 - sum(
+                    (num_left[x] / i) ** 2 for x in range(self.num_classes)
+                )
+
+                gini_right = 1.0 - sum(
+                    (num_right[x] / (m - i)) ** 2 for x in range(self.num_classes)
+                )
+
+                gini = (i * gini_left + (m - i) * gini_right) / m
+
+                # check that we don't try to split two pts with identical values
+                if thresholds[i] == thresholds[i - 1]:
+                    continue
+
+                if (gini < best_gini):
+                    best_gini = gini
+                    best_idx = idx
+                    best_thr = (thresholds[i] + thresholds[i - 1]) / 2  # midpoint
+                #end if
+            #end for
+        #end for
+        return best_idx, best_thr
+    
+    def gini(self, y_train):
+        m = y_train.size
+        return 1.0 - sum((np.sum(y_train == k) / m) ** 2 for k in range(self.num_classes))
+
+    def fit(self, X_train, y_train):
+        self.num_classes = len(set(y_train))
+        self.num_features = X_train.shape[1]
+        self.tree = self.grow_tree(X_train, y_train)
+
+    def grow_tree(self, X_train, y_train, depth=0):
+        
+        num_samples_class = [np.sum(y_train == k) for k in range(self.num_classes)]
+        class_label = np.argmax(num_samples_class)
+        
+        node = TreeNode(
+            gini=self.gini(y_train),
+            num_samples=y_train.size,
+            num_samples_class=num_samples_class,
+            class_label=class_label,
+        )
+
+        # split recursively until maximum depth is reached
+        if depth < self.max_depth:
+            idx, thr = self.best_split(X_train, y_train)
+            if idx is not None:
+                indices_left = X_train[:, idx] < thr
+                X_left, y_left = X_train[indices_left], y_train[indices_left]
+                X_right, y_right = X_train[~indices_left], y_train[~indices_left]
+                node.feature_index = idx
+                node.threshold = thr
+                node.left = self.grow_tree(X_left, y_left, depth + 1)
+                node.right = self.grow_tree(X_right, y_right, depth + 1)
+
+        return node
+
+    def predict(self, X_test):
+        return [self.predict_helper(x_test) for x_test in X_test]
+
+    def predict_helper(self, x_test):
+        node = self.tree
+        while node.left:
+            if x_test[node.feature_index] < node.threshold:
+                node = node.left
+            else:
+                node = node.right
+        return node.class_label
+
+
+if __name__ == "__main__":
+
+    #load data
+    iris = load_iris()
+    X = iris.data[:, [2,3]]
+    y = iris.target
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+          
+    print("decision tree classifier...")
+    tree_clf = DecisionTreeClassifier(max_depth = 3)
+    tree_clf.fit(X_train, y_train)
+
+    print("prediction...")
+    y_pred = tree_clf.predict(X_test)
+    
+    tree_clf_acc = accuracy_score(y_test, y_pred)
+    print("test set accuracy: ", tree_clf_acc) 
+```
+
+Test acc: ~80%
+
+---
+
+## Machine Learning Algorithms in Depth Chapter 6: Regression
+
+### Bayesian Linear Regression
+
+* In Bayesian linear regression defined by y(x) =wTx + ϵ, we assume the noise term is a zero-mean Gaussian random variable
+
+```python
+import math
+import numpy as np 
+import pandas as pd
+
+import matplotlib.pyplot as plt
+from sklearn.datasets import fetch_california_housing 
+
+class ridge_reg():
+
+    def __init__(self, n_iter=20, learning_rate=1e-3, lmbda=0.1):
+        self.n_iter = n_iter
+        self.learning_rate = learning_rate
+        self.lmbda = lmbda 
+    
+    def fit(self, X, y):
+        #insert const 1 for bias term
+        X = np.insert(X, 0, 1, axis=1)
+        
+        self.loss = []
+        self.w = np.random.rand(X.shape[1])
+
+        for i in range(self.n_iter):
+            y_pred = X.dot(self.w)
+            mse = np.mean(0.5*(y - y_pred)**2 + 0.5*self.lmbda*self.w.T.dot(self.w))
+            self.loss.append(mse)
+            print(" %d iter, mse: %.4f" %(i, mse))
+            #compute gradient of NLL(w) wrt w
+            grad_w = - (y - y_pred).dot(X) + self.lmbda*self.w
+            #update the weights
+            self.w -= self.learning_rate * grad_w
+
+    def predict(self, X):
+        #insert const 1 for bias term
+        X = np.insert(X, 0, 1, axis=1)
+        y_pred = X.dot(self.w)
+        return y_pred
+
+if __name__ == "__main__":
+    
+    X, y = fetch_california_housing(return_X_y=True)
+    X_reg = X[:,2].reshape(-1,1) #avg number of rooms 
+    X_std = (X_reg - X_reg.mean())/X.std() #standard scaling
+    y_std = (y - y.mean())/y.std() #standard scaling
+
+    X_std = X_std[:200,:]
+    y_std = y_std[:200]
+
+    rr = ridge_reg()
+    rr.fit(X_std, y_std)
+    y_pred = rr.predict(X_std)
+
+    print(rr.w)
+
+    plt.figure()
+    plt.plot(rr.loss)
+    plt.xlabel('Epoch')
+    plt.ylabel('Loss')
+    plt.tight_layout()
+    plt.show()
+
+    plt.figure()
+    plt.scatter(X_std, y_std)
+    plt.plot(np.linspace(-1,1), rr.w[1]*np.linspace(-1,1)+rr.w[0], c='red')
+    plt.xlim([-0.01,0.01])
+    plt.xlabel("scaled avg num of rooms")
+    plt.ylabel("scaled house price")
+    plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/24a93906-c9c7-41d1-9d1a-95aaaeea21f3)
+![image](https://github.com/user-attachments/assets/49cf2d7c-6195-4921-b188-b04ab6e40f72)
+
+* The Bayesian linear regression successfully captures the trend of increasing house prices with the average number of rooms
+
+### Hierarchical Bayesian regression
+
+* Hierarchical models enable the sharing of features among groups. A hierarchical model assumes there is a common distribution from which individual parameters are sampled and, therefore, captures similarities between groups
+
+```python
+import numpy as np
+import pandas as pd
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+import pymc3 as pm 
+
+def main():
+    
+    #load data
+    data = pd.read_csv('./data/radon.txt')
+    
+    county_names = data.county.unique()
+    county_idx = data['county_code'].values
+    
+    with pm.Model() as hierarchical_model:
+        
+        # Hyperpriors
+        mu_a = pm.Normal('mu_alpha', mu=0., sd=100**2)
+        sigma_a = pm.Uniform('sigma_alpha', lower=0, upper=100)
+        mu_b = pm.Normal('mu_beta', mu=0., sd=100**2)
+        sigma_b = pm.Uniform('sigma_beta', lower=0, upper=100)
+    
+        # Intercept for each county, distributed around group mean mu_a
+        a = pm.Normal('alpha', mu=mu_a, sd=sigma_a, shape=len(data.county.unique()))
+        # Slope for each county, distributed around group mean mu_b
+        b = pm.Normal('beta', mu=mu_b, sd=sigma_b, shape=len(data.county.unique()))
+    
+        # Model error
+        eps = pm.Uniform('eps', lower=0, upper=100)
+    
+        # Expected value
+        radon_est = a[county_idx] + b[county_idx] * data.floor.values
+    
+        # Data likelihood
+        y_like = pm.Normal('y_like', mu=radon_est, sd=eps, observed=data.log_radon)
+        
+    
+    with hierarchical_model:
+        # Use ADVI for initialization
+        mu, sds, elbo = pm.variational.advi(n=100000)
+        step = pm.NUTS(scaling=hierarchical_model.dict_to_array(sds)**2, is_cov=True)
+        hierarchical_trace = pm.sample(5000, step, start=mu)
+
+                
+    pm.traceplot(hierarchical_trace[500:])
+    plt.show()        
+        
+if __name__ == "__main__":    
+    main()
+```
+
+### KNN regression
+
+* K nearest neighbors (KNNs) regression is a nonparametric model, in which for a given query data point q, we find its KNN in the training set and compute the average response variable y
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn import datasets
+from sklearn.model_selection import train_test_split
+
+np.random.seed(42)
+
+class KNN():
+    
+    def __init__(self, K):
+        self.K = K
+
+    def euclidean_distance(self, x1, x2):
+        dist = 0
+        for i in range(len(x1)):
+            dist += np.power((x1[i] - x2[i]), 2)
+        return np.sqrt(dist)
+    
+    def knn_search(self, X_train, y_train, Q):
+        y_pred = np.empty(Q.shape[0])        
+
+        for i, query in enumerate(Q):
+            #get K nearest neighbors to query point
+            idx = np.argsort([self.euclidean_distance(query, x) for x in X_train])[:self.K]            
+            #extract the labels of KNN training labels
+            knn_labels = np.array([y_train[i] for i in idx])
+            #label query sample as the average of knn_labels
+            y_pred[i] = np.mean(knn_labels)
+
+        return y_pred 
+        
+
+if __name__ == "__main__":
+    
+    plt.close('all')
+        
+    #iris dataset
+    iris = datasets.load_iris()
+    X = iris.data[:,:2]
+    y = iris.target            
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    K = 4
+    knn = KNN(K)    
+    y_pred = knn.knn_search(X_train, y_train, X_test)
+    
+    plt.figure(1)
+    plt.scatter(X_train[:,0], X_train[:,1], s = 100, marker = 'x', color = 'r', label = 'data')
+    plt.scatter(X_test[:,0], X_test[:,1], s = 100, marker = 'o', color = 'b', label = 'query')
+    plt.title('K Nearest Neighbors (K=%d)'% K)
+    plt.legend()
+    plt.xlabel('X1')
+    plt.ylabel('X2')
+    plt.grid(True)
+    plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/75a3b898-89df-4ce8-ae38-bb7f80943332)
+
+Finding the exact nearest neighbors in high-dimensional space is often computationally intractable, and therefore, there exist approximate methods. There are two classes of approximate methods: those that partition the space into regions, such as k-d tree implemented in fast library for approximate nearest neighbors (FLANN), and hashing-based methods, such as locality-sensitive hashing (LSH)
+
+### Gaussian process regression
+
+* Gaussian processes (GPs) define a prior over functions that can be updated to a posterior once we have observed data. A GP assumes the function is defined at a finite and arbitrary chosen set of points x1, ..., xn, such that p(f(x1), ..., f(xn)) is jointly Gaussian with a mean of μ(x) and covariance of Σ(x), where Σij = κ(xi, xj) and κ is a positive definite kernel function.
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.spatial.distance import cdist
+
+np.random.seed(42)
+
+class GPreg:
+    
+    def __init__(self, X_train, y_train, X_test):        
+
+        self.L = 1.0
+        self.keps = 1e-8
+        
+        self.muFn = self.mean_func(X_test)
+        self.Kfn = self.kernel_func(X_test, X_test) + 1e-15*np.eye(np.size(X_test))
+        
+        self.X_train = X_train
+        self.y_train = y_train
+        self.X_test = X_test
+            
+    def mean_func(self, x):
+        muFn = np.zeros(len(x)).reshape(-1,1)
+        return muFn
+        
+    def kernel_func(self, x, z):                
+        sq_dist = cdist(x/self.L, z/self.L, 'euclidean')**2
+        Kfn = 1.0 * np.exp(-sq_dist/2)
+        return Kfn
+    
+    def compute_posterior(self):
+        K = self.kernel_func(self.X_train, self.X_train)  #K
+        Ks = self.kernel_func(self.X_train, self.X_test)  #K_*
+        Kss = self.kernel_func(self.X_test, self.X_test) + self.keps*np.eye(np.size(self.X_test))  #K_**
+        Ki = np.linalg.inv(K) #O(Ntrain^3)        
+        
+        postMu = self.mean_func(self.X_test) + np.dot(np.transpose(Ks), np.dot(Ki, (self.y_train - self.mean_func(self.X_train))))
+        postCov = Kss - np.dot(np.transpose(Ks), np.dot(Ki, Ks))
+        
+        self.muFn = postMu
+        self.Kfn = postCov            
+
+        return None    
+    
+    def generate_plots(self, X, num_samples=3):
+        plt.figure()
+        for i in range(num_samples):
+            fs = self.gauss_sample(1)
+            plt.plot(X, fs, '-k')
+            #plt.plot(self.X_train, self.y_train, 'xk')
+                
+        mu = self.muFn.ravel()
+        S2 = np.diag(self.Kfn)
+        plt.fill(np.concatenate([X, X[::-1]]), np.concatenate([mu - 2*np.sqrt(S2), (mu + 2*np.sqrt(S2))[::-1]]), alpha=0.2, fc='b')
+        plt.show()    
+        
+    def gauss_sample(self, n):
+        # returns n samples from a multivariate Gaussian distribution
+        # S = AZ + mu        
+        A = np.linalg.cholesky(self.Kfn)
+        Z = np.random.normal(loc=0, scale=1, size=(len(self.muFn),n))
+        S = np.dot(A,Z) + self.muFn        
+        return S    
+        
+def main():
+    
+    # generate noise-less training data
+    X_train = np.array([-4, -3, -2, -1, 1])
+    X_train = X_train.reshape(-1,1)
+    y_train = np.sin(X_train)
+
+    # generate  test data
+    X_test = np.linspace(-5, 5, 50)
+    X_test = X_test.reshape(-1,1)
+    
+    gp = GPreg(X_train, y_train, X_test)
+    gp.generate_plots(X_test,3)  #samples from GP prior
+    gp.compute_posterior()
+    gp.generate_plots(X_test,3)  #samples from GP posterior
+    
+        
+if __name__ == "__main__":    
+    main()
+```
+
+![image](https://github.com/user-attachments/assets/1b00c7b4-b318-4d5d-acc8-f08e2f822ffc)
+![image](https://github.com/user-attachments/assets/0f1feb78-66b1-46da-867e-09770958b5ac)
+
+The above shows three functions drawn at random from a GP prior (first) and GP posterior (second) after observing five data points in the case of noise-free observations. The shaded area corresponds to two times the standard deviation around the mean (95% confidence region). We can see that the model perfectly interpolates the training data and that the predictive uncertainty increases as we move further away from the observed data.
+
+## Not Google Devs Society
+
+Yep. I will try to start my own society relateted to anything data. 
+
+### The name?
+
+I know there are official [Google Developer Student Clubs](https://www.google.com/search?q=google+developer+society+uni&rlz=1C5CHFA_enKR1063KR1063&oq=google&gs_lcrp=EgZjaHJvbWUqCAgBEEUYJxg7Mg4IABBFGCcYOxiABBiKBTIICAEQRRgnGDsyBggCEEUYOzIVCAMQLhhDGMcBGMkDGNEDGIAEGIoFMgYIBBBFGDwyBggFEEUYPDIGCAYQRRg8MgYIBxAFGEDSAQgxMzY4ajBqN6gCALACAA&sourceid=chrome&ie=UTF-8) however the deadline for this year has passed and applications will open for the 2025-26 academic year. And by that time I will have graduated. 
+
+So the names describes the sociecty members exactly as - *Not* Google Devs. I even 'designed' a logo. 
+
+![image](https://github.com/user-attachments/assets/61959252-6197-4831-ab7d-3990c1b11174)
+
+And also created flyers that I will put around the uni. Below is the translated content.
+
+#### 2024-2nd semester looking for:
+‭
+- Vice President‬
+- Event planner‬
+- General members
+‭
+#### What?
+
+‭- We are the Not Google Devs society‬
+- Learn and Practice Data Engineering, Data, Science, AI, Cloud, other‬
+- Attend bi-weekly sessions‬
+- Work on projects alone and with others‬
+‭
+#### Who is this for:
+- someone that loves new challenges
+- someone that loves to code
+- someone that can speak english or is studying english
+
+#### Currently planned schedule
+- MLOps
+- Data engineering
+- What is HuggingFace
+- RAG apps
+- Hackathon
+- Collab with Uni of Essex's CS society
+
+Then some words from myself and contact info. 
+
+### Why do I want to make a society?
+
+Being part of a society is one of the most memorable experiences of my undergraduate years. I had fun, but also got to learn soft skills that later I could add to my CV. With this society, I not only want to share knowledge of whatever I know from my work and learning experience, but also create an atmosphere of collaboration and learning. I hope that whoever joins the society can learn a lot, create project, work in a team, and later on add these experiences to their CV - just like I did. As for the meetings, I think online/in-person would be fine (depending on the amount of people). The idea is to have presentation sessions where a member or myself present a project, a new tool, something interesting that people want to or might be benefitial. 
+
+I am also looking for help - a VP and an Event Planner, so hopefully I get some volunteers ^^
+
+I also made a [LinkedIn page](https://www.linkedin.com/company/not-google-devs) for the society so that people can even add it on linkedin. I need to print out the promotion material and hang it around the school in the coming days. 
+
+For completeness, here is the poaster I have at the moment:
+
+<img width="533" alt="image" src="https://github.com/user-attachments/assets/53cbfc87-4fc1-4328-956e-27e17aff7bfc">
+
+That is all for today!
+
+See you tomorrow :)

From ebbc2d233fcacf7757f03be1435008fb72247dca Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 16 Aug 2024 20:52:13 +0900
Subject: [PATCH 384/638] Update 2024-08-16-day-228.md

---
 _posts/2024-08-16-day-228.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-16-day-228.md b/_posts/2024-08-16-day-228.md
index 81f6efa00..a57aab31e 100644
--- a/_posts/2024-08-16-day-228.md
+++ b/_posts/2024-08-16-day-228.md
@@ -1061,11 +1061,11 @@ Being part of a society is one of the most memorable experiences of my undergrad
 
 I am also looking for help - a VP and an Event Planner, so hopefully I get some volunteers ^^
 
-I also made a [LinkedIn page](https://www.linkedin.com/company/not-google-devs) for the society so that people can even add it on linkedin. I need to print out the promotion material and hang it around the school in the coming days. 
+I also made a [LinkedIn page](https://www.linkedin.com/company/not-google-devs) for the society so that people can even add it on linkedin. I need to print out the promotion material and hang it around the school in the coming days. I also created a [GitHub repo](https://github.com/divakaivan/not-google-devs) that we can potential use. For now it just has the promotional info.
 
 For completeness, here is the poaster I have at the moment:
 
-<img width="533" alt="image" src="https://github.com/user-attachments/assets/53cbfc87-4fc1-4328-956e-27e17aff7bfc">
+<img width="535" alt="image" src="https://github.com/user-attachments/assets/1fdecb38-3467-4218-a753-d3f7bbdd1798">
 
 That is all for today!
 

From 57557fc25e281a8bc0a108d5cf5d170403afbaca Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 17 Aug 2024 12:43:19 +0900
Subject: [PATCH 385/638] update day name

---
 _posts/2024-08-16-day-228.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-08-16-day-228.md b/_posts/2024-08-16-day-228.md
index a57aab31e..c2dc3014a 100644
--- a/_posts/2024-08-16-day-228.md
+++ b/_posts/2024-08-16-day-228.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 227) Making a poaster for the Not Google Devs Society
+title: (Day 228) Making a poaster for the Not Google Devs Society
 categories: [statistics,theory,applying-knowledge]
 ---
 

From 49120a2f37d8fe629cb05b425e0fc746309e5d98 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 17 Aug 2024 14:04:17 +0900
Subject: [PATCH 386/638] fix poaster - poster

---
 _posts/2024-08-16-day-228.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-08-16-day-228.md b/_posts/2024-08-16-day-228.md
index c2dc3014a..067ebc9e8 100644
--- a/_posts/2024-08-16-day-228.md
+++ b/_posts/2024-08-16-day-228.md
@@ -1,6 +1,6 @@
 ---
 layout: post
-title: (Day 228) Making a poaster for the Not Google Devs Society
+title: (Day 228) Making a poster for the Not Google Devs Society
 categories: [statistics,theory,applying-knowledge]
 ---
 
@@ -1063,7 +1063,7 @@ I am also looking for help - a VP and an Event Planner, so hopefully I get some
 
 I also made a [LinkedIn page](https://www.linkedin.com/company/not-google-devs) for the society so that people can even add it on linkedin. I need to print out the promotion material and hang it around the school in the coming days. I also created a [GitHub repo](https://github.com/divakaivan/not-google-devs) that we can potential use. For now it just has the promotional info.
 
-For completeness, here is the poaster I have at the moment:
+For completeness, here is the poster I have at the moment:
 
 <img width="535" alt="image" src="https://github.com/user-attachments/assets/1fdecb38-3467-4218-a753-d3f7bbdd1798">
 

From d6028d510839403189d1a6fdddcd7922d24e030f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 17 Aug 2024 22:25:33 +0900
Subject: [PATCH 387/638] Create 2024-08-17-day-229.md

---
 _posts/2024-08-17-day-229.md | 218 +++++++++++++++++++++++++++++++++++
 1 file changed, 218 insertions(+)
 create mode 100644 _posts/2024-08-17-day-229.md

diff --git a/_posts/2024-08-17-day-229.md b/_posts/2024-08-17-day-229.md
new file mode 100644
index 000000000..17ca25fc5
--- /dev/null
+++ b/_posts/2024-08-17-day-229.md
@@ -0,0 +1,218 @@
+---
+layout: post
+title: (Day 229) 'probabl' - a gem of a youtube channel
+categories: [statistics,theory,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 229!
+A quick summary of today:
+* watched a few amazing videos from this [probabl](https://www.youtube.com/@probabl_ai/videos) channel
+
+I have been subscribed to this channel for a while. It is created by some of the creators and devs of scikit-learn, and today I decided to watch their content. The video titles always looked interesting and I finally gave it a shot. Tldr: the videos are awesome! Everything is explain so well and in a simple way! 
+
+## [GridSearch](https://www.youtube.com/watch?v=KdIcUDqMVpE&t=883s) vs [RandomSearch](https://youtu.be/1FMnKAcaVPk)
+
+Param search can get slow. For instance, if we search over two params: and for each we use 5 values - that would result in a total of 5x5 = 25 fits. We usually do cross-validation as well, so that makes the number (using cv=5) -> 25x5 = 125 fits. 
+
+* One idea to speed things up is to add `refit=False`. This is useful if we are just interested in finding params, as once the best params are found, the pipeline will not fit on them once more - so we are saving a fit here. 
+* Another idea is adding `njobs` for multiprocessing and more CPU utilisation so that multiple fits can happen at the same time as they do not depend on each other
+* A powerful option is to add caching. This is done by adding `memory="cache_name"` when we are initialising our scikit-learn pipeline. This may work better for GridSearch as in that case we have fixed values that we go over and a param's values can be cached and then reused. This will not work so well in RandomSearch as there... well it is random - the param values for each param of interest are sampled from a distribution and on every fit, the value is different so caching will not really give us any benefits. 
+
+To better utilise the cache if using RandomSearch we can have a compromise:
+
+```python
+param_distribution = {
+  "C": np.logspace(0.001, 2, 100), # random sampling
+  "n_components": [10,20,50,100] # we will hit the cache a few times with this
+}
+```
+
+<img width="1002" alt="image" src="https://github.com/user-attachments/assets/5574f85b-7ace-428c-a635-ea326ddd9055">
+
+Another thing when comparing the two: in a grid we are picking pre-determined values for a param, say 1, 3, and 6. But what if the best value is 1.5? The GridSearch will not be able to find that by itself unless more values are provided (but we have to be careful as we might run into grid explosion), however a RandomSearch might. 
+
+## Building pipelines using scikit-learn's API [[1](https://www.youtube.com/watch?v=7TZ7j4HSzmE), [2](https://youtu.be/lhMqqauXtW0)]
+
+I have used their API for pipelines but did not know it was so cool and powerful. Example:
+
+<img width="975" alt="image" src="https://github.com/user-attachments/assets/af6739b0-283a-43fe-8b5d-4ab94e40fff6">
+
+Creaded using:
+
+```python
+from sklearn.pipeline import Pipeline, FeatureUnion, make_pipeline, make_union
+from sklearn.preprocessing import OneHotEncoder, Binarizer
+from sklearn.impute import SimpleImputer
+from skrub import SelectCols
+from sklearn.ensemble import HistGradientBoostingClassifier
+
+feat_pipe = make_union(
+    make_pipeline(
+        SelectCols(["pclass"]),
+        OneHotEncoder(sparse_output=False)
+    ),
+    make_pipeline(
+        SelectCols(["sex"]),
+        OneHotEncoder(sparse_output=False)
+    ),
+    make_pipeline(
+        SelectCols(["age"]),
+        SimpleImputer(fill_value=19, strategy="constant"),
+        make_union(
+            Binarizer(threshold=18),
+            Binarizer(threshold=12),
+        )
+    ),
+    SelectCols(["fare", "age"])
+)
+
+pipe = make_pipeline(
+    feat_pipe,
+    HistGradientBoostingClassifier()
+)
+```
+
+## [Scaling datasets](https://youtu.be/atehB1lM1Uc)
+
+To show the different scalers a simple classification set was used
+
+Decision Boundary without scaling:
+
+<img width="715" alt="image" src="https://github.com/user-attachments/assets/39eba91d-22e9-4b53-bef1-2f0bdcc3cca0">
+
+Decision Boundary using the StandardScaler:
+
+<img width="875" alt="image" src="https://github.com/user-attachments/assets/809f8222-74f4-4f9a-9430-9ef9f8813a77">
+
+MinMax Scaler:
+
+<img width="821" alt="image" src="https://github.com/user-attachments/assets/95aa2597-30ce-4e60-b0f8-374b0bbe3679">
+
+Quantile Scaler:
+
+<img width="804" alt="image" src="https://github.com/user-attachments/assets/bc97b29b-539f-407a-afd3-35f02b038bb8">
+
+## Boosting [[1](https://www.youtube.com/watch?v=o6seqpMJSTI),[2](https://www.youtube.com/watch?v=H-Nnk3IFBmY),[3](https://www.youtube.com/watch?v=5okmBJaE0kY&t=261s)]
+
+Gradient Boosting is an ensemble technique where each model (typically a decision tree) corrects the errors of the previous models. Each model uses the previous' errors to learn. Example:
+
+<img width="823" alt="image" src="https://github.com/user-attachments/assets/431fc669-9fe0-4c2a-a2c3-be982ef82c3e">
+
+In gradient boosting, the algorithm repeatedly adds trees to correct errors made by the previous trees.
+
+The process of finding the best split point in decision trees can be computationally expensive, especially with large datasets.
+
+[HistGradientBoostingRegressor](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.HistGradientBoostingRegressor.html) simplifies this process by grouping continuous features into discrete bins (histograms). This reduces the number of possible split points, making the split-finding process faster and more efficient.
+
+## Better boosting using the monotonic param
+
+<img width="701" alt="image" src="https://github.com/user-attachments/assets/5daf70e9-4fc7-4260-b45e-25f1f6c139f7">
+
+The chart shows the probability of being alive after 10 years and comparing smoker vs non-smokers. The picture is also the output of a boosted tree. We see something weird - these jumps. For example, after 70 there is a weird spike in probability, but that does not make sense. We expect the probability of being alive at 80 (regardless of smoking or not) to be down compared to 60, and 70. This is due to how the data is. 
+
+We can overcome this issue using the `monotonic_cst` param
+
+```python
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.pipeline import make_pipeline
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import OneHotEncoder, FunctionTransformer
+
+pipe = make_pipeline(
+    ColumnTransformer([
+        ('smoke', OneHotEncoder(), ['smoker']),
+        ('age', FunctionTransformer(), ['age'])
+    ]),
+    HistGradientBoostingClassifier(monotonic_cst=[0, 0, 1])
+)
+```
+
+The 3 numbers represent the 3 columns used for the model. Smoker results in 2 columns as they are OHE, and the 3rd column is age. The param makes sure that the 3rd column is monotonic. So when this setting is set the graph is:
+
+<img width="700" alt="image" src="https://github.com/user-attachments/assets/f99044ba-38f5-477b-99cb-63ae402535e7">
+
+And now the graph represent the logical thing - probability of being alive after 10 years goes down with age, regardless of being smoker or not. 
+
+## [Benchmarking boosted trees against overfitting](https://youtu.be/oZm4nGN8YMg)
+
+This was about how different params can influence a boosting model's performance. On the Y axis, the difference between train and test performance is show (lower is better), and iterations are shown on the X axis.
+
+We start with the below chart with 4000 data samples (from simulated data)
+
+<img width="643" alt="image" src="https://github.com/user-attachments/assets/d4127c07-e93b-4037-ac70-5ae9b300a8be">
+
+As we increase the samples to 10000, we see another model performing better:
+
+<img width="635" alt="image" src="https://github.com/user-attachments/assets/152dbfe3-4796-4c63-bcb7-a50f9eac34ff">
+
+And increasing to 50000:
+
+<img width="635" alt="image" src="https://github.com/user-attachments/assets/89cc8828-b54b-4d67-8021-640215e89631">
+
+So when we are creating a model, we need to be aware that parameters might work for a certain amount of data. Note! In this case the data is simulated so it comes from the same distribution, and in real life we might have something different and also face data drift. Nevertheless it cautions to be aware of the difference the amount of data can make on a model's performance.
+
+## [Features in Histogram Gradient Boosting Trees](https://scikit-learn.org/stable/auto_examples/ensemble/plot_hgbt_regression.html)
+
+The video instructor keeps saying that the examples in the scikit-learn website are very good, so I decided to check one related to the HGBT model. The instructor keeps using it in their examples so maybe it is a very good model. Apparently it is indeed.
+
+* HGBT is efficient, particularly on large datasets, due to its histogram-based approach to gradient boosting
+* it requires minimal hyperparameter tuning, making it accessible for users without deep expertise in machine learning
+* handling of Missing Data: The model can handle missing data natively, which simplifies preprocessing
+
+Taking about missing data ~ 
+
+<img width="1035" alt="image" src="https://github.com/user-attachments/assets/5b920839-139a-4164-b30f-f21f0194b97c">
+
+[This](https://youtu.be/FrS0TGsogyU) video talks about scikit-learn's native ability to handle missing values. Most of the library's algorithms can handle nan values natively and the instructor encourages us to leave them in and maybe our model can learn something extra from them
+
+## [PCA used for outlier detection ?](https://youtu.be/ZK3_T-UgKWM)
+
+<img width="1080" alt="image" src="https://github.com/user-attachments/assets/d4dd49f9-cc41-447d-91c3-3974aaaf3c4b">
+
+PCA's goal is to reduce the dimensionality of our data while keeping as much information as possible. It uses a transformation matrix to convert our high-dimensional data into a low-dimensional representation. But what if we use the inverse of that transformation matrix to go from the low-dimensional representation back to a high-dimensional representation again? Because we are trying to reduce dimensions, we might lose some information, so the original high-dimensional data and the reconstructed high-dimensional data might not be the same.
+
+But what we can take from that is: if a data point 'survived' the transformations, that means it is likely a normal point. If a point did 'not survive,' it can be thought of as an outlier.
+
+What does 'surviving' mean? If the difference between the original data point and the reconstructed data point is above some threshold, then the point could be an outlier. This is because PCA tries to preserve the most common information when doing its transformation, and if that data point's information was not able to be preserved (i.e., it had some different kind of information), then it might be an outlier.
+
+## Using a KernelDensityEstimator 
+
+<img width="1025" alt="image" src="https://github.com/user-attachments/assets/9e4ffd3d-045b-4318-bc9d-605557d3916d">
+
+We can use a histogram to try to represent our data distribution:
+
+<img width="649" alt="image" src="https://github.com/user-attachments/assets/9b6f5eaa-8cf9-49f1-87cb-8aa06d045e6e">
+
+But this representation may vary depending on the amount of bins we choose. Also it over- and undershoots in many places. This issue may be fixed with more data but in that case there is the curse of dimensionality - if we have many dims we won't have that much data per dimension to work with. Instead of using bins and a histogram, we can use a kernel. 
+
+We have a few different options:
+
+<img width="1063" alt="image" src="https://github.com/user-attachments/assets/92e0d82d-5729-4f0b-b847-e3600a1ddbf5">
+
+Where is this useful? 
+* outlier detection
+* fraud detection
+
+<img width="1043" alt="image" src="https://github.com/user-attachments/assets/944ef378-eca2-440d-af6d-bbb43f6af565">
+
+If we have a pipeline where a new data point comes in -> this point goes through a trained kernelEstimator to determine if it is an outlier or not, and depending on our design - if it is an outlier, it won't go through the model at all. 
+
+## [Using sample weights for better models](https://youtu.be/L2xY-c5Tyxg)
+
+<img width="617" alt="image" src="https://github.com/user-attachments/assets/30ab8487-81e6-45f5-939a-34913906bed8">
+
+The above is a dataset where we have fit a Ridge regression model. As we can see there is more density in the lower parts compared to the top curvy part and the fitted line reflects that. But what if for our use case we are particularly interested in predicting the top high values on the Y axis? Well we can add sample weights to our Ridge model.
+
+Each weight will be associated with a value and will affect how much the loss on that data point contributes to the overall loss. So a way we can add sample weights to the Ridge model is by using y values themselves as the weights (this is one of the options). This way the model will pay more attention to the high values and we can see this in the result:
+
+<img width="626" alt="image" src="https://github.com/user-attachments/assets/c13de8bf-5c10-4a18-8498-747aee897f86">
+
+And while the top values are fit better, we pay the price for some of the lower values. So it is not a free lunch ~ 😆
+
+---
+
+These are most of the videos on the channel. However there is a 'Live' section with ~1h livestream, so I need to go over these as well ^^ Thank you, Vincent (the instructor from probabl)
+
+That is all for today!
+
+See you tomorrow :) 

From 7b96a02aa53de54d51f0bb0d2659a4b4c91ba3c8 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 18 Aug 2024 20:23:32 +0900
Subject: [PATCH 388/638] Create 2024-08-18-day-230.md

---
 _posts/2024-08-18-day-230.md | 138 +++++++++++++++++++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100644 _posts/2024-08-18-day-230.md

diff --git a/_posts/2024-08-18-day-230.md b/_posts/2024-08-18-day-230.md
new file mode 100644
index 000000000..e9adaa772
--- /dev/null
+++ b/_posts/2024-08-18-day-230.md
@@ -0,0 +1,138 @@
+---
+layout: post
+title: (Day 230) Watching more educational videos from probabl
+categories: [traditional-machine-learning,applying-knowledge,theory]
+---
+
+# Hello :) Today is Day 230!
+A quick summary of today:
+* Protecting against temporal data leaks and skrub's TableVectorizer
+* Embeddings in scikit-learn pipelines
+* Little about timeseries
+* Sparse data pipelines
+
+## Protecting against temporal data leaks and skrub's TableVectorizer
+
+* [protecting against data leaks](https://www.youtube.com/watch?v=uevp7zJTM_c) - when creating time series models, it's crucial to ensure that the model learns patterns based only on past data and doesn't 'cheat' by accessing future information. However, it's easy to accidentally introduce data leakage, where the model unintentionally uses future data during training. To help data scientists avoid this mistake, Vincent (the probable instructor) has developed a library that safeguards against such errors by ensuring that the model only learns from past data
+
+* [TableVectorizer](https://www.youtube.com/watch?v=wn0g1wBzDfQ) for quick feature preprocessing - this comes from skrub (a library that seems to be maintained by some sklearn devs) and we can include it in a pipeline with some model, and then get a working pipeline. The TV automatically determines data types and what kind of processing it will do to them - ordinal encoding, OHE, scaling, etc. It is a good tool to get us started. As Vincent said - some of his sklearn colleagues get a new dataset, put it through a TableVectorizer and a HistGradientBoostingClassifier (or Regressor) and get a pretty good model that can serve as an initial checkpoint
+
+## Embeddings in scikit-learn pipelines
+
+Vincent has been maintaining an embedding library that is built upon scikit-learn classes so it directly integrates with other scikit-learn components - [embetter](https://github.com/koaning/embetter/)
+
+Taken from the github repo, with just a few lines of code we can get started with a text sentiment classifier:
+
+```python
+import pandas as pd
+from sklearn.pipeline import make_pipeline 
+from sklearn.linear_model import LogisticRegression
+
+from embetter.grab import ColumnGrabber
+from embetter.text import SentenceEncoder
+
+# This pipeline grabs the `text` column from a dataframe
+# which then get fed into Sentence-Transformers' all-MiniLM-L6-v2.
+text_emb_pipeline = make_pipeline(
+  ColumnGrabber("text"),
+  SentenceEncoder('all-MiniLM-L6-v2')
+)
+
+# This pipeline can also be trained to make predictions, using
+# the embedded features. 
+text_clf_pipeline = make_pipeline(
+  text_emb_pipeline,
+  LogisticRegression()
+)
+
+dataf = pd.DataFrame({
+  "text": ["positive sentiment", "super negative"],
+  "label_col": ["pos", "neg"]
+})
+X = text_emb_pipeline.fit_transform(dataf, dataf['label_col'])
+text_clf_pipeline.fit(dataf, dataf['label_col']).predict(dataf)
+```
+
+It is not here to replace pytorch or tensorflow as they are powerful libraries. Its purpose is for an easy setup and start up to get a prototype going.
+
+## [Little about timeseries](https://youtu.be/93tV-1ckqto)
+
+For periodic features in scikit-learn there is this SplineTransformer 
+
+```python
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import FunctionTransformer, SplineTransformer
+
+pipe = make_pipeline(
+    FunctionTransformer(datetime_feats),
+    SplineTransformer(n_knots=12, extrapolation="periodic")
+)
+```
+
+With 12 knots we get the below. The data is daily weather data.
+
+<img width="704" alt="image" src="https://github.com/user-attachments/assets/bcddc98d-3b82-467c-a74c-c78b060305cc">
+
+Then using this as a feature pipeline, and adding a Ridge on top we get:
+
+<img width="983" alt="image" src="https://github.com/user-attachments/assets/61f177ce-df7b-46cd-b52d-da0a8675017b">
+
+To get something with quantiles, we can use the `QuantileRegressor`:
+
+```python
+from sklearn.linear_model import QuantileRegressor
+
+feat_pipe = make_pipeline(
+    FunctionTransformer(datetime_feats),
+    SplineTransformer(n_knots=6, extrapolation="periodic")
+)
+
+mod_pipe_q_lower = make_pipeline(feat_pipe, QuantileRegressor(quantile=0.1, alpha=0.001))
+mod_pipe_q_upper = make_pipeline(feat_pipe, QuantileRegressor(quantile=0.9, alpha=0.001))
+
+mod_pipe_q_lower.fit(X, y)
+mod_pipe_q_upper.fit(X, y)
+```
+
+<img width="982" alt="image" src="https://github.com/user-attachments/assets/b8591e76-c4c1-4419-85fd-af65ced82f47">
+
+Finally, Vincent (the instructor) just talked about being careful when using lags in fear of data leakage. 
+
+## [Sparse data pipelines](https://youtu.be/u_cFkXFGj0c)
+
+#### Dense Matrices
+- matrices where most elements are non-zero
+- requires `O(m * n)` memory for an `m x n` matrix
+- optimized for general operations, but inefficient for large matrices with many zeros
+
+#### Sparse Matrices
+- matrices with most elements being zero
+- represented by `scipy.sparse` (e.g., CSR, CSC, COO formats)
+- only stores non-zero elements and their positions, leading to significant memory savings
+- efficient for operations involving non-zero elements; some operations may require conversion to dense format
+
+#### Use Cases
+- Dense: Use when the matrix is small or fully populated.
+- Sparse: Use for large matrices with many zeros (e.g., term-document matrices, graph algorithms).
+
+Example use case of sparse matrices is with text. And scikit-learn uses sparse matrices off the shelf
+
+```python
+from sklearn.feature_extraction.text import CountVectorizer
+from joblib import dump
+from pathlib import Path
+
+out = CountVectorizer().fit_transform(texts) # .todense()
+dump(out, 'tmp.pickle')
+Path('tmp.pickle').stat().st_size
+```
+
+The size we would get is: ~2.2m bytes. If we use a dense matrix (by adding `.todense()` to the code), we would get 1.3b bytes. So we can clearly see the efficiency of sparse matrices when it comes to memory if we have to save something to disk. 
+
+---
+
+The streams are 1h long each so it takes some time to watch. Nevertheless its quite interesting!
+
+That is all for today!
+
+See you tomorrow :)

From 683d1566f8f34a2db8c6571fa04c9fd5fc228408 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 18 Aug 2024 21:14:04 +0900
Subject: [PATCH 389/638] 230 -  watched 1 more video

---
 _posts/2024-08-18-day-230.md | 39 ++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/_posts/2024-08-18-day-230.md b/_posts/2024-08-18-day-230.md
index e9adaa772..fd189fd50 100644
--- a/_posts/2024-08-18-day-230.md
+++ b/_posts/2024-08-18-day-230.md
@@ -10,6 +10,7 @@ A quick summary of today:
 * Embeddings in scikit-learn pipelines
 * Little about timeseries
 * Sparse data pipelines
+* GMM for outlier detection
 
 ## Protecting against temporal data leaks and skrub's TableVectorizer
 
@@ -129,6 +130,44 @@ Path('tmp.pickle').stat().st_size
 
 The size we would get is: ~2.2m bytes. If we use a dense matrix (by adding `.todense()` to the code), we would get 1.3b bytes. So we can clearly see the efficiency of sparse matrices when it comes to memory if we have to save something to disk. 
 
+## [Expanding on GMMs](https://youtu.be/821aPPsFPsY)
+
+Yesterday I learned about using PCA for outlier detection, today Guassian Mixture Models.
+
+```python
+import numpy as np
+import matplotlib.pylab as plt
+
+from sklearn.datasets import make_moons
+from sklearn.preprocessing import StandardScaler
+
+from sklego.mixture import GMMOutlierDetector
+
+n = 1000
+X = make_moons(n)[0] + np.random.normal(n, 0.12, (n, 2))
+X = StandardScaler().fit_transform(X)
+U = np.random.uniform(-2, 2, (10000, 2))
+
+mod = GMMOutlierDetector(n_components=16, threshold=0.95).fit(X)
+
+plt.figure(figsize=(14, 5))
+plt.subplot(121)
+plt.scatter(X[:, 0], X[:, 1], c=mod.score_samples(X), s=8)
+plt.title("likelihood of points given mixture of 16 gaussians");
+
+plt.subplot(122)
+plt.scatter(U[:, 0], U[:, 1], c=mod.predict(U), s=8)
+plt.title("outlier selection");
+```
+
+![image](https://github.com/user-attachments/assets/ebc79739-69ae-46c4-855c-a1146a2e79b3)
+
+Depending on the threshold:
+
+![image](https://github.com/user-attachments/assets/71f5f455-cf9b-4f38-8bf7-f2d83857cd8c)
+
+In the scikit-lego library maintained by Vincent there is also a GMMClassifier 
+
 ---
 
 The streams are 1h long each so it takes some time to watch. Nevertheless its quite interesting!

From 35146a76cbd7c580607b28227ac6456c7b495976 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 19 Aug 2024 22:56:06 +0900
Subject: [PATCH 390/638] Create 2024-08-19-day-231.md

---
 _posts/2024-08-19-day-231.md | 831 +++++++++++++++++++++++++++++++++++
 1 file changed, 831 insertions(+)
 create mode 100644 _posts/2024-08-19-day-231.md

diff --git a/_posts/2024-08-19-day-231.md b/_posts/2024-08-19-day-231.md
new file mode 100644
index 000000000..da91fd557
--- /dev/null
+++ b/_posts/2024-08-19-day-231.md
@@ -0,0 +1,831 @@
+---
+layout: post
+title: (Day 231) Advancing to the finals of the 6th Kukmin Bank Future Finance AI competition!!!
+categories: [traditional-machine-learning,applying-knowledge,theory]
+---
+
+# Hello :) Today is Day 231!
+A quick summary of today:
+* the results from the KB competition are out
+* watching the latest probabl videos
+* reading chapter 7 from the Machine Learning Algorithms in Depth book
+
+## The results from the 6th Kukmin Bank Future Finance AI competition are out!
+
+And we are in the top 10 and advancing to the finals. [Here is the link](http://www.kbdatory.com/notice/view) to the announcement (it is all in Korean only)
+
+And the advancing teams are:
+
+<img width="541" alt="image" src="https://github.com/user-attachments/assets/67458625-8f17-4091-ba28-a9a88623db55">
+
+The order is alphabetical and my labmate and my project is the 10th one. They just show the 1st name from our team - which is my labmate's name. Here is a translted table:
+
+| NO | Team Leader   | Topic                                                               |
+|----|---------------|---------------------------------------------------------------------|
+| 1  | Kang Hee-won   | Overseas Financial Report Generation Platform REPOGEN.ai           |
+| 2  | Kim Bong-seok  | KB:OT                                                               |
+| 3  | Park Jae-jae   | SUPPLY LINK AI                                                     |
+| 4  | Son Hye-seon   | Fund Customer Asset Management through KFRI Development            |
+| 5  | Oh Seung-min   | Counselor Support Service Utilizing LLM (Large Language Model)     |
+| 6  | Lee Chang-mook | Voice Phishing Prevention AI through Counselor Voice Recognition   |
+| 7  | Jang Hye-young | I-Diary: Helping Retail Investors with Their Investments           |
+| 8  | Cho Myung-geun | KB-HI                                                              |
+| 9  | Choi Yong-woo  | AI Content Marketer 'Kard-Toon'                                    |
+| 10 | >>> Choi Jae-hyuk - this is us <<< | Fraud Detection through Real-time Graph DB Analysis                |
+
+The finals are on the 28th in Ehwa University in Seoul, so we have to prepare well!
+
+🥳🥳🥳 
+
+
+## [UMAP vs PCA](https://youtu.be/P07xnm2TOhY)
+
+Both PCA and UMAP aim to reduce the dimansionality of our data - go from X_wide to X_thin. But what is the difference?
+
+To show the different we will use the dataset in the pic. Yes, it looks like a swiss roll.
+
+<img width="470" alt="image" src="https://github.com/user-attachments/assets/a42d012a-36fe-4097-abbf-4ec7350582e1">
+
+We will reduce the dimensionality to 2 (from 3d), and see what is preserved. Before we continue, one of the main differences is that PCA keeps in tact the global space (and distances), while UMAP is more interested in preserving the local space (and distance).
+
+### PCA result
+
+<img width="618" alt="image" src="https://github.com/user-attachments/assets/af05b7f0-4789-4572-9447-fbd61d3ff266">
+
+We can see that the global structure (the Swiss roll) is preserved. However, if we examine this result closely, we may notice that for some points, searching for their nearest neighbors might not yield points of the same color. This is because, in some cases, points of a different color are closer to each other in the PCA projection. PCA looks at the overall variance and total information in the data. While it captures the major directions of variance, it does not prioritize local clusters. PCA does not specifically focus on preserving local neighborhood relationships; instead, it maximizes the global variance. 
+
+### UMAP result
+
+<img width="620" alt="image" src="https://github.com/user-attachments/assets/1ae26e01-6a65-4839-a85a-e2e6074550fa">
+
+The notion of global distance is less significant here, and we see that all the points that were close together locally remain close in this 2D space as well. UMAP is effective at preserving local structures and keeping clusters intact from the original data. It focuses on maintaining local neighborhood relationships, ensuring that points that were neighbors in the high-dimensional space stay close in the lower-dimensional projection.
+
+### PCA and UMAP on text
+
+Vincent (the instructor) also showed an example where text is embedded and then transformed to 2 dimensions using PCA and UMAP, and he shows the difference. I recommed watching the video as all the little things cannot be capture by pictures/text.
+
+#### UMAP
+
+<img width="430" alt="image" src="https://github.com/user-attachments/assets/5c372fbb-81d6-467b-b040-47573d1cef42">
+
+The text is embedded and using the search box we can find similar text. We can also see the difference in how the 'blob' looks like in UMAP vs PCA (PCA pic is below).
+
+When we search for 'call of duty' (the text dataset is related to Tesco), in UMAP that top-left cluster shines meaning that text most similar to it is there. 
+
+#### PCA
+
+This is the 'call of duty' cluster when the text dims are reduced to 2d using PCA. 
+
+<img width="391" alt="image" src="https://github.com/user-attachments/assets/728d969c-5c4e-48dd-959e-b1dbcc251921">
+
+We can clearly see that PCA looks a bit more 'structured' as in more of a circle, and also we just by examining the shapes of the two we cannot say where clusters exist. If we look at the UMAP picture again, we can see certain parts that look like individual islands where particular clusters may exist. In UMAP, clusters in the original data can remain in tact. 
+
+## [Mixture Density Networks](https://youtu.be/bPcI5bReUMQ)
+
+Traditional regression models, including linear regression or even neural networks, typically assume that the target variable y is a deterministic function of the input variable x, or at best, has a simple noise model like Gaussian noise. This means they often output a single value (or mean and variance for simple Gaussian noise), which is insufficient for capturing multimodal distributions. For example, if y could take multiple distinct values for a given x (e.g., the bimodal distribution of outcomes), a traditional neural network wouldn't capture this properly.
+
+<img width="994" alt="image" src="https://github.com/user-attachments/assets/703b6e83-34ba-4509-beaf-784e3f7fe2db">
+
+MDNs overcome this limitation by modeling the conditional probability distribution p(y|x) as a mixture of several distributions (typically Gaussians), where the parameters of these distributions are functions of the input x.
+
+Instead of outputting a single value, the network outputs the parameters of a mixture distribution:
+* Mixture coefficients: 𝜋 (sum to 1, representing the weight of each Gaussian)
+* Means for each Gaussian component
+* Variances for each Gaussian component
+
+Sample architecture:
+
+<img width="965" alt="image" src="https://github.com/user-attachments/assets/473ed23a-15ac-4361-9cea-4381d74eb998">
+
+And in code:
+
+```python
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+import matplotlib.pyplot as plt
+
+class MixtureDensityNetwork(nn.Module):
+    def __init__(self, input_dim, hidden_dim, output_dim, n_gaussians):
+        super(MixtureDensityNetwork, self).__init__()
+        self.input_dim = input_dim
+        self.hidden_dim = hidden_dim
+        self.output_dim = output_dim
+        self.n_gaussians = n_gaussians
+
+        self.hidden_layer = nn.Linear(input_dim, hidden_dim)
+        self.pi_layer = nn.Linear(hidden_dim, n_gaussians)
+        self.mu_layer = nn.Linear(hidden_dim, n_gaussians * output_dim)
+        self.sigma_layer = nn.Linear(hidden_dim, n_gaussians * output_dim)
+
+    def forward(self, x):
+        h = F.softplus(self.hidden_layer(x))
+        pi = F.softmax(self.pi_layer(h), dim=1)
+        mu = self.mu_layer(h).view(-1, self.n_gaussians, self.output_dim)
+        sigma = F.softplus(self.sigma_layer(h)).view(-1, self.n_gaussians, self.output_dim)
+
+        return pi, mu, sigma
+
+def mdn_loss(pi, mu, sigma, target):
+    normal = torch.distributions.Normal(mu, sigma)
+    log_prob = normal.log_prob(target.unsqueeze(1).expand_as(mu))
+    weighted_logprob = log_prob + torch.log(pi.unsqueeze(-1))
+    return -torch.logsumexp(weighted_logprob, dim=1).mean()
+```
+
+Using a toy dataset, we can see and output when using an MDN:
+
+<img width="912" alt="image" src="https://github.com/user-attachments/assets/f6acf39f-6ac5-4f48-a848-ba33b388d02b">
+
+We can also see what is happening internally in the MDN:
+
+<img width="905" alt="image" src="https://github.com/user-attachments/assets/cf4bc9d4-5ea9-4850-8f7a-41a5637879ca">
+
+In the [latest stream](https://youtu.be/K0VY-5GuMCQ), Vincent actually talks about a side project - [scikit-mdn](https://github.com/koaning/scikit-mdn) where he developed an mdn using pytorch and then used scikit components to allow scikit-learn type of user experience. In this livestream he was just talking about the implementation and how it works. 
+
+## Interview with Vincent
+
+I youtube searched his name and turns out that he did an interview with Alexey from DataTalksClub. [Link](https://youtu.be/UPlIETGwTg8)
+
+Apparently he is writing a book related to expectations vs reality in the data world, and I am looking forward to it. 
+
+From the interview, I found out about Vincent's platform called [calm code](https://calmcode.io/). It has many many videos about different tools to help the python dev process <- so I need to check this out as well.
+
+<img width="1247" alt="image" src="https://github.com/user-attachments/assets/50133023-ac60-4bb7-bbe6-8d16c74d3ed9">
+
+## Machine Learning Algorithms in Depth - Chapter 7: Selected supervised learning algorithms
+
+### Markov models: page rank and HMM
+
+* Markov models have the Markov property that conditioned on the present state—the future states are independent of the past. In other words, the present state serves as a sufficient statistic for the next state
+* Page rank is a stationary distribution of the Markov chain described by the transition matrix, which is recurrent and aperiodic. At scale, the page rank algorithm is computed using power iterations
+
+```python
+import numpy as np
+from numpy.linalg import norm
+
+np.random.seed(42)
+
+class page_rank():
+
+    def __init__(self):
+        self.max_iter = 100
+        self.tolerance = 1e-5
+    
+    def power_iteration(self, A):
+        n = np.shape(A)[0]
+        v = np.random.rand(n)
+        converged = False
+        iter = 0
+
+        while (not converged) and (iter < self.max_iter):
+            old_v = v
+            v = np.dot(A, v)
+            v = v / norm(v)
+            lambd = np.dot(v, np.dot(A, v))
+            converged = norm(v - old_v) < self.tolerance
+            iter += 1
+        #end while
+
+        return lambd, v
+    
+if __name__ == "__main__":
+
+    #construct a symmetric real matrix
+    X = np.random.rand(10,5)
+    A = np.dot(X.T, X)
+    
+    pr = page_rank()
+    lambd, v = pr.power_iteration(A)
+    
+    print(lambd)
+    print(v)
+
+    #compare against np.linalg implementation
+    eigval, eigvec = np.linalg.eig(A)
+    idx = np.argsort(np.abs(eigval))[::-1]
+    top_lambd = eigval[idx][0]
+    top_v = eigvec[:,idx][0]    
+
+    assert np.allclose(lambd, top_lambd, 1e-3)
+    assert np.allclose(v, top_v, 1e-3)
+```
+
+* Hidden Markov models model time series data and consist of a latent state Markov chain, a transition matrix between the latent states, and an emission matrix that models observed data emitted from each state. Inference is carried out using either the EM algorithm or the forward–backward algorithm with a Viterbi maximum likelihood sequence decoder
+
+```python
+import numpy as np
+from scipy.sparse import coo_matrix
+import matplotlib.pyplot as plt
+
+np.random.seed(42)
+
+class HMM():
+    def __init__(self, d=3, k=2, n=10000):
+        self.d = d #dimension of data
+        self.k = k #dimension of latent state
+        self.n = n #number of data points
+
+        self.A = np.zeros((k,k)) #transition matrix
+        self.E = np.zeros((k,d)) #emission matrix
+        self.s = np.zeros(k)     #initial state vector
+
+        self.x = np.zeros(self.n) #emitted observations
+
+    def normalize_mat(self, X, dim=1):
+        z = np.sum(X, axis=dim)
+        Xnorm = X/z.reshape(-1,1)
+        return Xnorm
+
+    def normalize_vec(self, v):
+        z = sum(v)
+        u = v / z
+        return u, z
+
+    def init_hmm(self):
+
+        #initialize matrices at random
+        self.A = self.normalize_mat(np.random.rand(self.k,self.k))
+        self.E = self.normalize_mat(np.random.rand(self.k,self.d))
+        self.s, _ = self.normalize_vec(np.random.rand(self.k))
+
+        #generate markov observations 
+        z = np.random.choice(self.k, size=1, p=self.s)
+        self.x[0] = np.random.choice(self.d, size=1, p=self.E[z,:].ravel())
+        for i in range(1, self.n):
+            z = np.random.choice(self.k, size=1, p=self.A[z,:].ravel())
+            self.x[i] = np.random.choice(self.d, size=1, p=self.E[z,:].ravel())
+        #end for
+
+    def forward_backward(self):
+        
+        #construct sparse matrix X of emission indicators
+        data = np.ones(self.n)
+        row = self.x
+        col = np.arange(self.n)
+        X = coo_matrix((data, (row, col)), shape=(self.d, self.n))
+
+        M = self.E * X
+        At = np.transpose(self.A)
+        c = np.zeros(self.n)  #normalization constants
+        alpha = np.zeros((self.k, self.n))  #alpha = p(z_t = j | x_{1:T})
+        alpha[:,0], c[0] = self.normalize_vec(self.s * M[:,0])
+        for t in range(1, self.n):
+            alpha[:,t], c[t] = self.normalize_vec(np.dot(At, alpha[:,t-1]) * M[:,t]) 
+        #end for
+        
+        beta = np.ones((self.k, self.n))
+        for t in range(self.n-2, 0, -1):
+            beta[:,t] = np.dot(self.A, beta[:,t+1] * M[:,t+1])/c[t+1]
+        #end for
+        gamma = alpha * beta
+
+        return gamma, alpha, beta, c
+
+    def viterbi(self):
+        
+        #construct sparse matrix X of emission indicators
+        data = np.ones(self.n)
+        row = self.x
+        col = np.arange(self.n)
+        X = coo_matrix((data, (row, col)), shape=(self.d, self.n))
+
+        #log scale for numerical stability
+        s = np.log(self.s)
+        A = np.log(self.A)
+        M = np.log(self.E * X)
+
+        Z = np.zeros((self.k, self.n))
+        Z[:,0] = np.arange(self.k)
+        v = s + M[:,0]
+        for t in range(1, self.n):
+            Av = A + v.reshape(-1,1)
+            v = np.max(Av, axis=0)
+            idx = np.argmax(Av, axis=0)
+            v = v.reshape(-1,1) + M[:,t].reshape(-1,1)
+            Z = Z[idx,:]
+            Z[:,t] = np.arange(self.k)
+        #end for
+        llh = np.max(v)
+        idx = np.argmax(v)
+        z = Z[idx,:]
+
+        return z, llh
+
+
+if __name__ == "__main__":
+
+    hmm = HMM()
+    hmm.init_hmm()
+
+    gamma, alpha, beta, c = hmm.forward_backward()
+    z, llh = hmm.viterbi()
+    import pdb; pdb.set_trace()
+```
+
+### Imbalanced Learning
+
+Most classification algorithms will only perform optimally when the number of samples in each class is roughly the same. Highly skewed datasets where the minority class is outnumbered by one or more classes commonly occur in fraud detection, medical diagnosis, and computational biology. One way of addressing this issue is by resampling the dataset to offset the imbalance and arrive at a more robust and accurate decision boundary. Resampling techniques can be broadly divided into four categories: undersampling the majority class, over-sampling the minority class, combining over and undersampling, and creating an ensemble of balanced datasets.
+
+1. Random Undersampling: This method randomly removes data points from the majority class to balance the dataset
+2. Cluster Centroids: This approach uses the K-means algorithm to replace a cluster of samples with the centroid of the cluster, with the number of clusters determined by the desired level of undersampling
+3. Tomek Links: This method removes overlapping examples between classes to clean up the decision boundary. A Tomek link is a pair of instances from different classes that are the closest to each other. By removing these pairs, the method helps in eliminating noise and refining class boundaries
+4. One-Sided Selection (OSS): OSS selects a subset of the majority class and combines it with all the minority class examples. The resulting set is then refined by removing any majority class Tomek links, leading to a more balanced and cleaner dataset for training
+
+<img width="827" alt="image" src="https://github.com/user-attachments/assets/74f81155-adab-443b-a741-470080bec5bd">
+
+Tomek Links algorithm:
+
+```python
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from sklearn.model_selection import train_test_split
+from sklearn.utils import shuffle
+from imblearn.under_sampling import TomekLinks
+
+rng = np.random.RandomState(42)
+
+def main():
+
+    #generate data    
+    n_samples_1 = 500
+    n_samples_2 = 50
+    X_syn = np.r_[1.5 * rng.randn(n_samples_1, 2), 0.5 * rng.randn(n_samples_2, 2) + [2, 2]]
+    y_syn = np.array([0] * (n_samples_1) + [1] * (n_samples_2))
+    X_syn, y_syn = shuffle(X_syn, y_syn)
+    X_syn_train, X_syn_test, y_syn_train, y_syn_test = train_test_split(X_syn, y_syn)
+
+    # remove Tomek links
+    tl = TomekLinks(sampling_strategy='auto')
+    X_resampled, y_resampled = tl.fit_resample(X_syn, y_syn)
+    idx_resampled = tl.sample_indices_
+    idx_samples_removed = np.setdiff1d(np.arange(X_syn.shape[0]),idx_resampled)
+    
+    #generate plots
+    fig = plt.figure()
+    ax = fig.add_subplot(1, 1, 1)
+
+    idx_class_0 = y_resampled == 0
+    plt.scatter(X_resampled[idx_class_0, 0], X_resampled[idx_class_0, 1], alpha=.8, marker = "o", label='Class #0')
+    plt.scatter(X_resampled[~idx_class_0, 0], X_resampled[~idx_class_0, 1], alpha=.8, marker = "s", label='Class #1')
+    plt.scatter(X_syn[idx_samples_removed, 0], X_syn[idx_samples_removed, 1], alpha=.8, marker = "v", label='Removed samples')
+    plt.title('Undersampling: Tomek links')
+    plt.legend()
+    plt.show()
+
+if __name__ == "__main__":
+    main()
+```
+
+<img width="756" alt="image" src="https://github.com/user-attachments/assets/e067e0c6-95cc-4b10-aa4f-73413f2b6afa">
+
+
+### Oversampling strategies
+
+1. Random Oversampling: This method adds data points to the minority class uniformly at random to balance the dataset
+2. Synthetic Minority Oversampling Technique (SMOTE): SMOTE generates synthetic examples by identifying K-nearest neighbors in the feature space and creating new data points along the line segments between the selected neighbors. The new data points are created by taking the difference between a sample and its nearest neighbor, multiplying it by a random number between 0 and 1, and adding this to the original sample
+3. Adaptive Synthetic Sampling (ADASYN)
+
+<img width="821" alt="image" src="https://github.com/user-attachments/assets/c3ff2d3a-eec3-4dd8-8f9c-48a63aaefbbd">
+
+SMOTE algorithm:
+
+```python
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from sklearn.datasets import make_classification
+from sklearn.decomposition import PCA
+
+from imblearn.over_sampling import SMOTE
+
+def plot_resampling(ax, X, y, title):
+    c0 = ax.scatter(X[y == 0, 0], X[y == 0, 1], label="Class #0", marker="o", alpha=0.5)
+    c1 = ax.scatter(X[y == 1, 0], X[y == 1, 1], label="Class #1", marker="s", alpha=0.5)
+    ax.set_title(title)
+    ax.spines['top'].set_visible(False)
+    ax.spines['right'].set_visible(False)
+    ax.get_xaxis().tick_bottom()
+    ax.get_yaxis().tick_left()
+    ax.spines['left'].set_position(('outward', 10))
+    ax.spines['bottom'].set_position(('outward', 10))
+    ax.set_xlim([-6, 8])
+    ax.set_ylim([-6, 6])
+
+    return c0, c1
+
+def main():
+    # generate the dataset
+    X, y = make_classification(n_classes=2, class_sep=2, weights=[0.3, 0.7],
+                               n_informative=3, n_redundant=1, flip_y=0,
+                               n_features=20, n_clusters_per_class=1,
+                               n_samples=80, random_state=10)
+
+    # fit PCA for visualization  
+    pca = PCA(n_components=2)
+    X_vis = pca.fit_transform(X)
+
+    # apply regular SMOTE
+    method = SMOTE()
+    X_res, y_res = method.fit_resample(X, y)
+    X_res_vis = pca.transform(X_res)
+
+    # generate plots
+    f, (ax1, ax2) = plt.subplots(1, 2)
+    c0, c1 = plot_resampling(ax1, X_vis, y, 'Original')
+    plot_resampling(ax2, X_res_vis, y_res, 'SMOTE')
+    ax1.legend((c0, c1), ('Class #0', 'Class #1'))
+    plt.tight_layout()
+    plt.show()
+    
+if __name__ == "__main__":
+    main()
+```
+
+<img width="731" alt="image" src="https://github.com/user-attachments/assets/d4e1c376-a9cd-4eda-8e5d-91efeefb075a">
+
+### Active Learning
+
+* Active Learning is a machine learning approach where the algorithm actively selects the most informative data samples to train on. This strategy is designed to maximize learning efficiency, allowing the model to achieve high performance with fewer labeled training examples. This is particularly useful in scenarios where obtaining labeled data is expensive or time-consuming. There are two primary query strategies in active learning:
+  * Uncertainty Sampling: The model selects data points for which it is least confident in its predictions. These are typically the samples near the decision boundary where the model's predictions have high uncertainty. By focusing on these uncertain samples, the model can learn more effectively and improve its performance more rapidly.
+  * Query by Committee (QBC): In this strategy, a committee of models with differing hypotheses is used. The models are trained on the same dataset but with different initializations or algorithms. The data points on which the committee members disagree the most are selected for labeling. This disagreement, or diversity in predictions, helps to identify samples that could provide the most learning benefit when labeled
+
+* The Bias–Variance Tradeoff is a fundamental concept in machine learning that deals with the tradeoff between two types of errors that a model can make:
+  * Bias: This is the error due to overly simplistic assumptions in the model. High bias can cause the model to miss relevant relations between features and target outputs, leading to underfitting
+  * Variance: This is the error due to the model's sensitivity to small fluctuations in the training data. High variance can cause the model to capture noise in the training data as if it were part of the true signal, leading to overfitting
+
+### Model selection: Hyperparameter tuning
+
+In model selection, we often want to follow the Occam’s razor principle and choose the simplest model that explains the data well. In hyperparameter optimization, in addition to grid search and random search, Bayesian Optimization uses an active learning approach in a way that reduces uncertainty and provides a balance between exploration and exploitation.
+
+####  Bayesian optimization
+
+Instead of randomly exploring the parameter space, Bayesian optimization adopts an active learning approach to select continuous parameter values that reduce uncertainty while balancing exploration and exploitation. This method leverages Gaussian Processes (GPs) to model the algorithm's generalization performance, providing a Bayesian framework for optimization.
+
+Bayesian optimization operates under the assumption that the performance function can be sampled from a GP, maintaining a posterior distribution as observations are made. The optimization process involves strategies like Expected Improvement (EI) and the Gaussian Process Upper Confidence Bound (UCB), which are effective in minimizing the number of evaluations required to find the global optimum of complex, multimodal functions.
+
+Unlike methods that rely on local gradient and Hessian approximations, Bayesian optimization utilizes all available information from previous evaluations. This approach is particularly useful for optimizing non-convex functions with relatively few evaluations, making it ideal for expensive processes like hyperparameter tuning in deep neural networks. The tradeoff is that more computation is required to determine the next point to evaluate, but this leads to a more efficient and automated optimization process.
+
+Algorithm:
+
+```python
+import numpy as np
+import pandas as pd
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from sklearn.datasets import make_classification
+from sklearn.model_selection import cross_val_score
+from sklearn.ensemble import RandomForestClassifier as RFC
+from sklearn.svm import SVC
+
+from bayes_opt import BayesianOptimization
+
+np.random.seed(42)
+
+# Load data set and target values
+data, target = make_classification(
+    n_samples=1000,
+    n_features=45,
+    n_informative=12,
+    n_redundant=7
+)
+target = target.ravel()
+
+def svccv(gamma):
+    val = cross_val_score(
+        SVC(gamma=gamma, random_state=0),
+        data, target, scoring='f1', cv=2
+    ).mean()
+
+    return val
+
+def rfccv(n_estimators, max_depth):
+    val = cross_val_score(
+        RFC(n_estimators=int(n_estimators),
+            max_depth=int(max_depth),
+            random_state=0
+        ),
+        data, target, scoring='f1', cv=2
+    ).mean()
+    return val
+
+if __name__ == "__main__":
+
+    gp_params = {"alpha": 1e-5}
+
+    #SVM
+    svcBO = BayesianOptimization(svccv,
+        {'gamma': (0.00001, 0.1)})
+
+    svcBO.maximize(init_points=3, n_iter=4, **gp_params)
+
+    #Random Forest
+    rfcBO = BayesianOptimization(
+        rfccv,
+        {'n_estimators': (10, 300),
+         'max_depth': (2, 10)
+        }
+    )
+    rfcBO.maximize(init_points=4, n_iter=4, **gp_params)
+
+    print('Final Results')
+    print(svcBO.max)
+    print(rfcBO.max)
+```
+
+<img width="831" alt="image" src="https://github.com/user-attachments/assets/0304d775-bbe1-4202-a087-0f872e1a166b">
+
+The F1 score was used as a performance objective function for a classification task. The figure on the left shows the Bayesian optimization of the F1 score as a function of the gamma parameter of the SVM RBF kernel \(K(x, x') = \exp{(-\gamma \|x - x'\|^2)}\), where gamma is the precision, equal to the inverse variance. We can see that after only seven iterations, we have discovered the gamma parameter that gives the maximum F1 score. The peak of the Expected Improvement (EI) utility function at the bottom indicates which experiment to perform next. 
+
+The figure on the right shows the Bayesian optimization of the F1 score as a function of maximum depth and the number of estimators of a Random Forest classifier. From the heatmap, we can observe that the maximum F1 score is achieved for 158 estimators with a depth equal to 10.
+
+### Ensemble methods
+
+Ensemble methods are meta-algorithms that combine several ML techniques into one predictive model to decrease variance (bagging), decrease bias (boosting), or improve predictions (stacking).
+
+Ensemble methods can be divided into two groups: sequential ensemble methods, where the base learners are generated sequentially (e.g., AdaBoost), and parallel ensemble methods, where the base learners are generated in parallel (e.g., random forest).
+
+To ensure ensemble methods are more accurate than any of their individual members, the base learners must be as accurate and diverse as possible.
+
+#### Bagging
+
+Bagging stands for bootstrap aggregation. One way to reduce the variance of an estimate is to average together multiple estimates. For example, we can train M different decision trees fm on different subsets of the data (chosen randomly with replacement) and compute the ensemble. Bagging uses bootstrap sampling to obtain the data subsets for training the base learners. For aggregating the outputs of base learners, bagging uses voting for classification and averaging for regression.
+
+```python
+import itertools
+import numpy as np
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+
+from sklearn import datasets
+
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier
+
+from sklearn.ensemble import BaggingClassifier
+from sklearn.model_selection import cross_val_score, train_test_split
+
+from mlxtend.plotting import plot_learning_curves
+from mlxtend.plotting import plot_decision_regions
+
+def main():
+
+    iris = datasets.load_iris()
+    X, y = iris.data[:, 0:2], iris.target
+    
+    clf1 = DecisionTreeClassifier(criterion='entropy', max_depth=None)
+    clf2 = KNeighborsClassifier(n_neighbors=1)    
+
+    bagging1 = BaggingClassifier(base_estimator=clf1, n_estimators=10, max_samples=0.8, max_features=0.8)
+    bagging2 = BaggingClassifier(base_estimator=clf2, n_estimators=10, max_samples=0.8, max_features=0.8)            
+
+    label = ['Decision Tree', 'K-NN', 'Bagging Tree', 'Bagging K-NN']
+    clf_list = [clf1, clf2, bagging1, bagging2]
+    
+    fig = plt.figure(figsize=(10, 8))
+    gs = gridspec.GridSpec(2, 2)
+    grid = itertools.product([0,1],repeat=2)
+
+    for clf, label, grd in zip(clf_list, label, grid):        
+        scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
+        print("Accuracy: %.2f (+/- %.2f) [%s]" %(scores.mean(), scores.std(), label))
+        
+        clf.fit(X, y)
+        ax = plt.subplot(gs[grd[0], grd[1]])
+        fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2)
+        plt.title(label)
+
+    plt.show()
+    #plt.savefig('./figures/bagging_ensemble.png')             
+
+    #plot learning curves
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
+    
+    plt.figure()
+    plot_learning_curves(X_train, y_train, X_test, y_test, bagging1, print_model=False, style='ggplot')
+    plt.show()
+    #plt.savefig('./figures/bagging_ensemble_learning_curve.png')             
+
+    #Ensemble Size
+    num_est = list(map(int, np.linspace(1,100,20)))
+    bg_clf_cv_mean = []
+    bg_clf_cv_std = []
+    for n_est in num_est:
+        print("num_est: ", n_est)
+        bg_clf = BaggingClassifier(base_estimator=clf1, n_estimators=n_est, max_samples=0.8, max_features=0.8)
+        scores = cross_val_score(bg_clf, X, y, cv=3, scoring='accuracy')
+        bg_clf_cv_mean.append(scores.mean())
+        bg_clf_cv_std.append(scores.std())
+    
+    plt.figure()
+    (_, caps, _) = plt.errorbar(num_est, bg_clf_cv_mean, yerr=bg_clf_cv_std, c='blue', fmt='-o', capsize=5)
+    for cap in caps:
+        cap.set_markeredgewidth(1)                                                                                                                                
+    plt.ylabel('Accuracy'); plt.xlabel('Ensemble Size'); plt.title('Bagging Tree Ensemble');
+    plt.show()
+    #plt.savefig('./figures/bagging_ensemble_size.png')
+
+if __name__ == "__main__":    
+    main()
+```
+
+<img width="820" alt="image" src="https://github.com/user-attachments/assets/e310d0b0-2231-4626-89de-bc0fc51678d7">
+
+The figure shows the decision boundary of a decision tree and KNN classifiers, along with their bagging ensembles applied to the iris dataset. The decision tree shows axes parallel boundaries, while the k =1 nearest neighbors fit closely to the data points. The bagging ensembles were trained using 10 base estimators with 0.8 subsampling of training data and 0.8 subsampling of features. The decision tree bagging ensemble achieved higher accuracy than the KNN bagging ensemble because KNN is less sensitive to perturbation on training samples; therefore, they are called stable learners. Combining stable learners is less advantageous, since the ensemble will not help improve generalization performance. Notice also that the decision boundary of the bagging KNN looks similar to the decision boundary of the bagging tree as a result of voting. The figure also shows how the test accuracy improves with the size of the ensemble. Based on cross-validation results, we can see the accuracy increases until approximately 20 base estimators and then plateaus afterward. Thus, adding base estimators beyond 20 only increases computational complexity, without accuracy gains for the iris dataset. The figure also shows learning curves for the bagging tree ensemble. We can see an average error of 0.15 on the training data and a U-shaped error curve for the testing data. The smallest gap between training and test errors occurs at around 80% of the training set size.
+
+A commonly used class of ensemble algorithms is forests of randomized trees. In a random forest, each tree in the ensemble is built from a sample drawn with replacement (i.e., a bootstrap sample) from the training set. In addition, instead of using all the features, a random subset of features is selected, further randomizing the tree. As a result, the bias of the forest increases slightly, but due to the averaging of less correlated trees, its variance decreases, resulting in an overall better model.
+
+#### Boosting
+
+Boosting refers to a family of algorithms that can convert weak learners to strong learners. The main principle of boosting is to fit a sequence of weak learners (i.e., models that are only slightly better than random guessing, such as small decision trees) to weighted versions of the data, where more weight is given to examples that were misclassified by earlier rounds. The predictions are then combined through a weighted majority vote (classification) or a weighted sum (regression) to produce the final prediction. The principal difference between boosting and the committee methods, such as bagging, is that base learners are trained in sequence on a weighted version of the data.
+
+AdaBoost algorithm:
+
+```python
+import itertools
+import numpy as np
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+
+from sklearn import datasets
+
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.linear_model import LogisticRegression
+
+from sklearn.ensemble import AdaBoostClassifier
+from sklearn.model_selection import cross_val_score, train_test_split
+
+from mlxtend.plotting import plot_learning_curves
+from mlxtend.plotting import plot_decision_regions
+
+def main():
+
+    iris = datasets.load_iris()
+    X, y = iris.data[:, 0:2], iris.target
+    
+    #XOR dataset
+    #X = np.random.randn(200, 2)
+    #y = np.array(map(int,np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)))
+    
+    clf = DecisionTreeClassifier(criterion='entropy', max_depth=1)
+
+    num_est = [1, 2, 3, 10]
+    label = ['AdaBoost (n_est=1)', 'AdaBoost (n_est=2)', 'AdaBoost (n_est=3)', 'AdaBoost (n_est=10)']
+    
+    fig = plt.figure(figsize=(10, 8))
+    gs = gridspec.GridSpec(2, 2)
+    grid = itertools.product([0,1],repeat=2)
+
+    for n_est, label, grd in zip(num_est, label, grid):     
+        boosting = AdaBoostClassifier(base_estimator=clf, n_estimators=n_est)   
+        boosting.fit(X, y)
+        ax = plt.subplot(gs[grd[0], grd[1]])
+        fig = plot_decision_regions(X=X, y=y, clf=boosting, legend=2)
+        plt.title(label)
+
+    plt.show()
+    #plt.savefig('./figures/boosting_ensemble.png')             
+
+    #plot learning curves
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
+
+    boosting = AdaBoostClassifier(base_estimator=clf, n_estimators=10)
+        
+    plt.figure()
+    plot_learning_curves(X_train, y_train, X_test, y_test, boosting, print_model=False, style='ggplot')
+    plt.show()
+    #plt.savefig('./figures/boosting_ensemble_learning_curve.png')             
+
+    #Ensemble Size
+    num_est = list(map(int, np.linspace(1,100,20)))
+    bg_clf_cv_mean = []
+    bg_clf_cv_std = []
+    for n_est in num_est:
+        print("num_est: ", n_est)
+        ada_clf = AdaBoostClassifier(base_estimator=clf, n_estimators=n_est)
+        scores = cross_val_score(ada_clf, X, y, cv=3, scoring='accuracy')
+        bg_clf_cv_mean.append(scores.mean())
+        bg_clf_cv_std.append(scores.std())
+        
+    plt.figure()
+    (_, caps, _) = plt.errorbar(num_est, bg_clf_cv_mean, yerr=bg_clf_cv_std, c='blue', fmt='-o', capsize=5)
+    for cap in caps:
+        cap.set_markeredgewidth(1)                                                                                                                                
+    plt.ylabel('Accuracy'); plt.xlabel('Ensemble Size'); plt.title('AdaBoost Ensemble');
+    plt.show()
+    #plt.savefig('./figures/boosting_ensemble_size.png')
+
+if __name__ == "__main__":    
+    main()
+```
+
+<img width="800" alt="image" src="https://github.com/user-attachments/assets/b935798d-7601-49b7-b588-26f15e5b2895">
+
+The boosting ensemble is illustrated in figure 7.16. Each base learner consists of a decision tree with depth 1, thus classifying the data based on a feature threshold that partitions the space into two regions separated by a linear decision surface parallel to one of the axes. The figure also shows how the test accuracy improves with the size of the ensemble and the learning curves for training and testing data. Gradient tree boosting is a generalization of boosting to arbitrary differentiable loss functions. It can be used for both regression and classification problems. Gradient boosting builds the model in a sequential way.
+
+At each stage the decision tree, hm(x) is chosen to minimize a loss function L given the current model Fm– 1(x).
+
+#### Stacking
+
+Stacking is an ensemble learning technique that combines multiple classification or regression models via a meta-classifier or meta-regressor. The base level models are trained based on complete training set, and then the meta-model is trained on the outputs of base-level model as features. The base level often consists of different learning algorithms; therefore, stacking ensembles are often heterogeneous.
+
+<img width="631" alt="image" src="https://github.com/user-attachments/assets/2bfe3093-8e6f-495a-a7e5-2b32d496a989">
+
+```python
+import itertools
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+
+from sklearn import datasets
+
+from sklearn.linear_model import LogisticRegression
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.naive_bayes import GaussianNB 
+from sklearn.ensemble import RandomForestClassifier
+from mlxtend.classifier import StackingClassifier
+
+from sklearn.model_selection import cross_val_score, train_test_split
+
+from mlxtend.plotting import plot_learning_curves
+from mlxtend.plotting import plot_decision_regions
+
+def main():
+
+    iris = datasets.load_iris()
+    X, y = iris.data[:, 1:3], iris.target
+
+    clf1 = KNeighborsClassifier(n_neighbors=1)
+    clf2 = RandomForestClassifier(random_state=1)
+    clf3 = GaussianNB()
+    lr = LogisticRegression()
+    sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], 
+                              meta_classifier=lr)
+
+    label = ['KNN', 'Random Forest', 'Naive Bayes', 'Stacking Classifier']
+    clf_list = [clf1, clf2, clf3, sclf]
+    
+    fig = plt.figure(figsize=(10,8))
+    gs = gridspec.GridSpec(2, 2)
+    grid = itertools.product([0,1],repeat=2)
+
+    clf_cv_mean = []
+    clf_cv_std = []
+    for clf, label, grd in zip(clf_list, label, grid):
+        
+        scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
+        print("Accuracy: %.2f (+/- %.2f) [%s]" %(scores.mean(), scores.std(), label))
+        clf_cv_mean.append(scores.mean())
+        clf_cv_std.append(scores.std())
+        
+        clf.fit(X, y)
+        ax = plt.subplot(gs[grd[0], grd[1]])
+        fig = plot_decision_regions(X=X, y=y, clf=clf)
+        plt.title(label)
+
+    plt.show()
+
+    #plot classifier accuracy    
+    plt.figure()
+    (_, caps, _) = plt.errorbar(range(4), clf_cv_mean, yerr=clf_cv_std, c='blue', fmt='-o', capsize=5)
+    for cap in caps:
+        cap.set_markeredgewidth(1)                                                                                                                                
+    plt.xticks(range(4), ['KNN', 'RF', 'NB', 'Stacking'], rotation='vertical')        
+    plt.ylabel('Accuracy'); plt.xlabel('Classifier'); plt.title('Stacking Ensemble');
+    plt.show()
+    #plt.savefig('./figures/stacking_ensemble_size.png')      
+        
+    #plot learning curves
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
+    
+    plt.figure()
+    plot_learning_curves(X_train, y_train, X_test, y_test, sclf, print_model=False, style='ggplot')
+    plt.show()
+    #plt.savefig('./figures/stacking_ensemble_learning_curve.png')             
+
+
+if __name__ == "__main__":
+    main()
+```
+
+<img width="802" alt="image" src="https://github.com/user-attachments/assets/ed886b93-050e-4ec5-a8b7-e29e10d4b3be">
+
+The stacking example above consists of KNN, Random Forest, and naive Bayes base classifiers whose predictions are combined by logistic regression as a meta-classifier. We can see the blending of decision boundaries achieved by the stacking classifier. The figure also shows that stacking achieves higher accuracy than individual classifiers, and based on learning curves, it shows no signs of overfitting.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 0abf877b726f637909f3197be23ad516074ab0be Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 20 Aug 2024 22:43:59 +0900
Subject: [PATCH 391/638] Create 2024-08-20-day-232.md

---
 _posts/2024-08-20-day-232.md | 126 +++++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100644 _posts/2024-08-20-day-232.md

diff --git a/_posts/2024-08-20-day-232.md b/_posts/2024-08-20-day-232.md
new file mode 100644
index 000000000..52dfbf773
--- /dev/null
+++ b/_posts/2024-08-20-day-232.md
@@ -0,0 +1,126 @@
+---
+layout: post
+title: (Day 232) Creating a script for the technical part of the KB project
+categories: [traditional-machine-learning,applying-knowledge,gnn,mlops]
+---
+
+# Hello :) Today is Day 232!
+A quick summary of today:
+* today we met with my lab mate in order to prepare our ppt
+
+We got an email from the competition organisers that the deadline for an updated ppt is this Sunday evening. So with the help of our lab professor (Prof. Minju Park) we created a plan for our updated ppt and my part is to create a script and provide detailed info about the technical parts of the project. 
+
+Below is the script I have so far
+
+## Architecture
+
+<img width="613" alt="image" src="https://github.com/user-attachments/assets/06dc9748-53ef-4395-849a-dbcb086a920b">
+
+For each number I have a short description, but it is in Korean. 
+
+* We use Docker to containerize everything so that it can easily be used across different systems
+* Mage is used as an orchestrator that allows the training and real-time pipelines to be developed and run. Once developed, Mage’s interface allows for the pipelines to be run by both technical engineers and non-technical analysts.
+* We use neo4j as our graph database
+* We developed and use models based on a few libraries - PyG, scikit-learn, catboost, and xgboost
+* We use mlflow for model tracking and registry
+* To imitate real-time data, we set up a script that sends transactions through Kafka. Kafka ensures the efficient processing of real-time data
+* For real-time inference, we run a pipeline that in real-time received transaction data from Kafka, takes models saved in mlflow, processes the transactions by adding predictions from our loaded models, and uploads it to neo4j
+* For insights and anslysis, we connect neo4j to Grafana to create a real-time dashboard that can update every second, or in an interval chosen by the user
+* Finally, using the models saved in mlflow, we provide a model dictionary that helps to better understand the decisions of the developed models
+
+## Data collection
+
+We thought we had to find actual credit card data to detect fraudulent transactions. Kaggle is an online platform that provides datasets for data science and machine learning, and allows data scientists around the world to use them to solve various problems. There were many options in Kaggle, but we needed to understand the specific requirements for detecting fraudulent transactions. 
+
+This involves (Seon - used by Revolut, Wise, 2024):
+* Information about the user - like personal information, location, and our credit card number; 
+* Information about the merchant - their name, and location
+* Information about the transaction itself - amount, time and date, and what was the transaction for (category) 
+
+## Database
+
+The data comes in tabular format from Kaggle, but because fraud detection is inherently a graph problem (Li et al., 2024, Acevedo-Viloria et al., 2021) we wanted to transform it to a graph for better visualisations, analysis and later model development. We chose the most widely used open-source graph database - neo4j. Neo4j is made specifically for graph data so queries are very fast and as you can see from the picture, we can do ad-hoc analysis of the data as a graph and get quick insights if needed. 
+
+<img width="649" alt="image" src="https://github.com/user-attachments/assets/b504f0c5-3d4a-43fb-a3f9-f1d26372e1f2">
+
+## Data preprocessing
+
+Fraud transaction data tends to be imbalanced. That is because most of the transactions that happen on a daily basis are not fraudulent. The not fraud to fraud ratio in our dataset reflects reality and we have 99.4% not fraud transactions and 0.6% fraud transactions.
+
+The Kaggle dataset comes with 1,852,393 transactions. We use 70% of the data to create our models and insert into our database. And the rest 30% we save to develop our pseudo streaming pipeline where every second transactions are sent through our real-time data processor Kafka.
+
+<img width="575" alt="image" src="https://github.com/user-attachments/assets/e9683d1d-50c5-4878-847f-149754f6d85a">
+
+Of the data used for the models, we split the data 80/20 into train/test.
+
+When performing feature selection, we took into account all features and considered their implications for AI ethics, so we did not want to include features based on name, gender, or job. 
+
+The selected variables are circled in red. Before using the features, extra features like day of week, and hour of day were created, and categorical features were turned into dummies.  
+
+To overcome the imbalance in our data we considered the 3 top methods - oversampling the minority class, downsampling the majority class, and a mixture of the previous two. However from our model development process, we found that downsampling the majority class fits our model the best. This results in around 1600 not fraud and 1600 fraud transactions left to train the models.
+
+We will see in a second, but our main model is a variation of a Graph Neural Network and we use lat, long, merch_lat, and merch_long only for that model because it is more powerful and can learn from those values without them being preprocessed.
+
+<img width="636" alt="image" src="https://github.com/user-attachments/assets/b3861f17-3274-4759-9a28-3cc14f5a43b7">
+
+## Model information
+
+Because fraud detection is inherently a graph problem, we wanted to construct a Graph Neural Network as our main model. A Graph Convolutional Network (GCN) is great for detecting fraud in transactions because it can handle large amounts of data efficiently - this is important because every second millions of transactions occur and we need a system that can handle this volume. In addition, GCNs understand and analyse complex connections between different accounts and transactions. It looks at how transactions are related to each other and can spot suspicious patterns that might be hard to see with other methods. A GCN updates each node’s features by aggregating and transforming features from its neighbours. It repeats this process across multiple layers to capture information from farther nodes, and improves its knowledge to predict whether a transaction is fraud or not fraud. 
+
+We wanted to give extra help to our decision-making system, so we looked into more traditional machine learning models like Logistic Regression, XGBoost, RandomForest, CatBoost and LGBMClassifier. Among those, XGBoost and CatBoost ended up performing the best on our test dataset. We looked into non-graph neural network models because while the GCN focuses on relationships between transactions, CatBoost and XGBoost enhance the system by providing high accuracy by analysing individual transactions. By providing a system with multiple models, we ensure that all types of fraud are effectively detected, and make our fraud detection system more robust and reliable.
+
+For evaluation metrics, we chose Recall (TP / (TP + FN)) as the main one as it measures how well the system identifies all the fraudulent transactions, which is crucial for minimising missed fraud cases.
+
+<img width="609" alt="image" src="https://github.com/user-attachments/assets/8747d533-a3c7-4828-bbb0-5bb6b2a68b6a">
+
+Next, let's quickly see how easy it is to train a model.
+
+## Model training pipeline
+
+I created a video using capcut. [Link to youtube](https://youtu.be/NV50SsTDgjA)
+
+If our models are outdated, or if we detect data drift or some other anomaly that signifies our models need to be updated, we can easily do that in Mage - our orchestrator.
+
+As seen in the clip, training a model of choice in our orchestrator is done only in a few clicks. This allows both engineers and non-engineers to develop hypotheses, train a model and evaluate their ideas. A person just needs to open mage, go to pipelines, select the training pipeline of choice and click run.
+
+After training a model, if an analyst wants to see more in-depth information related to feature importance, confusion matrix, etc - they can go to the model dictionary. 
+
+## Model dictionary
+
+I created a video for this too. [Link to youtube](https://youtu.be/WV_qnh468fc)
+
+The model dictionary automatically picks up newly trained models and includes information related to model metrics, confusion matrices, feature importance and SHAP values. CatBoost and XGBoost models have SHAP values on their pages as SHAP values help with explaining predictions of machine learning models by attributing the contribution of each feature to the prediction. As for explaining our Graph Convolutional Network, neural networks are known for being black box models and as users we are not able to understand how they made decisions. However, the graph neural network library that we use (PyG) provides explainability features which, just like for a non-neural network model, can show us the feature importance.
+
+## Real-time pipeline
+
+<img width="616" alt="image" src="https://github.com/user-attachments/assets/5ae68964-1cfe-4e2e-a4af-36466edf0593">
+
+Just like with the training pipelines in our orchestrator - Mage, we can run the real-time pipeline with a click of a button, and it will run indefinitely. We use Kafka which is an industry standard when it comes to real-time data processing. 
+
+In the pipeline, models from our model registry are loaded and used for inference. Real-time transaction data comes in, it is put through the pipeline and saved in neo4j. 
+
+Below is an example. A transaction with all of its data goes into the pipeline, and comes out with the same transaction data plus predictions whether it is fraud or not fraud according to our models.
+
+Data that goes in: {'trans_date_trans_time': '2020-06-21 18:25:53', 'cc_num': 5289285402893489, 'merchant': 'fraud_Zboncak LLC', 'category': 'food_dining', 'amt': 8.77, 'first': 'Amanda', 'last': 'Adams', 'gender': 'F', 'street': '08580 Jeremy Falls', 'city': 'Bay City', 'state': 'OR', 'zip': 97107, 'lat': 45.5197, 'long': -123.8761, 'city_pop': 1530, 'job': 'Colour technologist', 'dob': '1986-11-24', 'trans_num': 'cbd1cd064824086c5854ec227a58ea8a', 'unix_time': 1371839153, 'merch_lat': 45.544353, 'merch_long': -124.396705}
+
+Data that goes out: {'trans_date_trans_time': '2020-06-21 18:25:53', 'cc_num': 5289285402893489, 'merchant': 'fraud_Zboncak LLC', 'category': 'food_dining', 'amt': 8.77, 'first': 'Amanda', 'last': 'Adams', 'gender': 'F', 'street': '08580 Jeremy Falls', 'city': 'Bay City', 'state': 'OR', 'zip': 97107, 'lat': 45.5197, 'long': -123.8761, 'city_pop': 1530, 'job': 'Colour technologist', 'dob': '1986-11-24', 'trans_num': 'cbd1cd064824086c5854ec227a58ea8a', 'unix_time': 1371839153, 'merch_lat': 45.544353, 'merch_long': -124.396705, 'is_fraud_gcn': 0, 'is_fraud_xgboost': 0, 'is_fraud_catboost': 0}
+
+Thanks to Kafka, we are ensured that real-time transaction data is received on time and in our pipeline we append 3 values: a prediction from a GCN model, an XGBoost model, and a CatBoost model.
+
+The last step in the real-time pipeline is sending the data to our graph database (neo4j). Mage and Kafka allow for real-time transaction processing with minimum lag between transaction occurrence and the transaction being sent to our database. 
+
+## Visualisation
+
+_TODO: create a video showing off Grafana_
+
+After a transaction has passed through our real-time pipeline, it is stored in neo4j. To better understand our data, and be able to create dashboards to share with upper management we use Grafana. Grafana one of the top dashboard options as it has integrations with all widely used databases, including neo4j. Grafana allows for both real-time and static dashboards as well, which is perfect for our case. In the video sample, we can see an example dashboard with the latest transaction sample, geomap showing fraud transactions, and other information that can be useful to an analyst. 
+
+Thanks to Grafana’s great accessibility, both engineers and business analysts can go to Grafana at any time, query the database and immediately create their own visualisations and share them with team members or other departments without any previous setup.
+
+---
+
+Tomorrow we continue!
+
+That is all for today!
+
+See you tomorrow :)

From a7274e27123b6b1a9eff125640dd433b0715c88f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 22 Aug 2024 00:33:30 +0900
Subject: [PATCH 392/638] Create 2024-08-21-day-233.md

---
 _posts/2024-08-21-day-233.md | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 _posts/2024-08-21-day-233.md

diff --git a/_posts/2024-08-21-day-233.md b/_posts/2024-08-21-day-233.md
new file mode 100644
index 000000000..f44de5ab7
--- /dev/null
+++ b/_posts/2024-08-21-day-233.md
@@ -0,0 +1,48 @@
+---
+layout: post
+title: (Day 233) Sending notifications for suspicious transactions to customers
+categories: [applying-knowledge,mlops]
+---
+
+# Hello :) Today is Day 233!
+A quick summary of today:
+* grafana alert issues
+* setting up KakaoTalk alerts
+
+This will be a quick one as it is midnight and I am really sleepy. I could not write this earlier as I was debugging issues with Grafana (at first), and then setting up my Kakao API.
+
+As for the Grafana issues. We wanted to send alerts whenever a fraud transaction is detected. In Grafana I set up an alert dashboard where if all models predict a transaction is fraud, the transaction appears there
+
+<img width="1132" alt="image" src="https://github.com/user-attachments/assets/cedcbd5b-41bf-49a9-b95f-31a44883a950">
+
+Part of the problem was that initially there was not a transaction in my database that was predicted to be fraud by all models so this dashboard kept not working even though I set up the code that queries the database to just return 1s - but this did not work, and it started working *like magic* at some point. Then I realised that I guess this dashboard requires data to be there. 
+
+Anyway, that was fixed.
+
+### Email notifications
+
+I wanted to setup to send an email every time a transaction was labaled as fraud, but this required some smtp server. After some searching online I could not find a way to set it up. I found some websites, but for some reason they denied my account(?) which I did not understand. 
+
+Because I could not set up an email notif system, I set it up in Discord which worked after some setup. This was fine, but not really ideal for presentation in front of the Korean judges. 
+
+### KakaoTalk notifications
+
+Then my project partner suggested to use KakaoTalk to send notifications to the customers. This was a great suggestion as I could easily write this in python and add it to our pipeline. After some platform setting up, I started getting messages on my personal account. 
+
+<img width="434" alt="image" src="https://github.com/user-attachments/assets/ec0ffbcc-66f3-4acf-a25c-cbf31746b7c8">
+
+The text is in Korean, but it is just basic information about the transaction and asking the customer to contact the bank if they do not recognise this suspicious transaction.
+
+The bad thing was that because technically I am sending these to myself, I do not get notifications. So I started looking at how to add a push notification. To achieve that I needed to add a 2nd account (pseudo user, if you will) to the KB-alarm-bot - this is the name of the Kakao App for notifying customers, that sends messages. And now instead of sending messages to myself, a bot sends messages to me so I can get sound and in-app notifications.
+
+<img width="276" alt="image" src="https://github.com/user-attachments/assets/6f2edb07-c2aa-4d9b-b710-df60394c280e">
+
+At the end, I recorded a video. But today my project partner and I will do the final touches to make sure it looks great because this is the final and most important video:
+
+[Here is a link to youtube](https://youtu.be/pUzIDMvu-Xo)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 168e23a72b92dca77a8b3195ff933d9c779f15c5 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 22 Aug 2024 00:38:34 +0900
Subject: [PATCH 393/638] Update 2024-08-21-day-233.md

---
 _posts/2024-08-21-day-233.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-08-21-day-233.md b/_posts/2024-08-21-day-233.md
index f44de5ab7..d3450a23b 100644
--- a/_posts/2024-08-21-day-233.md
+++ b/_posts/2024-08-21-day-233.md
@@ -39,7 +39,7 @@ The bad thing was that because technically I am sending these to myself, I do no
 
 At the end, I recorded a video. But today my project partner and I will do the final touches to make sure it looks great because this is the final and most important video:
 
-[Here is a link to youtube](https://youtu.be/pUzIDMvu-Xo)
+[Here is a link to youtube](https://youtu.be/m4AusVLP7Hg)
 
 ---
 

From 168e1234dec5e869cf5820c1e3217d772af0479f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 22 Aug 2024 01:00:14 +0900
Subject: [PATCH 394/638] Update 2024-08-21-day-233.md

---
 _posts/2024-08-21-day-233.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-08-21-day-233.md b/_posts/2024-08-21-day-233.md
index d3450a23b..f1c196775 100644
--- a/_posts/2024-08-21-day-233.md
+++ b/_posts/2024-08-21-day-233.md
@@ -39,7 +39,7 @@ The bad thing was that because technically I am sending these to myself, I do no
 
 At the end, I recorded a video. But today my project partner and I will do the final touches to make sure it looks great because this is the final and most important video:
 
-[Here is a link to youtube](https://youtu.be/m4AusVLP7Hg)
+[Here is a link to youtube](https://youtu.be/FZKdIZHGGjA)
 
 ---
 

From 33cd6ace45fd095276c21badf9c62909e6f0da1e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 23 Aug 2024 01:35:53 +0900
Subject: [PATCH 395/638] Create 2024-08-22-day-234.md

---
 _posts/2024-08-22-day-234.md | 116 +++++++++++++++++++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 _posts/2024-08-22-day-234.md

diff --git a/_posts/2024-08-22-day-234.md b/_posts/2024-08-22-day-234.md
new file mode 100644
index 000000000..bf54039ab
--- /dev/null
+++ b/_posts/2024-08-22-day-234.md
@@ -0,0 +1,116 @@
+---
+layout: post
+title: (Day 234) Improving the Grafana dashboard and writing a final script for the KB project presentation
+categories: [applying-knowledge,mlops]
+---
+
+# Hello :) Today is Day 234!
+A quick summary of today:
+* worked in improving the Grafana dashboard
+* wrote a final script for the technical part of the presentation
+
+I fell asleep and I woke up and was about to completely go to bed, but I am writing this blog again right before bed as I am a bit exhausted from today.
+
+Mainly I played around with Grafana - making a bit more interesting dashboards so that in the real-time pipeline demo, the grafana dashboard looks a bit more interesting.
+
+In the below pics there are empty visualisations mainly because I cleaned the database so for tomorrow's recording I get a clean start. 
+
+<img width="1493" alt="image" src="https://github.com/user-attachments/assets/adc62f14-0c1e-44dd-bb96-4919fa3d7003">
+
+In the above picture, at the moment there are no fraud transactions, but if there are where it says 'No data in response' would show the transaction amount distribution for fraud cases.
+
+In addition in the table at the top, if a transaction is labeled as fraud from our system, the row will be red. I wanted to also add a hyperlink to the grafana alert dashboard but I could not figure out how to do conditional formatting to only add hyperlinks where the value of a row is something specific. What I could figure out is adding a hyperlink to each value in the table but that is not really useful to us.
+
+<img width="1494" alt="image" src="https://github.com/user-attachments/assets/d72d9c45-761f-46cf-8894-0409d58833e5">
+
+The bottom part is similar, and if there are fraud cases, one of the empty graphs will show a pie chart with the transaction's categories. And the bottom left (atm empty) table will show how many times there is a consensus between the models, and how many times there is a majority voting (2 out of 3). 
+
+## As for the script
+
+This is just the technical part and does not include the background, intro part, and going forward/improvements part.
+
+```
+기능구현 설계도
+This is our overall architecture. 
+We use Docker to containerize everything so that it can easily be used across different systems
+Mage is used as an orchestrator that allows the training and real-time pipelines to be developed and easily run. Mage’s interface allows for the pipelines to be run by both technical engineers and non-technical analysts.
+We use neo4j as our graph database as it is one of the most popular free, open-sourced and widely used graph databases.
+
+We develop two main pipelines: model training one and one for real-time inference.
+
+For the model training pipeline:
+We developed them using a few libraries - pytorch-geometric (PyG is an acronym) (for graph neural networks), scikit-learn, catboost, and xgboost
+We use mlflow for seamless model tracking and registry
+Using the models saved in mlflow, we provide a model dictionary that helps to better understand the decisions of the developed models and provides AI explainability features for both traditional machine learning methods and graph neural networks.
+
+Next, about our real-time pipeline: 
+To imitate real-time data, we set up a script that sends transactions to Kafka every second, and Kafka ensures its efficient processing.
+For real-time inference, we run a real-time pipeline in Mage that takes transaction data from Kafka, takes selected models that are saved in mlflow, processes the transactions by adding predictions from our models, and inserts it into neo4j
+For insights and anslysis, we connect neo4j to Grafana to create a real-time dashboard that can update in real-time. In Grafana, we also provide an alarm dashboard that keeps track of suspicious transactions.
+Finally, in the case of our models predicting that a transaction is suspicious, we immediately send a kakaotalk message to the customer.
+
+Next, we will look into each part of this architecture in a bit more detail. 
+
+Dataset (1)
+We first needed to find real credit card data to detect fraudulent transactions. As one of the most popular platforms for open-source data science, we looked into Kaggle and found a dataset that we believe reflects real-life transaction data as it consists of user, merchant, and transaction information. The transactions in the dataset are from the US for the period 2019-2020 and there are a total of 1.8m transactions. Just like reality, the majority of transactions are not fraudulent with 99.4% not fraud, and only 0.6% fraud cases. As mentioned before, we are developing two main pipelines - for model training and for real-time inference, so for this reason we decide to split our data 70% for model development, and 30% to simulate real-time data.
+First, we will talk about the model training pipeline and what happens inside of it.
+Dataset (2) 
+Given the complex and interconnected nature of transactions, utilising a graph database and a graph neural network becomes particularly advantageous for managing transaction data and improving fraud detection models. By capturing the relationships between transactions, these tools allow for more sophisticated analysis, uncovering patterns and anomalies that might be missed with traditional methods.
+
+To use Neo4j, we first convert our table data into a graph format. Since, in real life, customers use credit cards at various merchants, we modelled our data similarly. We created nodes to represent CreditCards and Merchants, with Transactions as edges connecting CreditCards to Merchants.
+
+For training our models, during our feature selection process, we took into account all features and considered their implications for AI ethics, so we did not want to include features based on name, gender, or job. So we ended up using only location information like longitude and latitude for the customer and merchant, and transaction information like amount, category, and date and time of occurance.
+
+After selecting our features, we did some preprocessing.
+
+Dataset (3)
+
+From our model development process, we found that downsampling the majority class fits our case the best, and this resulted in around 7500 not fraud and 7500 fraud transactions left to train and test the models. Next, since we have text data such as credit card state, transaction category, date, and time, we convert these features into dummy variables to make them usable in our models. And our train/test split ratio is 80/20. 
+
+Modelling (1) - 모델 정보
+
+Because fraud detection is inherently a graph problem, we wanted to construct a Graph Neural Network as our main model. A Graph Convolutional Network (GCN) is great for detecting fraud in transactions because it can handle large amounts of data efficiently - this is important because every second millions of transactions occur and we need a system that can handle this volume. In addition, GCNs understand and analyse complex connections between different credit cards and transactions. It looks at how transactions are related to each other and can spot suspicious patterns that might be hard to see with other methods. A GCN updates each node’s features by aggregating and transforming features from its neighbours. It repeats this process across multiple layers to capture information from farther nodes, and improves its knowledge to predict whether a transaction is fraud or not fraud. 
+
+We wanted to give extra help to our decision-making system, so we looked into more traditional machine learning models like Logistic Regression, XGBoost, RandomForest, CatBoost and LGBMClassifier. Among those, after hyperparameter tuning, XGBoost and CatBoost ended up performing the best on our test dataset. And the final hyperparameters can be seen. 
+
+We looked into non-graph neural network models because while the GCN focuses on relationships between transactions, CatBoost and XGBoost enhance the system by providing high accuracy by analysing individual transactions. By providing a system with multiple models, we hope that all types of fraud are effectively detected, and our fraud detection system is more robust and reliable.
+
+If our models are outdated, or if we detect data drift or some other anomaly that signifies our models need to be updated, we can easily do that in Mage - our orchestrator. And next, we will see a short demo on how to do that.
+
+Modelling (2) - 모델 훈련 Demo/모델 훈련 파이프라인 Demo
+
+- n/a
+
+Modelling (3) - 모델 사전 (Model Dictionary)
+
+The model dictionary automatically picks up newly trained models and displays it in our UI hosted by Streamlit. The dictionary includes information related to model metrics, confusion matrices, feature importance and SHAP values. CatBoost and XGBoost models have SHAP values on their pages as SHAP values help with explaining predictions of machine learning models by attributing the contribution of each feature to the prediction. As for explaining our Graph Convolutional Network, neural networks are known for being black box models and as users we are not able to understand how they made decisions. However, the graph neural network library that we use (pytorch-geometric) provides explainability features which, just like for a non-neural network model, can show us the feature importance.
+
+Modelling (4) - 모델 테스트 지표 요약
+
+In our real-time pipeline, about which we will talk next, we currently use 3 different models - a Graph Convolutional Network, CatBoost and XGBoost. This table shows the results of those models on our test dataset. For evaluation metrics, we chose Recall (TP / (TP + FN)) as the main one as it measures how well the system identifies all the fraudulent transactions, which is crucial for minimising missed fraud cases. And as you can see they all compliment each other when it comes to both recall and accuracy. 
+
+실시간 파이프라인
+
+Our real-time pipeline is designed for continuous and efficient processing of transaction data, ensuring timely predictions for fraud detection. The pipeline seamlessly integrates multiple components, from data ingestion to inference and storage, using industry-standard tools and our machine learning models.
+
+(Suggestion: For completeness, we can include mlflow and kakaotalk in the flow. Also under Mage, it is better to write: 실시간 추론 파이프라인)
+
+
+Of the total dataset from Kaggle, we left out 30% for our real-time pipeline development and execution. 
+We simulate real-world transactions using a custom script that generates and sends transaction data to Kafka every second. Kafka, known for its high-throughput and fault-tolerant messaging system, ensures that all incoming transaction data is captured in real-time. And as each transaction enters Kafka, it immediately goes through our real-time pipeline in Mage, ensuring minimal latency in data flow
+Mage pulls our main 3 models from mlflow (our model registry) and applies them to the incoming transactions
+For each transaction, predictions are generated using our models, and then the transaction data is sent to neo4j
+We have connected neo4j with Grafana, a widely used visualisation platform where we provide a real-time dashboard for analysts to play with; an alert dashboard which keeps track and updates whenever a fraudulent transaction is found.
+
+Finally, if a transaction is deemed fraudulent by our system, a kakaotalk message is sent to the associated customer as a warning
+```
+
+## Sending messages with Kakao
+
+Today, my lab mate gave me some suggestions for the real-time pipeline video that I shared yesterday. I wanted to implement them (like going slower, explaining things, just making the video clearer) and record a new video, but apparently for my free account there is an API limit for sending messages. It reset at midnight so tomorrow morning after I wake up I will re-record the final video and edit it. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From fe13e20eb2b9b4d6bd7d94ab3d09c6a48b14539b Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 24 Aug 2024 01:33:09 +0900
Subject: [PATCH 396/638] Create 2024-08-23-day-235.md

---
 _posts/2024-08-23-day-235.md | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 _posts/2024-08-23-day-235.md

diff --git a/_posts/2024-08-23-day-235.md b/_posts/2024-08-23-day-235.md
new file mode 100644
index 000000000..59fc76cb7
--- /dev/null
+++ b/_posts/2024-08-23-day-235.md
@@ -0,0 +1,29 @@
+---
+layout: post
+title: (Day 235) Re-recording the real-time pipeline video and getting final feedback on our ppt for the KB project
+categories: [applying-knowledge,mlops]
+---
+
+# Hello :) Today is Day 235!
+A quick summary of today:
+* recording the final real-time pipeline video
+* meeting with our lab professor for final feedback on our ppt
+
+
+Yesterday when I was recording the final video that shows the real-time pipeline in Mage
+
+<img width="1107" alt="image" src="https://github.com/user-attachments/assets/8da4416d-e2b5-4805-97ec-61e3baef6a87">
+
+I hit the kakaotalk messaging api rate limit so today I had to do the final version. As a complete novice when it comes to recording and editing - it is a bit hard. I spent around 3 hours recording and editing - and the final video is 40 seconds 😆 haha. [I uploaded it to youtube.](https://youtu.be/tdUVbWcj7oM)
+
+This was my morning till lunch. After that I went to the uni lab to meet with my project partner and our professor to talk about our ppt. When we have the final final version of our ppt I will share it here as well. The submission deadline is this Sunday afternoon as Monday evening is a rehersal and Tuesday is the event itself with all 10 finalists - I will share info about these events in due course as well. 
+
+In our ppt we basically go over background info, why our idea for a real-time pipeline for fraud detection, and then we dive into the two main pipelines - model training pipeline and real-time pipeline like shown in the picture below:
+
+<img width="1105" alt="image" src="https://github.com/user-attachments/assets/d0e52d03-3ed2-42eb-bcfb-dae2a6a12a95">
+
+We met with our professor on Monday (or Tuesday) this week for initial feedback, and compared to know the professor said it is way better and is happy with our progress, and we can add just small things before we submit it on Sunday. 
+
+That is all for today!
+
+See you tomorrow :)

From b934ae4c407cf8dab6c3a9c9be6a297bb423229a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 25 Aug 2024 01:30:26 +0900
Subject: [PATCH 397/638] Create 2024-08-24-day-236.md

---
 _posts/2024-08-24-day-236.md | 583 +++++++++++++++++++++++++++++++++++
 1 file changed, 583 insertions(+)
 create mode 100644 _posts/2024-08-24-day-236.md

diff --git a/_posts/2024-08-24-day-236.md b/_posts/2024-08-24-day-236.md
new file mode 100644
index 000000000..1386e46e0
--- /dev/null
+++ b/_posts/2024-08-24-day-236.md
@@ -0,0 +1,583 @@
+---
+layout: post
+title: (Day 236) Reading about unsupervised learning algorithms + making the *final* version of our ppt videos
+categories: [traditional-machine-learning,applying-knowledge,theory]
+---
+
+# Hello :) Today is Day 236!
+A quick summary of today:
+* reading about fundamental unsupervised learning algorithms (Chapter 8 from the Machine Learning Algorithms in Depth book)
+* final final final video versions for our ppt
+
+Applications of unsupervised learning span from clustering customer segments in e-commerce to extracting features from image data. The author says the algorithms in this chapter were
+selected for their mathematical depth and usefulness in real-world applications.
+
+## Dirichlet process K-means
+
+The Dirichlet process K-means is a Bayesian nonparametric extension of the K-means algorithm, in which the number of clusters grows with data.
+
+### Key Concepts:
+- Dynamic Clustering: New clusters are created when a data point is sufficiently far from existing cluster centroids. This flexibility allows the algorithm to adapt to the data, automatically increasing the number of clusters as needed.
+- Objective Function: DP-means optimizes a K-means-like objective that includes a penalty term for the number of clusters, converging to a local optimum.
+- Reassignment Step: Similar to K-means, data points are reassigned to the nearest cluster, or a new cluster is initiated if the squared Euclidean distance to all existing centroids exceeds a threshold, λ.
+
+### Performance Metrics:
+- Normalized Mutual Information (NMI): Measures the similarity between the true label assignments (if available) and the computed cluster assignments. NMI ranges between 0 and 1, with higher values indicating more similar assignments.
+- Variation of Information (VI): Quantifies the difference between label assignments. VI decreases as the overlap between predicted and true labels increases.
+- Adjusted Rand Index (ARI): Computes the similarity between two clusters by analyzing all pairs of samples, approaching 1 when the predicted clusters closely match the true clusters.
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+
+import time
+from sklearn import metrics
+from sklearn.datasets import load_iris
+
+np.random.seed(42)
+
+class dpmeans:
+    
+    def __init__(self,X):
+        # Initialize parameters for DP means
+        self.K = 1
+        self.K_init = 4
+        self.d = X.shape[1]
+        self.z = np.mod(np.random.permutation(X.shape[0]),self.K)+1
+        self.mu = np.random.standard_normal((self.K, self.d))
+        self.sigma = 1
+        self.nk = np.zeros(self.K)
+        self.pik = np.ones(self.K)/self.K 
+        
+        #init mu
+        self.mu = np.array([np.mean(X,0)])
+        
+        #init lambda
+        self.Lambda = self.kpp_init(X,self.K_init)
+        
+        self.max_iter = 100
+        self.obj = np.zeros(self.max_iter)
+        self.em_time = np.zeros(self.max_iter)   
+        
+    def kpp_init(self,X,k):
+        #k++ init
+        #lambda is max distance to k++ means
+
+        [n,d] = np.shape(X)
+        mu = np.zeros((k,d))        
+        dist = np.inf*np.ones(n)
+            
+        mu[0,:] = X[int(np.random.rand()*n-1),:]
+        for i in range(1,k):
+            D = X-np.tile(mu[i-1,:],(n,1))
+            dist = np.minimum(dist, np.sum(D*D,1))
+            idx = np.where(np.random.rand() < np.cumsum(dist/float(sum(dist))))
+            mu[i,:] = X[idx[0][0],:]
+            Lambda = np.max(dist)
+        
+        print("Lambda: ", Lambda)
+        
+        return Lambda
+        
+    def fit(self,X):
+
+        obj_tol = 1e-3
+        max_iter = self.max_iter        
+        [n,d] = np.shape(X)
+        
+        obj = np.zeros(max_iter)
+        em_time = np.zeros(max_iter)
+        print('running dpmeans...')
+        
+        for iter in range(max_iter):
+            tic = time.time()
+            dist = np.zeros((n,self.K))
+            
+            #assignment step
+            for kk in range(self.K):
+                Xm = X - np.tile(self.mu[kk,:],(n,1))
+                dist[:,kk] = np.sum(Xm*Xm,1)
+            
+            #update labels
+            dmin = np.min(dist,1)
+            self.z = np.argmin(dist,1)
+            idx = np.where(dmin > self.Lambda)
+            
+            if (np.size(idx) > 0):
+                self.K = self.K + 1
+                self.z[idx[0]] = self.K-1 #cluster labels in [0,...,K-1]
+                self.mu = np.vstack([self.mu,np.mean(X[idx[0],:],0)])                
+                Xm = X - np.tile(self.mu[self.K-1,:],(n,1))
+                dist = np.hstack([dist, np.array([np.sum(Xm*Xm,1)]).T])
+             
+            #update step
+            self.nk = np.zeros(self.K)
+            for kk in range(self.K):
+                self.nk[kk] = self.z.tolist().count(kk)
+                idx = np.where(self.z == kk)
+                self.mu[kk,:] = np.mean(X[idx[0],:],0)
+            
+            self.pik = self.nk/float(np.sum(self.nk))
+            
+            #compute objective
+            for kk in range(self.K):
+                idx = np.where(self.z == kk)
+                obj[iter] = obj[iter] + np.sum(dist[idx[0],kk],0)                
+            obj[iter] = obj[iter] + self.Lambda * self.K
+            
+            #check convergence
+            if (iter > 0 and np.abs(obj[iter]-obj[iter-1]) < obj_tol*obj[iter]):
+                print('converged in %d iterations\n'% iter)
+                break
+            em_time[iter] = time.time()-tic
+        #end for
+        self.obj = obj
+        self.em_time = em_time
+        return self.z, obj, em_time
+        
+    def compute_nmi(self, z1, z2):
+        # compute normalized mutual information
+        
+        n = np.size(z1)
+        k1 = np.size(np.unique(z1))
+        k2 = np.size(np.unique(z2))
+        
+        nk1 = np.zeros((k1,1))
+        nk2 = np.zeros((k2,1))
+
+        for kk in range(k1):
+            nk1[kk] = np.sum(z1==kk)
+        for kk in range(k2):
+            nk2[kk] = np.sum(z2==kk)
+            
+        pk1 = nk1/float(np.sum(nk1))
+        pk2 = nk2/float(np.sum(nk2))
+        
+        nk12 = np.zeros((k1,k2))
+        for ii in range(k1):
+            for jj in range(k2):
+                nk12[ii,jj] = np.sum((z1==ii)*(z2==jj))
+        pk12 = nk12/float(n)        
+        
+        Hx = -np.sum(pk1 * np.log(pk1 + np.finfo(float).eps))
+        Hy = -np.sum(pk2 * np.log(pk2 + np.finfo(float).eps))
+        
+        Hxy = -np.sum(pk12 * np.log(pk12 + np.finfo(float).eps))
+        
+        MI = Hx + Hy - Hxy;
+        nmi = MI/float(0.5*(Hx+Hy))
+        
+        return nmi
+
+    def generate_plots(self,X):
+
+        plt.close('all')
+        plt.figure(0)
+        for kk in range(self.K):
+            #idx = np.where(self.z == kk)
+            plt.scatter(X[self.z == kk,0], X[self.z == kk,1], \
+                        s = 100, marker = 'o', c = np.random.rand(3,), label = str(kk))
+        #end for
+        plt.xlabel('X1')
+        plt.ylabel('X2')
+        plt.legend()
+        plt.title('DP-means clusters')
+        plt.grid(True)
+        plt.show()
+        
+        plt.figure(1)
+        plt.plot(self.obj)
+        plt.title('DP-means objective function')
+        plt.xlabel('iterations')
+        plt.ylabel('penalized l2 squared distance')
+        plt.grid(True)
+        plt.show()
+        
+if __name__ == "__main__":        
+    
+    iris = load_iris()
+    X = iris.data
+    y = iris.target
+    
+    dp = dpmeans(X)
+    labels, obj, em_time = dp.fit(X)
+    dp.generate_plots(X)
+
+    nmi = dp.compute_nmi(y,labels)
+    ari = metrics.adjusted_rand_score(y,labels)
+    
+    print("NMI: %.4f" % nmi)
+    print("ARI: %.4f" % ari)
+```
+
+![image](https://github.com/user-attachments/assets/0b5d0665-b6a6-487d-9c94-ce580556ef19)
+![image](https://github.com/user-attachments/assets/82d897d7-6b35-4d05-862e-85760a1da8b5)
+
+NMI: 0.7582
+
+ARI: 0.7302
+
+## Gaussian mixture models
+
+I meet them again. For the third time actually. First in the papers where I read about predicting taxi demand and using a GMM on top of a normal model so the output is a distribution, 2nd when I watched the `probabl` youtube channel, and now third. 
+
+Mixture models are commonly used to model complex density distributions. For example, you may be interested in discovering patterns in census data consisting of information about the person’s age, income, occupation, and other dimensions. If we plot the resulting data in high-dimensional space, we’ll likely discover nonuniform density characterized by groups or clusters of data points. We can model each cluster using a base probability distribution. Mixture models consist of a convex combination of K base models. In the case of Gaussian mixture models, the base distribution is Gaussian and can be written as follows.
+
+![image](https://github.com/user-attachments/assets/b182d38e-631a-4569-a9f1-7defe21ee5ec)
+
+where π are the mixing proportions, and the total is 1. 
+
+### Expectation maximization (EM) algorithm
+
+EM is an iterative algorithm that alternates between inferring the missing values given the parameters (E step) and then optimizing the parameters given filled-in data (M step).
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+from sklearn.cluster import KMeans
+from scipy.stats import multivariate_normal
+from scipy.special import logsumexp
+from scipy import linalg
+
+np.random.seed(3)
+
+class GMM:
+
+    def __init__(self, n=1e3, d=2, K=4):
+        self.n = int(n)  #number of data points
+        self.d = d  #data dimension 
+        self.K = K  #number of clusters
+
+        self.X = np.zeros((self.n, self.d))
+
+        self.mu = np.zeros((self.d, self.K))
+        self.sigma = np.zeros((self.d, self.d, self.K))
+        self.pik = np.ones(self.K)/K
+
+    def generate_data(self):
+        #GMM generative model
+        alpha0 = np.ones(self.K) 
+        pi = np.random.dirichlet(alpha0)
+
+        #ground truth mu and sigma
+        mu0 = np.random.randint(0, 10, size=(self.d, self.K)) - 5*np.ones((self.d, self.K))
+        V0 = np.zeros((self.d, self.d, self.K))
+        for k in range(self.K):
+            eigen_mean = 0 
+            Q = np.random.normal(loc=0, scale=1, size=(self.d, self.d))
+            D = np.diag(abs(eigen_mean + np.random.normal(loc=0, scale=1, size=self.d)))
+            V0[:,:,k] = abs(np.transpose(Q)*D*Q)
+        
+        #sample data
+        for i in range(self.n):
+            z = np.random.multinomial(1,pi)
+            k = np.nonzero(z)[0][0]
+            self.X[i,:] = np.random.multivariate_normal(mean=mu0[:,k], cov=V0[:,:,k], size=1)
+
+        plt.figure()
+        plt.scatter(self.X[:,0], self.X[:,1], color='b', alpha=0.5)
+        plt.title("Ground Truth Data"); plt.xlabel("X1"); plt.ylabel("X2")
+        plt.show()
+
+        return mu0, V0
+
+    def gmm_em(self):
+        
+        #init mu with k-means
+        kmeans = KMeans(n_clusters=self.K, random_state=42).fit(self.X)
+        self.mu = np.transpose(kmeans.cluster_centers_)
+
+        #init sigma
+        for k in range(self.K):
+            self.sigma[:,:,k] = np.eye(self.d)
+
+        #EM algorithm
+        max_iter = 10
+        tol = 1e-5
+        obj = np.zeros(max_iter)
+        for iter in range(max_iter):
+            print("EM iter ", iter)
+            #E-step
+            resp, llh = self.estep()
+            #M-step
+            self.mstep(resp)
+            #check convergence
+            obj[iter] = llh
+            if (iter > 1 and obj[iter] - obj[iter-1] < tol*abs(obj[iter])):
+                break
+            #end if
+        #end for
+        plt.figure()
+        plt.plot(obj)
+        plt.title('EM-GMM objective'); plt.xlabel("iter"); plt.ylabel("log-likelihood")
+        plt.show()
+
+    def estep(self):
+        
+        log_r = np.zeros((self.n, self.K))
+        for k in range(self.K):
+            log_r[:,k] = multivariate_normal.logpdf(self.X, mean=self.mu[:,k], cov=self.sigma[:,:,k])
+        #end for
+        log_r = log_r + np.log(self.pik)
+        L = logsumexp(log_r, axis=1)
+        llh = np.sum(L)/self.n  #log likelihood
+        log_r = log_r - L.reshape(-1,1) #normalize
+        resp = np.exp(log_r)
+        return resp, llh
+
+    def mstep(self, resp):
+
+        nk = np.sum(resp, axis=0)
+        self.pik = nk/self.n
+        sqrt_resp = np.sqrt(resp)
+        for k in range(self.K):
+            #update mu
+            rx = np.multiply(resp[:,k].reshape(-1,1), self.X)
+            self.mu[:,k] = np.sum(rx, axis=0) / nk[k]
+
+            #update sigma
+            Xm = self.X - self.mu[:,k]
+            Xm = np.multiply(sqrt_resp[:,k].reshape(-1,1), Xm)
+            self.sigma[:,:,k] = np.maximum(0, np.dot(np.transpose(Xm), Xm) / nk[k] + 1e-5 * np.eye(self.d))
+        #end for
+
+if __name__ == '__main__':
+
+    gmm = GMM()
+    mu0, V0 = gmm.generate_data()
+    gmm.gmm_em()
+    
+    for k in range(mu0.shape[1]):
+        print("cluster ", k)
+        print("-----------")
+        print("ground truth means:")
+        print(mu0[:,k])
+        print("ground truth covariance:")
+        print(V0[:,:,k])
+    #end for 
+    
+    for k in range(mu0.shape[1]):
+        print("cluster ", k)
+        print("-----------")
+        print("GMM-EM means:")
+        print(gmm.mu[:,k])
+        print("GMM-EM covariance:")
+        print(gmm.sigma[:,:,k])
+
+    plt.figure()
+    ax = plt.axes()
+    plt.scatter(gmm.X[:,0], gmm.X[:,1], color='b', alpha=0.5)
+
+    for k in range(mu0.shape[1]):
+
+        v, w = linalg.eigh(gmm.sigma[:,:,k])
+        v = 2.0 * np.sqrt(2.0) * np.sqrt(v)
+        u = w[0] / linalg.norm(w[0])
+
+        # plot an ellipse to show the Gaussian component
+        angle = np.arctan(u[1] / u[0])
+        angle = 180.0 * angle / np.pi  # convert to degrees
+        ell = mpl.patches.Ellipse(gmm.mu[:,k], v[0], v[1], 180.0 + angle, color='r', alpha=0.5)
+        ax.add_patch(ell)
+
+        # plot cluster centroids
+        plt.scatter(gmm.mu[0,k], gmm.mu[1,k], s=80, marker='x', color='k', alpha=1)
+    plt.title("Gaussian Mixture Model"); plt.xlabel("X1"); plt.ylabel("X2")
+    plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/cd461566-8191-456f-ab88-dcfadae81e4b)
+
+EM over the iterations:
+
+![image](https://github.com/user-attachments/assets/5817e358-9ab6-4078-bc46-4659f66b76a0)
+
+```
+cluster  0
+-----------
+ground truth means:
+[-2.  2.]
+ground truth covariance:
+[[0.00169216 0.        ]
+ [0.         1.33783795]]
+cluster  1
+-----------
+ground truth means:
+[4. 1.]
+ground truth covariance:
+[[2.45922460e-03 0.00000000e+00]
+ [0.00000000e+00 2.63330463e+00]]
+cluster  2
+-----------
+ground truth means:
+[ 4. -5.]
+ground truth covariance:
+[[1.43773758 0.        ]
+ [0.         0.03995228]]
+cluster  3
+-----------
+ground truth means:
+[ 0. -1.]
+ground truth covariance:
+[[0.29126579 0.        ]
+ [0.         0.10456419]]
+cluster  0
+-----------
+GMM-EM means:
+[3.99752939 0.92575035]
+GMM-EM covariance:
+[[2.38145097e-03 3.76559287e-03]
+ [3.76559287e-03 2.63147414e+00]]
+cluster  1
+-----------
+GMM-EM means:
+[ 0.05492661 -0.99472004]
+GMM-EM covariance:
+[[0.2790043  0.        ]
+ [0.         0.10012724]]
+cluster  2
+-----------
+GMM-EM means:
+[ 3.89443404 -4.99587958]
+GMM-EM covariance:
+[[1.78361804 0.00829965]
+ [0.00829965 0.04524815]]
+cluster  3
+-----------
+GMM-EM means:
+[-2.00245926  2.05932057]
+GMM-EM covariance:
+[[0.00141642 0.        ]
+ [0.         1.3039054 ]]
+```
+
+Inferred Gaussian mixture overlayed on the original data:
+
+![image](https://github.com/user-attachments/assets/e5df8e43-4e72-43a6-8a99-55b798e957a7)
+
+## Dimensionality reduction
+
+### PCA
+
+We would like to project our data vector x ∈ R^D onto a lower dimensional vector z ∈ R^L with L < D such that the variance of the projected data is maximized. Maximizing the variance of the projected data is the core principle of PCA, and it allows us to preserve the unique characteristics of our data.
+
+pseudo-code for PCA:
+
+![image](https://github.com/user-attachments/assets/c51adc1e-0d72-4930-94bc-0d7fe9e4a4ab)
+
+### t-SNE manifold learning on images
+
+Images are high dimensional objects that live on manifolds. A manifold is a topological space that locally resembles Euclidean space. By modeling image spaces as manifolds, we can study their geometric properties. We can visualize high dimensional objects with the help of an embedding. For this, two embeddings are considered: t-SNE and Isomap, and applied on the MNIST dataset.
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+
+from time import time
+from sklearn import manifold
+
+from sklearn.datasets import load_digits
+from sklearn.neighbors import KDTree
+
+def plot_digits(X):
+
+    n_img_per_row = np.amin((20, np.int(np.sqrt(X.shape[0]))))
+    img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row))
+    for i in range(n_img_per_row):
+        ix = 10 * i + 1
+        for j in range(n_img_per_row):
+            iy = 10 * j + 1
+            img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8))
+
+    plt.figure()
+    plt.imshow(img, cmap=plt.cm.binary)
+    plt.xticks([])
+    plt.yticks([])
+    plt.title('A selection from the 64-dimensional digits dataset')    
+
+def mnist_manifold():
+            
+    digits = load_digits()
+    
+    X = digits.data
+    y = digits.target
+    
+    num_classes = np.unique(y).shape[0]
+        
+    plot_digits(X)
+        
+    #TSNE 
+    #Barnes-Hut: O(d NlogN) where d is dim and N is the number of samples
+    #Exact: O(d N^2)
+    t0 = time()
+    tsne = manifold.TSNE(n_components = 2, init = 'pca', method = 'barnes_hut', verbose = 1)
+    X_tsne = tsne.fit_transform(X)
+    t1 = time()
+    print('t-SNE: %.2f sec' %(t1-t0))
+    tsne.get_params()
+    
+    plt.figure()
+    for k in range(num_classes):
+        plt.plot(X_tsne[y==k,0], X_tsne[y==k,1],'o')
+    plt.title('t-SNE embedding of digits dataset')
+    plt.xlabel('X1')
+    plt.ylabel('X2')
+    axes = plt.gca()
+    axes.set_xlim([X_tsne[:,0].min()-1,X_tsne[:,0].max()+1])
+    axes.set_ylim([X_tsne[:,1].min()-1,X_tsne[:,1].max()+1])
+    plt.show()
+        
+    #ISOMAP
+    #1. Nearest neighbors search: O(d log k N log N)
+    #2. Shortest path graph search: O(N^2(k+log(N))
+    #3. Partial eigenvalue decomposition: O(dN^2)
+    
+    t0 = time()
+    isomap = manifold.Isomap(n_neighbors = 5, n_components = 2)
+    X_isomap = isomap.fit_transform(X)
+    t1 = time()
+    print('Isomap: %.2f sec' %(t1-t0))
+    isomap.get_params()
+    
+    plt.figure()
+    for k in range(num_classes):
+        plt.plot(X_isomap[y==k,0], X_isomap[y==k,1], 'o', label=str(k), linewidth = 2)
+    plt.title('Isomap embedding of the digits dataset')
+    plt.xlabel('X1')
+    plt.ylabel('X2')
+    plt.show()
+    
+    #Use KD-tree to find k-nearest neighbors to a query image
+    kdt = KDTree(X_isomap)
+    Q = np.array([[-160, -30],[-102, 14]])
+    kdt_dist, kdt_idx = kdt.query(Q,k=20)
+    plot_digits(X[kdt_idx.ravel(),:])
+                                       
+if __name__ == "__main__":    
+    mnist_manifold()
+```
+
+![image](https://github.com/user-attachments/assets/48812fe8-66be-4fec-b28e-b97fd89dd9ea)
+![image](https://github.com/user-attachments/assets/6e5b999e-0cf6-4309-be95-5dd6e1df1171)
+![image](https://github.com/user-attachments/assets/bdcddd6b-d48b-4668-8c6a-ba6a04b83706)
+![image](https://github.com/user-attachments/assets/73f7ae2f-b6d1-4f81-9bbf-07c0ef23fed8)
+
+
+The above shows a t-SNE embedding with 10 clusters in 2D, where each cluster corresponds to a digit from 0 to 9. We can see that without labels, we are able to discover 10 clusters that use t-SNE embedding in the two-dimensional space. Moreover, we expect adjacent clusters of digits to be similar to each other. The image on the right-hand side of figure 8.7 shows sample digits from two adjacent clusters (digit 0 and digit 6). We can visualize adjacent clusters by constructing a KD tree to find KNN to a query point
+
+It’s important to be aware of some of the pitfalls of t-SNE. For example, t-SNE results may vary based on the perplexity hyperparameter. The t-SNE algorithm also doesn’t always produce similar outputs on successive runs and there are additional hyperparameters related to the optimization process. Moreover, the cluster sizes (as measured by standard deviation) and distances between clusters might not mean anything. Thus, the high flexibility of t-SNE also makes the results of the algorithm tricky to interpret.
+
+## KB project ppt
+
+Today I went to the lab and met with my project partner, we talked about the ppt and made final touches to it. I will share it tomorrow 
+
+[Here](https://youtu.be/wbqSekL2cgk) is the final final final video. We added some audio to make it nicer.
+
+And [here](https://youtu.be/FfEBdpXi-Pw) is a video showing how to re-train a model using Mage. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 3341129582d5dbbc374fb54f4a3f8a01717dfe14 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 25 Aug 2024 23:04:49 +0900
Subject: [PATCH 398/638] Create 2024-08-25-day-237.md

---
 _posts/2024-08-25-day-237.md | 444 +++++++++++++++++++++++++++++++++++
 1 file changed, 444 insertions(+)
 create mode 100644 _posts/2024-08-25-day-237.md

diff --git a/_posts/2024-08-25-day-237.md b/_posts/2024-08-25-day-237.md
new file mode 100644
index 000000000..7dc75301b
--- /dev/null
+++ b/_posts/2024-08-25-day-237.md
@@ -0,0 +1,444 @@
+---
+layout: post
+title: (Day 237) More unsupervised learning algorithms + submitting the KB project ppt
+categories: [traditional-machine-learning,applying-knowledge,theory]
+---
+
+# Hello :) Today is Day 237!
+A quick summary of today:
+* reading chapter 9 from the Machine Learning Algorithms in Depth book
+* re-watching previous year's KB Future Finance AI competition
+
+
+## Latent Dirichlet Allocation (LDA)
+
+Latent Dirichlet Allocation (LDA) is a topic model used to analyze discrete data, such as text documents. It represents each document as a mixture of topics, where each topic is a distribution over words. The goal of LDA is to learn shared topic distributions and topic proportions for each document.
+
+<img width="656" alt="image" src="https://github.com/user-attachments/assets/6b3d817e-8e67-4c41-8b4d-db1d1780985c">
+
+### Bag of Words Model
+LDA assumes a bag of words model, meaning that words are treated as exchangeable, and sentence structure is ignored. Each document is represented as a vector of word counts, and the entire corpus is summarized in a term-document matrix.
+
+### Matrix Factorization
+LDA can be viewed as a nonnegative matrix factorization problem. The term-document matrix is factorized into a product of topics and topic proportions:
+- **A = WH**
+  - **W**: Topics (distribution over words)
+  - **H**: Topic proportions
+
+### TF-IDF and Smoothing
+Term frequency–inverse document frequency (tf-idf) is often used to adjust word counts, reducing the influence of words that frequently occur across documents. This method helps in identifying discriminative words and improving model performance. For n-grams, different smoothing techniques like Laplace smoothing are used to handle sparse data.
+
+### Generative Model
+In LDA, each word in a document is associated with a topic label, and each document is associated with topic proportions (θd). Topics (βk) are shared across all documents, and hyperparameters α and η capture prior knowledge about topic proportions and topics.
+
+### Inference and Variational Bayes
+The key inference problem in LDA is computing the posterior distribution of latent variables for a given document. This can be approximated using a variational distribution that is optimized to maximize the evidence lower bound (ELBO). The variational parameter updates can be done in an online setting, allowing for topic analysis on large datasets or streaming data.
+
+## Density estimators
+
+The goal of density estimation is to model the probability density of data.
+
+### Kernel density estimator
+
+An alternative approach to a K-component mixture model is a kernel density estimator (KDE) that allocates one cluster center per data point. KDEs are an application of kernel smoothing for probability density estimation that use kernels as weights
+
+The advantage of KDEs over parametric models, such as density mixtures, is that no model fitting is required (except for fine-tuning the bandwidth parameter h) and there is no need to pick the number of mixtures K. The disadvantage is that the model takes a lot of memory to store as well as time to evaluate. In other words, KDE is suitable when an accurate density estimate is required for a relatively small dataset (small number of points N)
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+
+np.random.seed(14)
+
+class KDE():
+
+    def __init__(self):
+        #Histogram and Gaussian Kernel Estimator used to
+        #analyze RNA-seq data for flux estimation of a T7 promoter
+        self.G = 1e9  #length of genome in base pairs (bp)
+        self.C = 1e3  #number of unique molecules
+        self.L = 100  #length of a read, bp
+        self.N = 1e6  #number of reads, L bp long
+        self.M = 1e4  #number of unique read sequences, bp
+        self.LN = 1000  #total length of assembled / mapped RNA-seq reads
+        self.FDR = 0.05  #false discovery rate
+
+        #uniform sampling (poisson model)
+        self.lmbda = (self.N * self.L) / self.G  #expected number of bases covered
+        self.C_est = self.M/(1-np.exp(-self.lmbda)) #library size estimate
+        self.C_cvrg = self.G - self.G * np.exp(-self.lmbda) #base coverage
+        self.N_gaps = self.N * np.exp(-self.lmbda) #number of gaps (uncovered bases)
+
+        #gamma prior sampling (negative binomial model)
+        #X = "number of failures before rth success"
+        self.k = 0.5 # dispersion parameter (fit to data)
+        self.p = self.lmbda/(self.lmbda + 1/self.k) # success probability
+        self.r = 1/self.k # number of successes
+        
+        #RNAP binding data (RNA-seq)
+        self.data = np.random.negative_binomial(self.r, self.p, size=self.LN)
+
+    def histogram(self):
+        self.bin_delta = 1  #smoothing parameter 
+        self.bin_range = np.arange(1, np.max(self.data), self.bin_delta)
+        self.bin_counts, _ = np.histogram(self.data, bins=self.bin_range)
+
+        #histogram density estimation 
+        #P = integral_R p(x) dx, where X is in R^3
+        #p(x) = K/(NxV), where K=number of points in region R
+        #N=total number of points, V=volume of region R
+
+        rnap_density_est = self.bin_counts/(sum(self.bin_counts) * self.bin_delta)
+        return rnap_density_est
+
+    def kernel(self):
+        #Gaussian kernel density estimator with smoothing parameter h
+        #sum N Guassians centered at each data point, parameterized by common std dev h
+
+        x_dim = 1  #dimension of x
+        h = 10 #standard deviation
+
+        rnap_density_support = np.arange(np.max(self.data))
+        rnap_density_est = 0
+        for i in range(np.sum(self.bin_counts)):
+            rnap_density_est += (1/(2*np.pi*h**2)**(x_dim/2.0))*np.exp(-(rnap_density_support - self.data[i])**2 / (2.0*h**2))
+        #end for
+        
+        rnap_density_est = rnap_density_est / np.sum(rnap_density_est)
+        return rnap_density_est
+
+if __name__ == "__main__":
+
+    kde = KDE()
+    est1 = kde.histogram()
+    est2 = kde.kernel()
+
+    plt.figure()
+    plt.plot(est1, '-b', label='histogram')
+    plt.plot(est2, '--r', label='gaussian kernel')
+    plt.title("RNA-seq density estimate based on negative binomial model")
+    plt.xlabel("read length, [base pairs]"); plt.ylabel("density"); plt.legend()
+    plt.show()
+```
+
+<img width="536" alt="image" src="https://github.com/user-attachments/assets/eff66466-d7a0-40f5-a9f9-8b225152ef7e">
+
+### Tangent portfolio optimization
+
+The objective of mean-variance analysis is to maximize the expected return of a portfolio for a given level of risk, as measured by the standard deviation of past returns. By varying the mixing proportions of each asset, we can achieve different risk–return tradeoffs.
+
+Example:
+
+```python
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from sklearn.neighbors import KDTree
+from pandas.plotting import scatter_matrix
+from scipy.spatial import ConvexHull
+
+import pandas_datareader.data as web
+from datetime import datetime
+import pytz
+
+STOCKS = ['SPY','LQD','TIP','GLD','MSFT']
+
+np.random.seed(42)    
+
+if __name__ == "__main__":
+
+    plt.close("all")
+    
+    #load data
+    #year, month, day, hour, minute, second, microsecond
+    start = datetime(2012, 1, 1, 0, 0, 0, 0, pytz.utc)
+    end = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)     
+    
+    data = pd.DataFrame()
+    series = []
+    for ticker in STOCKS:
+        price = web.DataReader(ticker, 'stooq', start, end)
+        series.append(price['Close'])
+
+    data = pd.concat(series, axis=1)
+    data.columns = STOCKS
+    data = data.dropna()
+
+    #plot data correlations
+    scatter_matrix(data, alpha=0.2, diagonal='kde')
+    plt.show()
+
+    #get current portfolio
+    cash = 10000
+    num_assets = np.size(STOCKS)
+    cur_value = (1e4-5e3)*np.random.rand(num_assets,1) + 5e3        
+    tot_value = np.sum(cur_value)
+    weights = cur_value.ravel()/float(tot_value)
+    
+    #compute portfolio risk
+    Sigma = data.cov().values
+    Corr = data.corr().values        
+    volatility = np.sqrt(np.dot(weights.T, np.dot(Sigma, weights)))
+    
+    plt.figure()
+    plt.title('Correlation Matrix')        
+    plt.imshow(Corr, cmap='gray')
+    plt.xticks(range(len(STOCKS)),data.columns)
+    plt.yticks(range(len(STOCKS)),data.columns)    
+    plt.colorbar()
+    plt.show()
+        
+    #generate random portfolio weights
+    num_trials = 1000
+    W = np.random.rand(num_trials, np.size(weights))    
+    W = W/np.sum(W,axis=1).reshape(num_trials,1)  #normalize
+    
+    pv = np.zeros(num_trials)   #portoflio value  w'v
+    ps = np.zeros(num_trials)   #portfolio sigma: sqrt(w'Sw)
+    
+    avg_price = data.mean().values
+    adj_price = avg_price
+    
+    for i in range(num_trials):
+        pv[i] = np.sum(adj_price * W[i,:])
+        ps[i] = np.sqrt(np.dot(W[i,:].T, np.dot(Sigma, W[i,:])))
+    
+    points = np.vstack((ps,pv)).T
+    hull = ConvexHull(points)
+    
+    plt.figure()
+    plt.scatter(ps, pv, marker='o', color='b', linewidth = 3.0, label = 'tangent portfolio')
+    plt.scatter(volatility, np.sum(adj_price * weights), marker = 's', color = 'r', linewidth = 3.0, label = 'current')
+    plt.plot(points[hull.vertices,0], points[hull.vertices,1], linewidth = 2.0)    
+    plt.title('expected return vs volatility')
+    plt.ylabel('expected price')
+    plt.xlabel('portfolio std dev')
+    plt.legend()
+    plt.grid(True)
+    plt.show()
+    
+    #query for nearest neighbor portfolio
+    knn = 5    
+    kdt = KDTree(points)
+    query_point = np.array([2, 115]).reshape(1,-1)
+    kdt_dist, kdt_idx = kdt.query(query_point,k=knn)
+    print("top-%d closest to query portfolios:" %knn)
+    print("values: ", pv[kdt_idx.ravel()])
+    print("sigmas: ", ps[kdt_idx.ravel()])
+```
+
+![image](https://github.com/user-attachments/assets/a5b8bec9-e916-42a2-8f5c-18a78bf2a679)
+![image](https://github.com/user-attachments/assets/9de9b10b-2f20-4306-b7c3-6d1bf43750d3)
+![image](https://github.com/user-attachments/assets/94a3e1ee-0361-49c5-a66e-085c1e703c6b)
+
+```
+top-5 closest to query portfolios:
+values:  [114.63340211 115.57074795 115.03231305 115.35426485 114.426947  ]
+sigmas:  [3.9943071  4.06987543 4.36175633 4.45779421 4.44304117]
+```
+
+## Structure learning
+
+Itinvolves discovering a graph's structure given relational data. The goal is to evaluate the probability of a graph \( G = (V, E) \) based on observed data \( D \). One challenge in this process is the exponential number of possible graphs. For example, in a directed graph with \( V \) vertices, there are \( O(2^{V(V-1)/2}) \) possible graphs due to the different edge directions. Since structure learning for general graphs is NP-hard, approximate methods are often used.
+
+### Approximate Methods
+Two common approaches for structure learning are:
+1. **Chow-Liu algorithm** for tree-based graphs
+2. **Inverse covariance estimation** for general graphs
+
+### Chow-Liu Algorithm
+The **Chow-Liu algorithm** focuses on tree structures. It uses a joint probability model for a tree \( T \), where:
+- \( p(x_t) \) is a node marginal.
+- \( p(x_s, x_t) \) is an edge marginal.
+
+To derive the Chow-Liu algorithm, the likelihood is expressed based on the mutual information \( I(x_s, x_t | \theta) \) between pairs of nodes. The tree topology that maximizes the log likelihood is computed via the maximum weight spanning tree, where edge weights are the pairwise mutual information values. The maximum spanning tree can be calculated using either Prim's or Kruskal's algorithm, both of which run in \( O(E \log V) \) time.
+
+### Inverse covariance estimation
+
+Identifying stock clusters helps one discover similar companies, which can be useful for comparative analysis or a pairs trading strategy. We can find similar clusters by estimating the inverse covariance (precision) matrix that can be used to construct a graph network of dependencies, using the fact that zeros in the precision matrix correspond to the absence of edges in the constructed graph.
+
+```python
+import numpy as np
+import pandas as pd
+from scipy import linalg
+
+from datetime import datetime
+import pytz
+
+from sklearn.datasets import make_sparse_spd_matrix
+from sklearn.covariance import GraphicalLassoCV, ledoit_wolf
+from sklearn.preprocessing import StandardScaler
+from sklearn import cluster, manifold
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+from matplotlib.collections import LineCollection
+
+import pandas_datareader.data as web
+
+np.random.seed(42)
+
+def main():
+        
+    #generate data (synthetic)
+    #num_samples = 60
+    #num_features = 20
+    #prec = make_sparse_spd_matrix(num_features, alpha=0.95, smallest_coef=0.4, largest_coef=0.7)
+    #cov = linalg.inv(prec)
+    #X = np.random.multivariate_normal(np.zeros(num_features), cov, size=num_samples)
+    #X = StandardScaler().fit_transform(X)    
+   
+    #generate data (actual)
+    STOCKS = {
+        'SPY': 'S&P500',
+        'LQD': 'Bond_Corp',
+        'TIP': 'Bond_Treas',
+        'GLD': 'Gold',
+        'MSFT': 'Microsoft',
+        'XOM':  'Exxon',
+        'AMZN': 'Amazon',
+        'BAC':  'BofA',
+        'NVS':  'Novartis'}
+      
+    symbols, names = np.array(list(STOCKS.items())).T
+    
+    #load data
+    #year, month, day, hour, minute, second, microsecond
+    start = datetime(2015, 1, 1, 0, 0, 0, 0, pytz.utc)
+    end = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)    
+
+    qopen, qclose = [], []
+    data_close, data_open = pd.DataFrame(), pd.DataFrame()    
+    for ticker in symbols:
+        price = web.DataReader(ticker, 'stooq', start, end)
+        qopen.append(price['Open'])
+        qclose.append(price['Close'])
+
+    data_open = pd.concat(qopen, axis=1)
+    data_open.columns = symbols
+    data_close = pd.concat(qclose, axis=1)
+    data_close.columns = symbols
+    
+    #per day variation in price for each symbol
+    variation = data_close - data_open
+    variation = variation.dropna()                        
+                
+    X = variation.values
+    X /= X.std(axis=0) #standardize to use correlations rather than covariance
+                
+    #estimate inverse covariance    
+    graph = GraphicalLassoCV()
+    graph.fit(X)
+    
+    gl_cov = graph.covariance_
+    gl_prec = graph.precision_
+    gl_alphas = graph.cv_alphas_
+    gl_scores = graph.cv_results_['mean_test_score']
+
+    plt.figure()        
+    sns.heatmap(gl_prec, xticklabels=names, yticklabels=names)
+    plt.xticks(rotation=45)
+    plt.yticks(rotation=45)    
+    plt.tight_layout()
+    plt.show()
+    
+    plt.figure()    
+    plt.plot(gl_alphas, gl_scores, marker='o', color='b', lw=2.0, label='GraphLassoCV')
+    plt.title("Graph Lasso Alpha Selection")
+    plt.xlabel("alpha")
+    plt.ylabel("score")
+    plt.legend()
+    plt.show()
+    
+    #cluster using affinity propagation
+    _, labels = cluster.affinity_propagation(gl_cov)
+    num_labels = np.max(labels)
+    
+    for i in range(num_labels+1):
+        print("Cluster %i: %s" %((i+1), ', '.join(names[labels==i])))
+    
+    #find a low dim embedding for visualization
+    node_model = manifold.LocallyLinearEmbedding(n_components=2, n_neighbors=6, eigen_solver='dense')
+    embedding = node_model.fit_transform(X.T).T
+    
+    #generate plots
+    plt.figure()
+    plt.clf()
+    ax = plt.axes([0.,0.,1.,1.])
+    plt.axis('off')
+    
+    partial_corr = gl_prec
+    d = 1 / np.sqrt(np.diag(partial_corr))    
+    non_zero = (np.abs(np.triu(partial_corr, k=1)) > 0.02)  #connectivity matrix
+    
+    #plot the nodes
+    plt.scatter(embedding[0], embedding[1], s = 100*d**2, c = labels, cmap = plt.cm.Spectral)
+    
+    #plot the edges
+    start_idx, end_idx = np.where(non_zero)
+    segments = [[embedding[:,start], embedding[:,stop]] for start, stop in zip(start_idx, end_idx)]
+    values = np.abs(partial_corr[non_zero])
+    lc = LineCollection(segments, zorder=0, cmap=plt.cm.hot_r, norm=plt.Normalize(0,0.7*values.max()))
+    lc.set_array(values)
+    lc.set_linewidths(2*values)
+    ax.add_collection(lc)
+    
+    #plot the labels
+    for index, (name, label, (x,y)) in enumerate(zip(names, labels, embedding.T)):
+        plt.text(x,y,name,size=12)
+    
+    plt.show()
+    
+if __name__ == "__main__":
+    main()
+```
+
+Graph lasso estimated precision matrix:
+
+<img width="395" alt="image" src="https://github.com/user-attachments/assets/70adfc70-41d4-4642-b0ee-ab7871c67a72">
+
+Stock clusters:
+
+<img width="419" alt="image" src="https://github.com/user-attachments/assets/c1fd7ea3-f814-4e20-bb91-1e6078003ae7">
+
+The edge values in the precision matrix greater than a threshold correspond to connected components from which we can compute stock clusters, as visualized on the right-hand side of the figure.
+
+## Simulated annealing
+
+It is a heuristic search method that allows for occasional transitions to less favorable states to escape local optima. We can formulate simulated annealing as an energy minimization problem with temperature parameter T, with which we can modify the energy landscape and select moves that are unfavorable in the short term but can lead to better longterm optima.
+
+## Genetic algorithms
+
+Genetic algorithms (GA) are inspired by and modeled after evolutionary biology. They consist of a population of (randomly initialized) individual genomes that are evaluated for their fitness with respect to a target. In genetic algorithms, two individuals combine and cross over their genome to produce an offspring. This crossover step is followed by a mutation by which individual bases can change according to a mutation rate. The resulting offspring are added to the population and scored according to their fitness level.
+
+---
+
+## Final ppt for the KB competition
+
+Today we finally submitted our presentation for the KB Future Finance AI competition - [here is a link to it on my google drive](https://docs.google.com/presentation/d/1OCQuaNCMoaUALETSD_e5TB9u6HMXqstV/edit?usp=sharing&ouid=109570875722775821585&rtpof=true&sd=true). The competition will be held on Tuesday. But there is a rehersal Monday evening where we are not sure what will happen but I will write about it tomorrow. Tomorrow (Monday) we will travel to Seoul mid-day.
+
+I am getting some nerves haha. Today I rewatched the competitions from previous years to see how the winners are picked and the points they get. Teams are graded on the spot, right after the ppt and there are two grades out of 10 - from a professional panel from KB, and from the audience. The competition is all in Korean, but I will share the link for completeness - [here](https://www.youtube.com/@kbaichallenge/videos)
+
+The winners from 2023 got the following points from professional judges + audience = total points
+
+```
+1. quantChat - 9.3 and 7.1 = 16.4
+2. Kbeep voice phishing - 9 and 7.3 = 16.3
+3. Ke ke ke - 8.2 and 8.0 = 16.2
+4. Kb Athena - 7.8 and 7.4 = 15.2
+5. KB DocSearch - 7.8 and 6.3 = 14.1
+6. Katering - 7 and 7.3 = 14.3
+
+Writer - 7.3 and 6.0 = 13.3
+Money laundering prevention  - 7.6 and 6.8 = 14.4
+Stock fund - 7.8 and 5.9 = 13.7
+KB-Dialect - 7.6 and 6.8 = 14.4
+```
+
+Only the top 6 teams get an award that is why there are 4 teams without ranking.
+
+That is all for today!
+
+See you tomorrow :)

From d12e8ce0195622d54ed76419b87f5cced28f9fd3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 26 Aug 2024 23:16:47 +0900
Subject: [PATCH 399/638] Create 2024-08-26-day-238.md

---
 _posts/2024-08-26-day-238.md | 43 ++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 _posts/2024-08-26-day-238.md

diff --git a/_posts/2024-08-26-day-238.md b/_posts/2024-08-26-day-238.md
new file mode 100644
index 000000000..99df68b21
--- /dev/null
+++ b/_posts/2024-08-26-day-238.md
@@ -0,0 +1,43 @@
+---
+layout: post
+title: (Day 238) Rehearsal for the KB AI competition
+categories: [theory,deep-learning]
+---
+
+# Hello :) Today is Day 238!
+A quick summary of today:
+* chapter 10 of the Machine Learning Algorithms in Depth book
+* rehearsal for the KB AI competition
+
+## Chapter 10: Fundamental DL algorithms
+
+This chapter introduced just basic concepts
+
+* Multilayer perceptron consists of multiple densely connected layers followed by a nonlinearity
+* Cross-entropy loss is used in classification tasks, while mean squared error loss is used in regression tasks
+* Neural networks are optimized via a backpropagation algorithm, based on the chain rule
+* Increasing model capacity to avoid underfitting can be achieved by changing the model architecture (e.g., increasing the number of layers and hidden units per layer)
+* Regularization to avoid overfitting can occur on multiple levels: weight decay, early stopping, and dropout
+* Convolutional neural nets work exceptionally well for image data and use convolution and pooling layers instead of vectorized matrix multiplications
+* Classic CNN architectures consist of a convolutional layer followed by ReLU nonlinear activation function and a max pooling layer
+* Pretrained CNNs can be used to extract feature vectors for images and used in applications such as image search
+* Recurrent neural nets are designed to process sequential data
+* Application of RNNs include language generation, sequence classification, and sequence translation.
+* Multi-input neural nets can be constructed by concatenating the feature vector representations from individual input branches.
+* Neural network optimizers include stochastic gradient descent, momentum, Nesterov momentum, AdaGrad, RMSProp, and Adam.
+
+## KB AI competition rehearsal
+
+![image](https://github.com/user-attachments/assets/5228a02b-d21a-4e49-a96d-4b4261c37ab4)
+
+Here is a picture of the room - Samsung Hall in Ehwa Women University in Seoul. 
+
+We arrived at the university at around 3.30pm and until the rehearsal we just walked around, and my project partner practiced his delivery. 
+
+The rehearsal started officilly at 5.30 and lasted until around 9pm. Each team did their presentation and my initial thoughts are that we are the most unique. While I have bias as a competitor, almost all the other projects revolve around using an LLM for some text assistance - from that point of view we are very unique and different in our topic, tools and overall project goal .
+
+We will see tomorrow ^^
+
+That is all for today!
+
+See you tomorrow :)

From 2863e2ba46385c7ce131396cc1ba77391db82ab2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 27 Aug 2024 18:52:37 +0900
Subject: [PATCH 400/638] Create 2024-08-27-day-239.md

---
 _posts/2024-08-27-day-239.md | 76 ++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 _posts/2024-08-27-day-239.md

diff --git a/_posts/2024-08-27-day-239.md b/_posts/2024-08-27-day-239.md
new file mode 100644
index 000000000..38dcf34ac
--- /dev/null
+++ b/_posts/2024-08-27-day-239.md
@@ -0,0 +1,76 @@
+---
+layout: post
+title: (Day 239) 7th place at the KB future finance competition 🥳
+categories: [deep-learning,theory]
+---
+
+# Hello :) Today is Day 239!
+A quick summary of today:
+* the kb project is finally over
+* finishing Machine Learning Algorithms in Depth
+
+![image](https://github.com/user-attachments/assets/2a82ef86-bbad-4718-b0ec-90da5caaa8cc)
+
+We ended up at 7th place in the comptition 🥳. I did not take many pictures. Just the above one when we were entering the hall. 
+
+To close out the project, I [un-archived the repo ](https://github.com/divakaivan/kb_project) and added the latest code and updated some of the pictures like:
+
+the architecture, which includes kakao talk customer messaging as the last step
+
+![image](https://github.com/user-attachments/assets/ce7a88cb-b71b-4f8c-99b1-28779624314e)
+
+And the monitoring dashboard:
+
+![image](https://github.com/user-attachments/assets/f189c76a-5eb9-4702-9d99-1004008c273d)
+
+## Chapter 11 (last chapter) Advanced deep learning algorithms of the ML algs in depth book
+
+It was similar to the previous chapter and it just briefly introduced some of the popular DL architecture types over the years, ending with the Transformer and GNNs.
+
+Out of the chapter, I got an idea to try to use a VAE for time-series analysis.
+
+- Autoencoders are unsupervised neural networks trained to reconstruct the input. The bottleneck layer of the autoencoder can be used for dimensionality reduction or feature extraction
+
+- A variational autoencoder consists of an encoder that takes in data as input, transforming it into a latent representation, and a decoder that takes a latent representation, returning a reconstruction.
+
+### VAE Anomaly Detection in Time Series
+
+This section discussed using a VAE for anomaly detection in time series data. The model architecture uses LSTM layers to process time series signals. The input consists of *n* signals, and the output is the log probability of observing a signal under normal (non-anomalous) conditions, defined by parameters μ (mean) and σ (variance).
+
+  - Training: The model is trained unsupervised on non-anomalous data.
+  - Anomaly Detection: When an anomaly occurs, the log likelihood of the signal drops, indicating the anomaly.
+  - Assumptions: Gaussian likelihood is assumed, where each sensor has two degrees of freedom (μ, σ) for anomaly representation.
+  - Embedding: Independent input signals are embedded into a joint latent space using the VAE’s sampling layer, structured as a Gaussian approximating N(0,1).
+  - Objective: The objective function maximizes the log likelihood across sensors and ensures the embedding space approximates N(0,1)
+
+- Amortized variational inference is the idea that instead of optimizing a set of free parameters, we can introduce a parameterized function that maps from observation space to the parameters of the approximate posterior distribution.
+
+- Mixture density networks are mixture models in which the parameters, such as means, covariances, and mixture proportions, are learned by a neural network
+
+- Attention allows a model to adaptively focus on different parts of the input by adjusting the attention weights
+
+- A transformer is a Seq2Seq model that uses attention in the encoder as well as the decoder. Self-attention is the key component of transformer architecture
+
+- Graph neural networks are neural models that capture dependence relationships via passing messages between the nodes of the graph
+
+## Overall 
+
+Great book overall. Great for beginners if they have math and stats knowledge. Otherwise some of the formulas and derivations might be scary. Nonetheless, it is a great source of more in-depth information about each othe algorithms presented, covering supervised, unsupervides, semi-supervised, and DL. 
+
+## What's next for me?
+
+Well, tomorrow (Wednesday) I will go to a career fair here in Seoul so I will give some info on that as well.
+
+But in terms of things to study, at the moment I think I will start with one of the below courses in Coursera by Duke University professors:
+
+- [LLMOps](https://www.coursera.org/specializations/large-language-model-operations)
+- [Cloud computing at scale](https://www.coursera.org/specializations/building-cloud-computing-solutions-at-scale)
+- [MLOps](https://www.coursera.org/specializations/mlops-machine-learning-duke)
+- [Data engineering](https://www.coursera.org/specializations/python-data-engineering)
+
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From bbeb78444cf94f01824e4d8048f244a3e03cc4e6 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 29 Aug 2024 00:36:14 +0900
Subject: [PATCH 401/638] Create 2024-08-28-day-240.md

---
 _posts/2024-08-28-day-240.md | 88 ++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 _posts/2024-08-28-day-240.md

diff --git a/_posts/2024-08-28-day-240.md b/_posts/2024-08-28-day-240.md
new file mode 100644
index 000000000..4515c7294
--- /dev/null
+++ b/_posts/2024-08-28-day-240.md
@@ -0,0 +1,88 @@
+---
+layout: post
+title: (Day 240) Example for transitioning from Docker to K8s
+categories: [mlops,theory]
+---
+
+# Hello :) Today is Day 240!
+A quick summary of today:
+* atteneded a career fair in Seoul
+* found a [short and nice article](https://www.docker.com/static/test-lab-docker-kubernetes-admin-magazine-article.pdf) about going from just Docker to a container orchestrator (kubernetes)
+* looking for books
+
+## Firstly, about the career fair
+
+![image](https://github.com/user-attachments/assets/870c858e-34bd-400a-930f-63773027df04)
+<img width="319" alt="image" src="https://github.com/user-attachments/assets/e1f1d80b-7c20-4a33-9ca3-b810faf10440">
+
+It was advertised as the biggest career event for foreigners in Korea, and when I got there it was 70% only for Korean people ... It is fair to say I was quite disappointed. 
+
+## Morgan Stanley
+
+Actually, on the topic of career events, I have been attending some online sessions run by Morgan Stanley in the UK
+
+<img width="510" alt="image" src="https://github.com/user-attachments/assets/30570384-5811-4a7b-8441-9370254d1531">
+
+Above is the schedule. So far 3 sessions have passed and they were all quite good - people from HR coming out, current interns, placement students, graduate students coming out and just talking and sharing about life at MS (as they call Morgan Stanley). 
+
+## Docker and Kubernetes
+
+* Nodes: the worker machines (either physical or virtual) in a Kubernetes cluster that run the applications. Each node contains the necessary components to manage and run containers, such as the container runtime (e.g., Docker)
+
+* Pods: the smallest deployable units in Kubernetes, consisting of one or more containers that share resources like storage and networking. Pods run on nodes and are the building blocks for applications in a Kubernetes cluster
+
+_Do I Need cri-dockerd?_
+
+A critical aspect of running Docker and Kubernetes together is understanding container runtimes. The article explains the deprecation of dockershim in Kubernetes and introduces cri-dockerd, an adapter ensuring compatibility between Docker Engine and Kubernetes. This section is crucial for those who want to continue using Docker Engine with Kubernetes.
+
+It also talked about the benefits of using the local Kubernetes cluster bundled with Docker Desktop. It allows developers to test their deployments locally before pushing them to production environments, such as AWS EKS.
+
+_The practical example_
+
+The code is all in the article, but simply put it was converting a simple app that uses nginx, redis and a flask app that is firstly built with docker-compose, and then converting that to using k8s (kubernetes). And the main change is enabling k8s in docker desktop (I did not know that is an option so next time I need one I will use the docker desktop provided one), and creating a bunch of yaml files for service and deployment. 
+
+As I have used Docker many times now and am quite comfortable with it, this was a nice and smooth intro to k8s and its usage. However, I do feel this taps even more into the DevOps role into a company, but still it is something good to be aware of. 
+
+## Lastly, books
+
+After the career fair (in Seoul), I went to one of the biggest book stores looking for a cool new book. As it is Korea, the availability of Western book is understandably limited, and even if a book was available it's price would be 2x, 3x or even more than the original. That being said, I almost bought [*Mathematics for Machine Learning*](https://mml-book.github.io/). I know it is freely available and provided by the authors, but having a physical copy feels so good. I did not end up buying it as understandably the price was too much. 
+
+And on my train ride back home, I looked at available books on O'Reilly. I found some good next reads:
+
+[**Big Data on K8s**](https://learning.oreilly.com/library/view/big-data-on/9781835462140/)
+
+* Leverage Kubernetes in a cloud environment to integrate seamlessly with a variety of tools
+* Explore best practices for optimizing the performance of big data pipelines
+* Build end-to-end data pipelines and discover real-world use cases using popular tools like Spark, Airflow, and Kafka
+
+[**Databricks Certified Associate Developer for Apache Spark Using Python**](https://learning.oreilly.com/library/view/databricks-certified-associate/9781804619780/)
+
+* Understand the fundamentals of Apache Spark to help you design robust and fast Spark applications
+* Delve into various data manipulation components for each phase of your data engineering project
+* Prepare for the certification exam with sample questions and mock exams, and get closer to your goal
+
+[**High Performance Spark**](https://learning.oreilly.com/library/view/high-performance-spark/9781491943199/)
+
+* How Spark SQL’s new interfaces improve performance over SQL’s RDD data structure
+* The choice between data joins in Core Spark and Spark SQL
+* Techniques for getting the most out of standard RDD transformations
+* How to work around performance issues in Spark’s key/value pair paradigm
+* Writing high-performance Spark code without Scala or the JVM
+* How to test for functionality and performance when applying suggested improvements
+* Using Spark MLlib and Spark ML machine learning libraries
+
+Actually there is a 2nd edition being written so I am saving this one just to remind myself that later this year a newer version will come out
+
+[**Streaming Databases**](https://learning.oreilly.com/library/view/streaming-databases/9781098154820/)
+
+* Explore stream processing and streaming databases
+* Learn how to build a real-time solution with a streaming database
+* Understand how to construct materialized views from any number of streams
+* Learn how to serve synchronous and asynchronous data
+* Get started building low-complexity streaming solutions with minimal setup
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From b23b750f87258358871a1ad7fc504899338c1a05 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 29 Aug 2024 23:34:16 +0900
Subject: [PATCH 402/638] Create 2024-08-29-day-241.md

---
 _posts/2024-08-29-day-241.md | 141 +++++++++++++++++++++++++++++++++++
 1 file changed, 141 insertions(+)
 create mode 100644 _posts/2024-08-29-day-241.md

diff --git a/_posts/2024-08-29-day-241.md b/_posts/2024-08-29-day-241.md
new file mode 100644
index 000000000..9920ed46a
--- /dev/null
+++ b/_posts/2024-08-29-day-241.md
@@ -0,0 +1,141 @@
+---
+layout: post
+title: (Day 241) Techniques for improving RAG pipes
+categories: [mlops,applying-knowledge,nlp]
+---
+
+# Hello :) Today is Day 241!
+A quick summary of today:
+* ways to improve RAG pipelines
+* sample project for the LLM zoomcamp
+
+
+## [Techniques for improving RAG pipes](https://github.com/DataTalksClub/llm-zoomcamp/tree/main/06-best-practices)
+
+* small-to-big chunk retrieval - use small chunks in the embedding stage and large chunks in the answering stage (helps tackling the problem of choosing the right embedding size)
+* leverage document metadata - adding doc name and path can be useful and also can allow the user to see the original document
+* hybrid search (vector-based and keyword-based search) - vector search looks for the semantically closest vectors in the embedding space, while keyword-search is looking for lexical similarity
+
+<img width="914" alt="image" src="https://github.com/user-attachments/assets/38891c24-93bf-488c-8f33-8be5fa580057">
+
+A simple formular for this could be: `hybrid_score = (1 - alpha) * match_score  + alpha * vec_score`
+
+* user query re-writing - rephrase user Qs in a more structured way
+* retrieved document re-ranking - can be done using an llm, or by using specific relevance score metrics like Normalized Discounted Cumulative Gain (NDCG), MAP@5, reciprocal rank fusion (RRF)
+
+<img width="1356" alt="image" src="https://github.com/user-attachments/assets/e19cf3bc-f4ad-445c-9c5f-5e8ae631a759">
+
+## [LLM zoomcamp project example](https://github.com/DataTalksClub/llm-zoomcamp/tree/main/07-project-example)
+
+Alexey, the main instructor and creator of DataTalksClub, decided to upload a series of videos showcasing how to create a sample project for the LLM zoomcamp. There are in total 7 videos and it is great as it gives others, and me as well, a clearer idea of what he is looking for in the final project for the course. 
+
+1. Creating a dataset
+
+First is creating a sample dataset using chatgpt (creating my own dataset is not required, it is just used as an example, and also to show chatgpt's abilities)
+
+<img width="835" alt="image" src="https://github.com/user-attachments/assets/ee13bf53-f4e6-4b02-92a6-4bad7ab6025d">
+
+2. Setting up a simple RAG flow
+
+3. Evaluating the retrieval
+
+For each exercise, chatGPT can be used to create sample questions. For instance, for push ups chatgpt creates 5 questions related to: form, back position, repetition, things like that. So these questions are then used to evaluate the retrieval - for a given question do we retrieve the relevant exercise and its info from our database. 
+
+Two evaluation approaches were presented: a basic approach using minsearch (a custom implementation for measuring similarity between vectors) and a more advanced approach involving hyperparameter tuning. In the advanced approach, focus was on identifying the most relevant parts of a document. As expected, exercise_name, body_part, and muscle_groups_activated were found to be the most relevant features, while type_of_activity, type_of_equipment, type, and instructions were less significant.
+
+The basic approach - using minsearch without any boosting - gave the following metrics:
+
+* Hit rate: 94%
+* MRR: 82%
+
+The improved version (with tuned boosting):
+
+* Hit rate: 94%
+* MRR: 90%
+
+4. Evaluating RAG
+
+The above generated 5 questions for each exercise are now asked through our RAG system. Here we are using LLM-as-a-judge to judge whether the answer generated by our RAG is relevant to the question (another option is cosine similarity if we had an original/actual answer). Here is a sample prompt for the LLM to judge:
+
+```
+You are an expert evaluator for a RAG system.
+Your task is to analyze the relevance of the generated answer to the given question.
+Based on the relevance of the generated answer, you will classify it as "NON_RELEVANT," "PARTLY_RELEVANT," or "RELEVANT."
+
+Here is the data for evaluation:
+
+Question: What is the starting position for doing push-ups?
+Generated Answer: The starting position for doing push-ups is to begin in a high plank position with your hands under your shoulders.
+
+Please analyze the content and context of the generated answer in relation to the question and provide your evaluation in parsable JSON without using code blocks:
+
+{
+   "Relevance": "NON_RELEVANT" | "PARTLY_RELEVANT" | "RELEVANT",
+   "Explanation": "[Provide a brief explanation for your evaluation]"
+}
+```
+
+The above particular case resulted in :
+
+```json
+{
+   "Relevance": "RELEVANT",
+   "Explanation": "The generated answer accurately describes the starting position for doing push-ups, which directly addresses the question asked."
+}
+```
+
+On a small dataset for validation with around 200 exercises, and for each a question and answer was generated. Among those, after using llm-as-a-judge, the results of the RAG evaluation are:
+
+* 167 (83%) RELEVANT
+* 30 (15%) PARTLY_RELEVANT
+* 3 (1.5%) NON_RELEVANT
+
+And such stats can be shown in a Grafana monitoring dashboard:
+
+![image](https://github.com/user-attachments/assets/2385e78f-0d54-4e65-866e-3831b61fde8b)
+
+---
+
+This got me thinking about whether I can make a RAG project related to this very blog. The problem is that I have a lot of pictures, and for many days I hand-wrote my notes when covering a course or a book, so I would have to pass those images through an LLM to get the text. But I cannot pass all images because I would get image info that is irrelevant, or maybe not describable, and it may cost money if I want the highest quality image descriptions - i.e. using gpt's api. Then again, I can just use the text part from my blogs and it may work fine. So it is just about finding what I want to do for this project.
+
+## Interview for a university course
+
+Today I was in a meeting with other scholarship students in my university, and our teacher (who is responsible for the scholarship students) mentioned that in the ['Hannam Design Factory'](http://df.hannam.ac.kr/kor/sub01/menu_02.html) (the page is all in english) department is looking for students for their projects. The way that particular department operates is it runs projects which server as 6 or 9 credit courses throughout the year, and the idea is for the students to work on some realistic project assigned from a company. Apparently they look for people with an engineering background, but I said that while I am not an engineer, if possible I can help with AI. Later, the teacher messaged me letting me know that I should go for an interview at the department tomorrow morning. This is the 1st time interviewing to take a university course haha. But jokes aside, I believe there is an interview as I would be part of some project by some company and they just want to make sure I understand the assignment and have the skills to do it. I will update on this tomorrow ^^
+
+## Big Data on Kubernetes Chapter 2: K8s architecture
+
+1. Control Plane Components:
+   - API Server: The entry point for the control plane, managing communication and maintaining the desired state of the cluster
+   - etcd: The distributed key-value store that holds cluster data, ensuring consistency across the system
+   - Controller Manager: Responsible for ensuring that the cluster state matches the desired state by controlling various controllers
+   - Schedulers: These assign workloads to nodes based on resource availability
+
+2. Node Components:
+   - kubelet: the agent running on each node that communicates with the control plane and ensures containers are running as expected
+   - kube-proxy: manages networking for Pods, handling communication between the nodes and the control plane
+
+3. API Resources:
+   - Pods: the smallest deployable units in Kubernetes, encapsulating containers with shared storage and networking
+   - Deployments: enable declarative management of Pod replicas, ensuring scalability and self-healing of applications
+   - StatefulSets: manage stateful applications by ensuring persistent storage and stable networking
+   - Jobs: Allow batch processing workloads to run to completion
+   - Services: Provide stable networking and enable loose coupling between Pods, abstracting away direct Pod access
+
+4. Ingress Resources and Controllers: Ingress resources define external access rules for cluster services, while ingress controllers handle the actual traffic routing. The Gateway API provides a new, centralized method for managing ingress configurations
+
+5. Storage in Kubernetes:
+   - PersistentVolumes (PVs): provide portable, network-attached storage that can be dynamically provisioned
+   - PersistentVolumeClaims (PVCs): request and bind to PVs for efficient storage management
+   - StorageClasses: define different classes of storage to cater to varying performance and availability requirements
+
+6. ConfigMaps and Secrets:
+   - ConfigMaps: inject configuration data into Pods in a decoupled manner, allowing for flexible application configuration
+   - Secrets: Securely manage sensitive data, such as passwords or API keys, and inject them into Pods while keeping them separate from the application code
+
+
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From e318dcf564102f1e2103f8f10a3439419c0b80df Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 30 Aug 2024 22:43:03 +0900
Subject: [PATCH 403/638] Create 2024-08-30-day-242.md

---
 _posts/2024-08-30-day-242.md | 399 +++++++++++++++++++++++++++++++++++
 1 file changed, 399 insertions(+)
 create mode 100644 _posts/2024-08-30-day-242.md

diff --git a/_posts/2024-08-30-day-242.md b/_posts/2024-08-30-day-242.md
new file mode 100644
index 000000000..f2a031ddc
--- /dev/null
+++ b/_posts/2024-08-30-day-242.md
@@ -0,0 +1,399 @@
+---
+layout: post
+title: (Day 242) PySpark day
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 242!
+A quick summary of today:
+* reading about 'the modern data stack'
+* successful interview for the Hannam Design Factory
+
+
+## Big Data on K8s: Chapter 4 - The Modern Data Stack
+
+### The Lambda architecture
+
+* Batch layer: Responsible for managing the master dataset. This layer ingests and processes data in bulk at regular intervals, typically every 24 hours. Once processed, the batch views are considered immutable and stored.
+* Speed layer: Responsible for recent data that has not yet been processed by the batch layer. This layer processes data in real time as it arrives to provide low-latency views.
+* Serving layer: Responsible for responding to queries by merging views from both the batch and speed layers.
+
+![image](https://github.com/user-attachments/assets/01d19995-7004-4258-bd56-0660ccd7e372)
+
+### The Kappa architecture
+
+The Kappa architecture handles all data processing through a single stream processing pathway. The key components include the following:
+
+* Stream processing layer: Responsible for ingesting and processing all data as streams. This layer handles both historical data (via replay of logs/files) as well as new incoming data.
+* Serving layer: Responsible for responding to queries by accessing views produced by the stream processing layer.
+
+![image](https://github.com/user-attachments/assets/2fc2bd72-a9cc-4591-b349-5ad4838f181d)
+
+### Lambda vs Kappa
+
+| Feature                                        | Lambda Architecture                           | Kappa Architecture                          |
+|------------------------------------------------|-----------------------------------------------|---------------------------------------------|
+| **Complexity**                                 | Manages separate batch and real-time systems  | Consolidates processing through streams     |
+| **Reprocessing**                               | Reprocesses historical batches                | Relies on stream replay and algorithms      |
+| **Latency**                                    | Lower latencies for recent data in the speed layer | Same latency for all data              |
+| **Throughput**                                 | Leverages batch engines optimized for throughput | Processes all data as streams           |
+
+In practice, modern data architectures often blend these approaches. For example, a batch layer on Lambda may run only weekly or monthly while real-time streams fill the gap. Kappa may leverage small batches within its streams to optimize throughput. The core ideas around balancing latency, throughput, and reprocessing are shared.
+
+For data lakes, Lambda provides a proven blueprint while Kappa offers a powerful alternative. While some may argue that Kappa offers a simpler operation, it is hard to implement and its costs can grow rapidly with scale. Another advantage of Lambda is that it is fully adaptable. We can implement only the batch layer if no data streaming is necessary (or financially viable).
+
+Data lake builders should understand the key principles of each to craft optimal architectures aligned to their businesses, analytics, and operational needs. By leveraging the scale and agility of cloud infrastructure, modern data lakes can implement these patterns to handle today’s data volumes and power advanced analytics.
+
+### Data lake design for big data
+
+#### Data Warehouses
+Data warehouses have been essential for business intelligence by storing structured data from various sources optimized for reporting. However, they face limitations such as difficulties with unstructured data, rigid schema-on-write designs, and latency due to batch processing. This makes them less flexible in handling large, diverse, and real-time data, leading to the rise of more adaptive architectures.
+
+#### The Rise of Big Data and Data Lakes
+Data lakes address the limitations of traditional data warehouses by enabling large-scale storage of diverse data types (structured, unstructured, and semi-structured) in native formats. They use a schema-on-read approach, allowing flexibility in data usage. However, data lakes face challenges like governance issues, complex data preparation, and inefficiencies in handling data modifications and schema management.
+
+#### The Rise of the Data Lakehouse
+Data lakehouses combine the strengths of both data warehouses and data lakes. They support diverse data types, offer SQL analytics, and ensure ACID transactions. Lakehouses allow schema enforcement, data updates, and querying within a unified system, streamlining the analytics life cycle and improving agility.
+
+#### Lakehouse Storage Layers
+- Bronze Layer: Raw data stored as received, acting as the source of truth.
+- Silver Layer: Refined and standardized data, ready for analysis.
+- Gold Layer: Aggregated and curated data models for business intelligence.
+- Landing Zone (optional): Prepares raw data before entering the bronze layer.
+
+### Implementing the lakehouse architecture
+
+Below is a possible implementation of a data lakehouse architecture in a Lambda design
+
+![image](https://github.com/user-attachments/assets/94462831-92a4-4c0b-8188-f56de7ba5d41)
+
+The first group on the left represents the possible data sources to work with this architecture. One of the key advantages of this approach is its ability to ingest and store data from a wide variety of sources and in diverse formats. As shown in the diagram, the data lake can connect to and integrate structured data from databases as well as unstructured data such as API responses, images, videos, XML, and text files. This schema-on-read approach allows the raw data to be loaded quickly without needing upfront modeling, making the architecture highly scalable. When analysis is required, the lakehouse layer enables querying across all these datasets in one place using schema-on-query. This makes it simpler to integrate data from disparate sources to gain new insights. The separation of loading from analysis also enables iterative analytics as a new understanding of the data emerges. Overall, the modern data lakehouse is optimized for rapidly landing multi-structured and multi-sourced data while also empowering users to analyze it flexibly.
+
+## Chapter 5 - Big Data Processing with Apache Spark
+
+![image](https://github.com/user-attachments/assets/8f17dc48-03a4-4503-a3c9-999aa68e3f25)
+
+The centerpiece that coordinates the Spark application is called the driver program. The driver program instantiates a SparkSession object that integrates directly with a Spark context. The Spark Context connects to a cluster manager that can provision resources across a computing cluster. When running locally, an embedded cluster manager runs within the same Java Virtual Machine (JVM) as the driver program. But in production, Spark should be configured to use a standalone cluster resource manager such as Yarn or Mesos.
+
+The cluster manager is responsible for allocating computational resources and isolating computations on the cluster. When the driver program requests resources, the cluster manager launches Spark executors to perform the required computations.
+
+Spark executors are processes launched on worker nodes in the cluster by the cluster manager. They run computations and store data for the Spark application. Each application has its own executors that stay up for the duration of the whole application and run tasks in multiple threads. Spark executes code snippets called tasks to perform distributed data processing.
+
+A Spark job triggers the execution of a Spark program. This gets divided into smaller sets of tasks called stages that depend on each other.
+
+### The DataFrame API and the Spark SQL API
+
+Spark provides different APIs built on top of the core RDD API (the native, low-level Spark language) to make it easier to develop distributed data processing applications. The two most popular higher-level APIs are the DataFrame API and the Spark SQL API.
+
+The DataFrames API provides a domain-specific language to manipulate distributed datasets organized into named columns. Conceptually, it is equivalent to a table in a relational database or a DataFrame in Python pandas, but with richer optimizations under the hood. The DataFrames API enables users to abstract data processing operations behind domain-specific terminology such as grouping and joining instead of thinking in map and reduce operations.
+
+The Spark SQL API builds further on top of the DataFrames API by exposing Spark SQL, a Spark module for structured data processing. Spark SQL allows users to run SQL queries against DataFrames to filter or aggregate data. The SQL queries get optimized and translated into native Spark code to be executed. This makes it easy for users familiar with SQL to run ad hoc queries against data.
+
+Both APIs rely on the Catalyst optimizer, which leverages advanced programming techniques such as predicate pushdown, projection pruning, and a variety of join optimizations to build efficient query plans before execution. This differentiates Spark from other distributed data processing frameworks by optimizing queries based on business logic instead of on hardware considerations.
+
+When working with Spark SQL and the DataFrames API, it is important to understand some key concepts that allow Spark to run fast, optimized data processing. These concepts are transformations, actions, lazy evaluation, and data partitioning.
+
+* Transformations define computations that will be done, while actions trigger the actual execution of those transformations
+* While transformations describe operations on DataFrames, actions actually execute the computation and return results
+* Lazy evaluation is a key technique that allows Apache Spark to run efficiently. As mentioned previously, when you apply transformations to a DataFrame, no actual computation happens at that time. Instead, Spark internally records each transformation as an operation to apply to the data. The actual execution is deferred until an action is called
+* Data partitioning - Spark’s speed comes from its ability to distribute data processing across a cluster. To enable parallel processing, Spark breaks up data into independent partitions that can be processed in parallel on different nodes in the cluster. When you read data into a Spark DataFrame or RDD, the data is divided into logical partitions. On a cluster, Spark will then schedule task execution so that partitions run in parallel on different nodes. Each node may process multiple partitions. This allows the overall job to process data much faster than if run sequentially on a single node. Understanding data partitioning in Spark is key to understanding the differences between narrow and wide transformations.
+  * Narrow transformations are operations that can be performed on each partition independently without any data shuffling across nodes. Examples include map, filter, and other per-record transformations. These allow parallel processing without network traffic overhead
+  * Wide transformations require data to be shuffled between partitions and nodes. Examples include groupBy aggregations, joins, sorts, and window functions. These involve either combining data from multiple partitions or repartitioning data based on a key
+
+## Chapter 6 - Building Pipelines with Apache Airflow
+
+### Airflow architecture
+
+Airflow is composed of different components that fit together to provide a scalable and reliable orchestration platform for data pipelines.
+
+At a high level, Airflow has the following:
+
+* a metadata database that stores state for DAGs, task instances, XComs, and so on
+* a web server that serves the Airflow UI
+* a scheduler that handles triggering DAGs and task instances
+* executors that run task instances
+* workers that execute tasks
+* other components, such as the CLI
+
+![image](https://github.com/user-attachments/assets/a1219d2b-d58b-43be-86ef-d99044508934)
+
+Airflow relies heavily on the metadata database as the source of truth for state. The web server, scheduler, and worker processes talk to this database. When you look at the Airflow UI, underneath, it simply queries this database to get info to display.
+
+The metadata database is also used to enforce certain constraints. For example, the scheduler uses database locks when examining task instances to determine what to schedule next. This prevents race conditions between multiple scheduler processes.
+
+### distributed architecture
+
+* Separation of concerns: Each component focuses on a specific job. The scheduler handles examining DAGs and scheduling. The worker runs task instances. This separation of concerns keeps components simple and maintainable.
+* Scalability: Components such as the scheduler, worker, and database can be easily scaled out. Run multiple schedulers or workers as your workload grows. Leverage a hosted database for automatic scaling.
+* Reliability: If one scheduler or worker dies, there is no overall outage since components are decoupled. The single source of truth in the database also provides consistency across Airflow.
+* Extensibility: You can swap out certain components, such as the executor or queueing service.
+
+### Example
+
+Using [astro](https://docs.astronomer.io/astro/cli/install-cli) an example pipeline is created
+
+Init a new project with `astro dev init`
+
+Start it with `astro dev start`
+
+And an airflow webserver and a postgres db are started
+
+<img width="483" alt="image" src="https://github.com/user-attachments/assets/06476a98-e8ca-4d58-9855-ea3751e6c828">
+
+Sample pipeline:
+
+```python
+from airflow.decorators import task, dag
+from airflow.operators.dummy import DummyOperator
+from airflow.operators.bash import BashOperator
+from datetime import datetime
+import requests
+import pandas as pd
+
+default_args = {
+    'owner': 'Ney',
+    'start_date': datetime(2024, 2, 12)
+}
+
+@dag(
+        default_args=default_args, 
+        schedule_interval="@once", 
+        description="Simple Pipeline with Titanic", 
+        catchup=False, 
+        tags=['Titanic']
+)
+def titanic_processing():
+
+    # Task Definition
+    start = DummyOperator(task_id='start')
+
+    @task
+    def first_task():
+        print("And so, it begins!")
+
+    @task
+    def download_data():
+        destination = "/tmp/titanic.csv"
+        response = requests.get(
+            "https://raw.githubusercontent.com/neylsoncrepalde/titanic_data_with_semicolon/main/titanic.csv", 
+            stream=True
+        )
+        with open(destination, mode="wb") as file:
+            file.write(response.content)
+        return destination
+
+    @task
+    def analyze_survivors(source):
+        df = pd.read_csv(source, sep=";")
+        res = df.loc[df.Survived == 1, "Survived"].sum()
+        print(res)
+
+    @task
+    def survivors_sex(source):
+        df = pd.read_csv(source, sep=";")
+        res = df.loc[df.Survived == 1, ["Survived", "Sex"]].groupby("Sex").count()
+        print(res)
+
+    last = BashOperator(
+        task_id="last_task",
+        bash_command='echo "This is the last task performed with Bash."',
+    )
+
+    end = DummyOperator(task_id='end')
+
+    # Orchestration
+    first = first_task()
+    downloaded = download_data()
+    start >> first >> downloaded
+    surv_count = analyze_survivors(downloaded)
+    surv_sex = survivors_sex(downloaded)
+
+    [surv_count, surv_sex] >> last >> end
+
+execution = titanic_processing()
+```
+
+<img width="1500" alt="image" src="https://github.com/user-attachments/assets/c3f71c8a-2c74-45b3-89f9-4c8e71b0eff3">
+
+After running it:
+
+<img width="1502" alt="image" src="https://github.com/user-attachments/assets/dcf6aa72-a786-46ab-8e2d-cfe00e0ae78a">
+
+
+## Real-time events and data ingestion using Kafka
+
+### Kafka's architecture
+
+Kafka's architecture is distributed and consists of brokers, producers, consumers, topics, partitions, and replicas. Producers publish messages to topics, which are managed by brokers. Consumers subscribe to topics to process messages. Kafka uses Zookeeper for cluster management, including broker coordination and leader election.
+
+Kafka follows a publish-subscribe (PubSub) design where producers write data to topics, and consumers read from them. Topics are split into partitions to allow parallelism and scalability, with partitions replicated across brokers for fault tolerance. Consumers are organized into groups, ensuring each message is processed by one consumer per group. Kafka guarantees ordered, at-least-once message delivery within a partition.
+
+Producers batch messages for efficiency and durability, choosing acknowledgment guarantees. Consumers track their reading position using offsets, which allow them to resume from the correct place after failures. Offsets also enable multiple consumers to read the same partition independently.
+
+Kafka can deliver exactly-once semantics by tightly integrating offset tracking with event delivery, ensuring each record is processed exactly once. This is achieved through careful coordination between storage, processing, and Kafka's infrastructure.
+
+![image](https://github.com/user-attachments/assets/855cb2a0-7559-4fbc-a20d-95e402a4cc24)
+
+### Kafka Connect
+
+Kafka Connect is a framework within the Apache Kafka ecosystem that facilitates the integration of Kafka with external systems, such as databases, message queues, and storage platforms. It simplifies the process of streaming data between Kafka and these systems by providing a standardized way to move large volumes of data into and out of Kafka.
+
+1. Scalability and Distributed Nature:
+   - Kafka Connect is designed to be distributed and scalable. It can run as a standalone process or as a distributed cluster, enabling it to handle large-scale data integration tasks.
+   - In distributed mode, Kafka Connect automatically balances the data load and handles connector tasks across multiple nodes.
+
+2. Source and Sink Connectors:
+   - Source Connectors: Import data from external systems into Kafka topics. For example, a source connector could pull data from a database and publish it to Kafka.
+   - Sink Connectors: Export data from Kafka topics to external systems. For example, a sink connector could send data from Kafka to a data warehouse or file system.
+
+3. Schema Management:
+   - Kafka Connect supports schema management, allowing for the validation and transformation of data as it moves between systems. This ensures data consistency and format are maintained across the pipeline.
+
+4. Fault Tolerance and Resilience:
+   - Kafka Connect is designed for high availability. In distributed mode, it can automatically recover from failures by restarting tasks on other nodes.
+   - It also supports offset management, allowing connectors to resume data streaming from where they left off in case of failures.
+
+5. Configuration and Management:
+   - Kafka Connect simplifies configuration and management through its REST API. Administrators can manage connectors, tasks, and workers programmatically, using simple HTTP requests.
+
+6. Transforms:
+   - Kafka Connect supports Single Message Transforms (SMTs), which are lightweight transformations applied to records as they flow through connectors. SMTs allow for modifying, filtering, or enriching data before it is published to Kafka or written to an external system.
+
+7. Compatibility with Confluent Hub:
+   - A wide range of pre-built connectors are available on the Confluent Hub, an online repository of connectors for Kafka Connect. These connectors cover various use cases, from databases to cloud storage, making it easy to integrate Kafka with many different systems.
+
+## Databricks Certified Associate Developer for Apache Spark Using Python: Chapter 1 - learning about the exam
+
+The chapter provides a guide to preparing for the Spark certification exam. It outlines a step-by-step methodology to approach the exam, starting with an overview of the exam structure, which consists of 60 questions to be answered in 120 minutes. To pass, a score of 70% (42 correct answers) is required.
+
+The exam covers three main topics: Spark Architecture (concepts and applications) and Spark DataFrame API applications, with the latter constituting 72% of the exam. The book emphasizes the importance of focusing on the DataFrame API while also covering architecture concepts for easy points.
+
+Resources such as mock exams and Databricks practice exams are recommended. During the exam, access to Spark documentation is provided via Webassessor, and familiarity with this interface is advised.
+
+The chapter also discusses the online proctored exam process, which includes webcam monitoring and specific requirements for the exam environment.
+
+Different types of questions in the exam include theoretical questions (e.g., explanation, connection, scenario, categorization, and configuration questions) and code-based questions (e.g., function identification, fill-in-the-blank, and ordering code).
+
+## Databricks Certified Associate Developer for Apache Spark Using Python: Chapter 2 - Understanding Apache Spark and Its Applications
+
+I have no plans for taking this certification exam but still covering the material would be a good idea.
+
+### Components of Spark
+
+* Spark Core: the backbone of Spark, provides core functionality and features
+* Spark SQL: provides SQL support for Spark, allows for querying data in RDDs and external sources
+* Spark Streaming: enables real-time data processing and ingestion from multiple sources
+* Spark MLlib: provides a framework for distributed and scalable machine learning
+* GraphX: provides a graph processing API for Spark
+
+### Why Choose Apache Spark?
+
+* Speed: one of the fastest processing frameworks for data
+* Reusability: can join batch and stream data seamlessly
+* In-memory computation: eliminates overhead of reading and writing to disks
+* Unified platform: integrates multiple components for a unified experience
+
+### Use Cases of Spark
+
+* Big data processing: handles high-volume, high-velocity, and high-variety data
+* Machine learning applications: provides a framework for distributed and scalable machine learning
+* Real-time streaming: enables real-time data processing and ingestion
+* Graph analytics: provides a graph processing API for Spark
+
+### Spark Users
+
+#### Data Analysts
+
+* Gather requirements from stakeholders
+* Identify relevant data sources
+* Collaborate with subject matter experts
+* Slice and dice data
+* Generate reports
+* Share results
+
+#### Data Engineers
+
+* Create, maintain, and optimize data pipelines
+* Collaborate with data analysts and data scientists
+
+#### Data Scientists
+
+* Create and test hypotheses
+* Transform data
+* Decide on machine learning algorithm
+* Prototype with different machine learning models
+* Create a baseline model
+* Tune the model
+* Tune the data
+* Transition models to production
+
+#### Machine Learning Engineers
+
+* Deploy machine learning models to production environments
+* Build machine learning pipelines
+* Monitor model performance
+* Handle data drift and model drift
+
+## Sample Qs from Chapter 3: Spark Architecture
+
+Normally, I would briefly talk about it, but given that the previous book I read today (chapter 5 to be specific), I do not want to write the same thing twice. 
+
+Below are some sample Qs related to Spark's architecture. 
+
+Question 1:
+
+What’s true about Spark’s execution hierarchy?
+
+**1. In Spark’s execution hierarchy, a job may reach multiple stage boundaries.**
+2. In Spark’s execution hierarchy, manifests are one layer above jobs.
+3. In Spark’s execution hierarchy, a stage comprises multiple jobs.
+4. In Spark’s execution hierarchy, executors are the smallest unit.
+5. In Spark’s execution hierarchy, tasks are one layer above slots.
+
+Question 2:
+
+What do executors do?
+
+1. Executors host the Spark driver on a worker-node basis.
+**2. Executors are responsible for carrying out work that they get assigned by the driver.**
+3. After the start of the Spark application, executors are launched on a per-task basis.
+4. Executors are located in slots inside worker nodes.
+5. The executors’ storage is ephemeral and as such it defers the task of caching data directly to the worker node thread.
+
+## Chapter 4: Spark DataFrames and their Operations 
+
+This chapter just introduced the basic PySpark API for
+
+* The Spark DataFrame API
+* Creating DataFrames
+* Viewing DataFrames
+* Manipulating DataFrames
+* Aggregating DataFrames
+
+Sample Q:
+
+Which of the following operations will trigger evaluation?
+
+1. DataFrame.filter()
+2. DataFrame.distinct()
+3. DataFrame.intersect()
+4. DataFrame.join()
+**5. DataFrame.count()**
+
+
+
+## MLP by hand
+
+Today I found this nice little excel sheet that teaches doing a forward prop in a neural network by hand (manually) - to practice matrix multiplication basically
+
+<img width="778" alt="image" src="https://github.com/user-attachments/assets/ad8c3d01-1b05-4675-83d6-a27eb5e9b47f">
+
+## [Hannam Design Factory](http://df.hannam.ac.kr/kor/sub01/menu_02.html) interview
+
+This morning, I went to the department and met with the team leader Ricky Park. Just a short talk about myself, my background in programming and projects I have developed, and finally some info about the projects that I might be doing when I take the practice course in the department. The course itself is worth 9 credicts (3x the amount of credits of an usual course), and is about taking on projects needed by actual companies and making them a reality. So from next week, every Friday from 9am to 6pm I will be doing this 'class'. 🥳
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 653541f748c7640d0d32443044357135fed1c1d9 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 1 Sep 2024 00:15:37 +0900
Subject: [PATCH 404/638] Create 2024-08-31-day-243.md

---
 _posts/2024-08-31-day-243.md | 265 +++++++++++++++++++++++++++++++++++
 1 file changed, 265 insertions(+)
 create mode 100644 _posts/2024-08-31-day-243.md

diff --git a/_posts/2024-08-31-day-243.md b/_posts/2024-08-31-day-243.md
new file mode 100644
index 000000000..0a183dc67
--- /dev/null
+++ b/_posts/2024-08-31-day-243.md
@@ -0,0 +1,265 @@
+---
+layout: post
+title: (Day 243) Finishing a PySpark book
+categories: [data-eng,theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 243!
+A quick summary of today:
+* finished the book - Databricks Certified Associate Developer for Apache Spark Using Python
+
+## Chapter 5: Advanced Operations and Optimizations in Spark
+
+### 1. Different Options to Group Data in Spark DataFrames
+
+Spark DataFrames provide several ways to group data for aggregation and analysis:
+
+- **GroupBy:** Groups data by one or more columns and allows you to perform aggregations on the grouped data.
+- **Rollup:** Similar to `GroupBy`, but it also creates subtotals along with the aggregations, which is useful for hierarchical data.
+- **Cube:** Extends `Rollup` by generating all possible combinations of columns, making it suitable for multidimensional data analysis.
+- **Pivot:** Transforms rows into columns, effectively reshaping the DataFrame by aggregating data across multiple categories.
+
+### 2. Various Types of Joins in Spark
+
+- **Inner Join:** Returns only the rows that have matching keys in both DataFrames. It’s the default join type.
+- **Left Join (Left Outer Join):** Returns all rows from the left DataFrame, and matched rows from the right DataFrame. Unmatched rows in the right DataFrame return nulls.
+- **Right Join (Right Outer Join):** Returns all rows from the right DataFrame, and matched rows from the left DataFrame. Unmatched rows in the left DataFrame return nulls.
+- **Outer Join (Full Outer Join):** Returns all rows when there is a match in either DataFrame, filling nulls where there is no match.
+- **Cross Join (Cartesian Join):** Returns the Cartesian product of two DataFrames, i.e., every row from the left is paired with every row from the right. This is a costly operation with large datasets.
+- **Broadcast Join:** Optimizes joins by broadcasting (distributing) a small DataFrame to all executors, eliminating the need for shuffling data.
+- **Shuffle Join:** Involves shuffling data across the network to group data from different partitions before performing the join. Typically used for larger datasets where broadcasting isn't feasible.
+
+### 3. Shuffle and Broadcast Joins
+
+- **Broadcast Hash Join:** Suitable when one of the DataFrames is small enough to fit in memory. Spark broadcasts the smaller DataFrame to all executors, avoiding shuffling and improving performance.
+- **Shuffle Sort-Merge Join:** Applied when both DataFrames are large. Data is shuffled across the cluster, and then sorted and merged on the join key. This join is expensive due to network I/O and sorting but is necessary for large datasets where broadcasting isn’t possible.
+- **Optimization Strategies:** Use broadcast joins when one DataFrame is small, and shuffle joins for large datasets. You can control the join behavior using Spark's `broadcast()` function and configuring settings like `spark.sql.autoBroadcastJoinThreshold`.
+
+### 4. Reading and Writing Data to Disk in Spark Using Different Data Formats
+
+- **CSV:** Commonly used text format, easy to read and write, but lacks efficient compression and schema enforcement.
+- **Parquet:** Columnar storage format with efficient compression, schema enforcement, and is optimized for analytical queries.
+- **ORC:** Similar to Parquet, but with better compression and performance for certain types of queries.
+- **Avro:** Row-based format, useful for serializing data for storage and data exchange.
+- **JSON:** Flexible text format, good for semi-structured data, but less efficient than Parquet or ORC.
+- **Delta:** Provides ACID transactions, schema enforcement, and time travel capabilities, built on top of Parquet.
+
+### 5. Using Spark SQL for Different Operations
+
+- **Querying Data:** Use SQL-like syntax to query structured data. Spark SQL supports complex queries with joins, aggregations, and subqueries.
+- **Data Manipulation:** Perform operations like filtering, grouping, sorting, and joining data using SQL queries.
+- **Data Integration:** Spark SQL allows integration with various data sources, such as Hive, JDBC, and others, enabling seamless data retrieval and storage.
+- **Window Functions:** Implement advanced analytics using functions like `ROW_NUMBER`, `RANK`, and `LEAD`/`LAG` for operations over specific data partitions.
+
+### 6. Catalyst Optimizer
+
+- **Rule-based Optimization:** Applies a set of predefined rules to simplify and optimize the logical plan of a query (e.g., predicate pushdown, constant folding).
+- **Cost-based Optimization (CBO):** Uses statistics and cost models to choose the most efficient physical plan by estimating the cost of different execution strategies.
+- **Logical Plan Optimization:** Transforms the initial logical plan into an optimized version by eliminating unnecessary operations.
+- **Physical Plan Optimization:** Converts the optimized logical plan into a physical plan that can be executed efficiently on the Spark cluster.
+
+### 7. Narrow and Wide Transformations in Spark
+
+- **Narrow Transformations:** Operations where each input partition contributes to at most one output partition. Examples include `map`, `filter`, and `flatMap`. These transformations are efficient because they do not involve shuffling data across the network.
+- **Wide Transformations:** Operations where input partitions contribute to multiple output partitions, requiring data to be shuffled across the network. Examples include `groupBy`, `reduceByKey`, and `join`. Wide transformations are more expensive due to the shuffling step.
+- **When to Use:** Prefer narrow transformations when possible to reduce network overhead. Use wide transformations when operations inherently require data shuffling.
+
+### 8. Data Persistence and Caching Techniques
+
+- **Caching:** Use `cache()` or `persist()` to store DataFrames in memory, reducing recomputation when the same data is accessed multiple times. By default, data is cached in memory, but you can choose other storage levels (e.g., disk, memory, or a combination).
+- **Memory Management:** Monitor the memory usage and cache only the necessary data. Use the `unpersist()` method to free up memory when the cached data is no longer needed.
+- **Best Practices:** Cache data only when you are going to reuse it, and choose the appropriate storage level based on your resource availability and use case.
+
+### 9. Data Partitioning Through Repartition and Coalesce
+
+- **Repartition:** Increases or decreases the number of partitions by shuffling data across the network. It is useful when you need to balance the workload across the cluster or improve parallelism for large datasets.
+- **Coalesce:** Reduces the number of partitions without a full shuffle. It’s more efficient than `repartition()` when decreasing the number of partitions, as it tries to minimize data movement.
+- **When to Use:** Use `repartition()` when you need to increase the number of partitions for better parallelism or when dealing with skewed data. Use `coalesce()` when reducing partitions without requiring a full shuffle.
+
+### 10. User-Defined Functions (UDFs) and Custom Functions
+
+- **User-Defined Functions (UDFs):** Allow you to implement custom logic that is not available in Spark’s built-in functions. UDFs can be applied to DataFrames for complex data transformations.
+- **Python UDFs (Pandas UDFs):** More efficient UDFs that leverage Apache Arrow for optimized data transfer between JVM and Python processes.
+- **When to Use:** Use UDFs when you need to perform operations that are not natively supported by Spark. However, prefer built-in functions whenever possible, as UDFs can be less performant due to serialization overhead.
+
+### 11. Performing Advanced Optimizations in Spark
+
+- **Catalyst Optimizer:** Leverage the Catalyst optimizer’s rule-based and cost-based optimizations to automatically improve query performance.
+- **Adaptive Query Execution (AQE):** Adjusts the execution plan at runtime based on actual data statistics, optimizing tasks like join strategies, reducing skew, and managing resources dynamically.
+- **Techniques:** Tune configurations (e.g., `spark.sql.shuffle.partitions`), use efficient data formats like Parquet, and apply optimizations like predicate pushdown and partition pruning.
+
+### 12. Data-Based Optimization Techniques
+
+- **Predicate Pushdown:** Filters data at the source, reducing the amount of data loaded into Spark.
+- **Partition Pruning:** Only reads the relevant partitions of data based on query filters, improving I/O efficiency.
+- **Skew Mitigation:** Handle skewed data by techniques like salting, which redistributes skewed keys, or using AQE to automatically handle skew.
+- **Efficient File Formats:** Use columnar formats like Parquet and ORC for better compression and query performance.
+- **Bucketing:** Pre-partition data into buckets on a specified column to optimize join and aggregation operations.
+
+
+## Chapter 6: SQL Queries in Spark
+
+### What is Spark SQL?
+
+1. **Advantages of Spark SQL:**
+   - **Unified Data Access:** Spark SQL allows you to query structured and semi-structured data using SQL, making it easier to work with data from different sources.
+   - **Integration with Spark Core:** Spark SQL seamlessly integrates with Spark’s core APIs, allowing you to mix SQL queries with DataFrame and RDD transformations.
+   - **Optimized Performance:** Spark SQL benefits from the Catalyst optimizer, which automatically optimizes query execution plans, resulting in faster data processing.
+   - **Support for Multiple Data Formats:** You can easily work with different data formats such as JSON, Parquet, and Hive tables using Spark SQL.
+   - **Compatibility:** It supports a wide range of SQL-based data processing operations, making it accessible for users familiar with traditional SQL databases.
+
+2. **Integration with Apache Spark:**
+   - **DataFrames and Datasets:** Spark SQL is built on top of DataFrames and Datasets, which provide a high-level abstraction for data manipulation. These structures integrate seamlessly with Spark’s core APIs, enabling complex data transformations and machine learning pipelines.
+   - **Interoperability:** Spark SQL queries can be written in conjunction with other Spark operations, allowing for a smooth transition between SQL queries and Spark’s functional programming API.
+   - **Performance Gains:** The integration ensures that operations such as filtering, grouping, and joining can benefit from Spark's in-memory computing capabilities.
+
+3. **Key Concepts – DataFrames and Datasets:**
+   - **DataFrames:** A DataFrame is a distributed collection of data organized into named columns. It is equivalent to a table in a relational database and provides an easy-to-use API for performing data operations.
+   - **Datasets:** Datasets are a strongly-typed version of DataFrames, offering the benefits of both RDDs (type-safety) and DataFrames (query optimization via the Catalyst optimizer).
+
+### Getting Started with Spark SQL
+
+1. **Loading and Saving Data:**
+   - **Reading Data:** Spark SQL supports reading data from various sources, including CSV, JSON, Parquet, Avro, ORC, and Hive tables. You can load data into DataFrames using the `read` API.
+   - **Saving Data:** Spark SQL also allows you to save DataFrames back to storage in different formats, offering options for compression, partitioning, and file format choices. The `write` API enables this functionality.
+   - **Examples:** For example, you can load a CSV file using `spark.read.csv("file.csv")` and save a DataFrame as Parquet using `df.write.parquet("output.parquet")`.
+
+2. **Utilizing Spark SQL to Filter and Select Data Based on Specific Criteria:**
+   - **Filtering:** Spark SQL allows you to filter data using SQL-like `WHERE` clauses or DataFrame `filter()`/`where()` methods. This is useful for narrowing down datasets based on specific conditions.
+   - **Selecting Data:** You can select specific columns or expressions using SQL’s `SELECT` statement or DataFrame methods like `select()` and `selectExpr()`.
+   - **Examples:** Using SQL: `SELECT * FROM table WHERE age > 30`, or with DataFrames: `df.filter(df("age") > 30).select("name", "age")`.
+
+3. **Exploring Sorting and Aggregation Operations Using Spark SQL:**
+   - **Sorting Data:** Spark SQL supports sorting data using SQL’s `ORDER BY` clause or DataFrame `orderBy()`/`sort()` methods. This is useful for organizing data based on specific columns.
+   - **Aggregation:** You can perform aggregation operations such as `SUM()`, `AVG()`, `COUNT()`, and `GROUP BY` in SQL or use DataFrame functions like `groupBy()` and `agg()`.
+   - **Examples:** SQL example: `SELECT department, COUNT(*) FROM employees GROUP BY department`, or with DataFrames: `df.groupBy("department").count()`.
+
+4. **Grouping and Aggregating Data – Grouping Data Based on Specific Columns and Performing Aggregate Functions:**
+   - **Grouping:** Grouping data is a common operation in data analysis. Spark SQL supports grouping by specific columns using `GROUP BY` in SQL or the `groupBy()` method in DataFrames.
+   - **Aggregate Functions:** After grouping, you can apply aggregate functions like `SUM()`, `MAX()`, `MIN()`, and custom aggregations to summarize the data.
+   - **Examples:** SQL example: `SELECT department, AVG(salary) FROM employees GROUP BY department`, or with DataFrames: `df.groupBy("department").agg(avg("salary"))`.
+
+### Advanced Spark SQL Operations
+
+1. **Leveraging Window Functions to Perform Advanced Analytical Operations on DataFrames:**
+   - **Window Functions:** Window functions allow you to perform calculations across a set of rows related to the current row, without collapsing the rows into a single output. Examples include `RANK()`, `ROW_NUMBER()`, and `LEAD()`/`LAG()`.
+   - **Use Cases:** These functions are useful for tasks like calculating moving averages, ranking data, and comparing data across rows.
+   - **Examples:** SQL example: `SELECT name, salary, RANK() OVER (ORDER BY salary DESC) AS rank FROM employees`, or with DataFrames: `df.withColumn("rank", rank().over(Window.orderBy("salary")))`.
+
+2. **User-Defined Functions (UDFs):**
+   - **Custom Logic:** UDFs allow you to define custom functions that can be applied to DataFrame columns. They are useful when built-in Spark functions do not cover specific use cases.
+   - **Performance Considerations:** While UDFs provide flexibility, they may introduce performance overhead due to serialization and deserialization. Use them judiciously, and prefer built-in functions when possible.
+   - **Examples:** Define a UDF in Python: `from pyspark.sql.functions import udf; @udf def square(x): return x * x; df.withColumn("squared", square(df("value")))`.
+
+3. **Working with Complex Data Types – Pivot and Unpivot:**
+   - **Pivoting:** Pivoting transforms rows into columns, which is useful for reshaping data. Spark SQL provides the `pivot()` function for this purpose.
+   - **Unpivoting:** Unpivoting, or reversing a pivot, converts columns back into rows. Although Spark SQL doesn't have a direct unpivot function, you can achieve this by using a combination of functions like `stack()`.
+   - **Examples:** Pivot example: `df.groupBy("category").pivot("year").sum("revenue")` to transform sales data across years into separate columns.
+
+## Chapter 7: Structured Streaming in Spark
+
+### Real-time Data Processing
+
+Real-time data processing involves continuous ingestion, processing, and analysis of streaming data. Key characteristics include low latency, scalability, fault tolerance, and a streaming data model. Advantages include rapid response, personalization, operational efficiency, and situational awareness.
+
+### Streaming Architectures
+
+Streaming architectures consist of streaming sources, streaming processing engines, and streaming sinks. Different architectures include event-driven, lambda, and unified streaming architectures. Each architecture has its strengths and trade-offs, making them suitable for various scenarios.
+
+### Spark Streaming
+
+Spark Streaming is a real-time data processing framework built on Apache Spark. It extends Spark's capabilities to support high-throughput, fault-tolerant, and scalable stream processing. Key concepts include:
+
+* **DStreams:** As mentioned earlier, DStreams represent the continuous stream of data in Spark Streaming. They are a sequence of RDDs, where each RDD contains data from a specific time interval. DStreams support various transformations and actions, enabling complex computations on the stream.
+* **Window operations:** Window operations allow you to apply transformations on a sliding window of data in the stream. It enables computations over a fixed window size or based on time durations, enabling tasks such as windowed aggregations or time-based joins.
+* **Stateful operations:** Spark Streaming gives you the ability to maintain stateful information across batches. It enables operations that require maintaining and updating state, such as cumulative counts.
+* **Checkpointing:** Checkpointing is a crucial mechanism in Spark Streaming to ensure fault tolerance and recovery. It periodically saves the metadata about the streaming application, including the configuration, DStream operations, and the processed data. It enables the recovery of the application if there are failures.
+
+Advantages include a unified processing model, high-level abstractions, fault tolerance, and scalability.
+
+### Structured Streaming
+
+Structured Streaming is a newer and more expressive streaming API in Apache Spark. It introduces a continuous processing model, treating streaming data as an unbounded table or DataFrame. Key features include:
+
+- **Expressive API**
+- **Fault tolerance**
+- **Scalability**
+- **Unified batch and streaming**
+- **Ecosystem integration**
+
+Advantages include low-latency, near-real-time results, and support for windowed computations.
+
+### Streaming Fundamentals
+
+- **Stateless streaming** processes events independently, while **stateful streaming** maintains and utilizes contextual information across events.
+- **Event time** and **processing time** are two important notions in streaming.
+- **Watermarking** and **late data handling** are mechanisms to deal with event time and handle delayed or late-arriving data.
+
+### Streaming Sources and Sinks
+
+- **Streaming sources** retrieve data from external systems, while **streaming sinks** receive processed data and write it to external destinations.
+- Built-in streaming sources include file, Kafka, socket, and Structured Streaming sources.
+- Built-in streaming sinks include console, file, Kafka, and foreach sinks.
+
+### Advanced Techniques
+
+Handling fault tolerance, schema evolution, and joins are advanced techniques in Structured Streaming. **Stream-stream joins** and **stream-static joins** are two types of joins in Structured Streaming.
+
+### Future Developments
+
+Future developments in Structured Streaming may include enhanced support for streaming sources and sinks, advanced event time handling, performance optimization, integration with AI and machine learning, and seamless integration with streaming data warehouses.
+
+Overall, this text provides a comprehensive overview of real-time data processing using Apache Spark, highlighting the key concepts, advantages, and techniques in Spark Streaming and Structured Streaming.
+
+## Chapter 8: ML with Spark ML
+
+### Advantages of Apache Spark for Large-Scale ML
+
+1. **Speed and Performance**: Spark's in-memory computing and optimized data processing accelerate large-scale data processing, reducing time for iterative ML algorithms.
+2. **Distributed Computing**: Spark distributes data and computations across multiple nodes, making it scalable and suitable for processing massive datasets.
+3. **Fault Tolerance**: Spark's built-in fault tolerance recovers from failures by tracking lineage with Resilient Distributed Datasets (RDDs).
+4. **Versatility and Flexibility**: Spark's APIs and libraries support various data processing tasks, with MLlib offering a wide range of distributed ML algorithms and tools.
+5. **Real-Time and Streaming Capabilities**: Spark Streaming enables real-time ML applications, valuable in scenarios like fraud detection and sentiment analysis.
+6. **Ecosystem and Community Support**: Spark has a vibrant community and a rich ecosystem of tools, ensuring continuous development and extensive resources for learning.
+
+### Spark MLlib vs Spark ML
+
+#### Spark MLlib
+
+- **RDD-based API**: Utilizes the RDD abstraction for distributed data processing, suitable for batch processing and iterative algorithms.
+- **Diverse Algorithms**: Offers a wide range of distributed ML algorithms, from classification to clustering.
+- **Feature Engineering**: Provides tools for feature extraction, transformation, and selection.
+- **Model Persistence**: Supports saving and loading trained models.
+
+#### Spark ML
+
+- **DataFrame-based API**: Uses DataFrames for a more intuitive and structured data representation, simplifying integration with Spark SQL.
+- **Pipelines**: Introduces pipelines for chaining data transformations and model training stages, streamlining complex ML workflows.
+- **Integrated Feature Transformers**: Includes built-in feature transformers for easier feature engineering.
+- **Unified API**: Provides a consistent API for various ML tasks like classification, regression, and clustering.
+
+### When to Use Spark MLlib
+
+- Working with older Spark versions without Spark ML support.
+- Need for low-level control with RDDs.
+- Access to specific algorithms not available in Spark ML.
+
+### When to Use Spark ML
+
+- Using Spark 2.0 or later versions.
+- Preference for a higher-level API with DataFrame support.
+- Building end-to-end ML pipelines with integrated transformers and pipelines.
+
+Both libraries offer robust ML capabilities, with Spark MLlib focusing on low-level control and distributed algorithms, while Spark ML provides a user-friendly API with DataFrame integration.
+
+## Remaining chapter 9 and 10
+
+These two chaters are two mock tests for the Databricks certificate. The certificate registration is $200 so I am not sure if I will do it. Regardless, the book was great ^^
+
+---
+
+I think next, I will read [Streaming Databases](https://learning.oreilly.com/library/view/streaming-databases/9781098154820/)
+
+That is all for today!
+
+See you tomorrow :)

From 80d9eb397238f06d20415a4fc55c024614817ec7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 1 Sep 2024 21:42:47 +0900
Subject: [PATCH 405/638] Create 2024-09-01-day-244.md

---
 _posts/2024-09-01-day-244.md | 937 +++++++++++++++++++++++++++++++++++
 1 file changed, 937 insertions(+)
 create mode 100644 _posts/2024-09-01-day-244.md

diff --git a/_posts/2024-09-01-day-244.md b/_posts/2024-09-01-day-244.md
new file mode 100644
index 000000000..1c512b2b6
--- /dev/null
+++ b/_posts/2024-09-01-day-244.md
@@ -0,0 +1,937 @@
+---
+layout: post
+title: (Day 244) Streaming databases book + advanced RAG techniques
+categories: [data-eng,theory,nlp]
+---
+
+# Hello :) Today is Day 244!
+A quick summary of today:
+* started reading Streaming Databases on O'Reilly
+* covered a short on-demand course on advanced RAG techniques
+
+## Streaming Databases Chapter 1: Streaming Foundations
+
+### Externalizing Database Features
+
+#### Write-Ahead Log
+
+The WAL is a mechanism that allows databases to ensure data durability and consistency. The spinning disks that databases write data upon don’t offer transactions. So databases are challenged to provide transactionality atop a device that doesn’t offer transactions. WALs are a way for databases to provide transactionality without having transactional disks.
+
+A transaction in a database refers to a sequence of one or more database operations executed as a single unit of work. These operations can include data insertion (INSERT), data modification (UPDATE), or data deletion (DELETE). The WAL acts as a buffer that can be overwritten as new changes are made. The WAL persists the change to disk
+
+![image](https://github.com/user-attachments/assets/fbcb15ce-c6b5-4a75-b5b1-1279ce4e28ea)
+![image](https://github.com/user-attachments/assets/63b5e12c-aae7-4d89-bce8-beb9a67d5032)
+
+When saving transactions on disk, the database follows these steps:
+
+1. The client starts a transaction by issuing a BEGIN statement
+2. The database writes a record to the WAL indicating that a transaction has started.
+3. The client makes changes to the database data.
+4. The client commits the transaction by issuing a COMMIT statement.
+5. The database writes a record to the WAL indicating that the transaction has been committed.
+6. The changes made by the transaction are written to disk.
+
+#### Streaming Platforms
+
+The most popular one is Apache Kafka, but there are alternatives. Platforms like that are distributed, scalable, and fault-tolerant systems designed to handle real-time data streams. They provide a powerful infrastructure for ingesting, storing, and processing large volumes of continuous data from various sources.
+
+Most streaming platforms have a construct called partitions. These constructs mimic WALs in a database. Transactions are appended to partitions like transactions to a WAL. Streaming platforms can hold many partitions to distribute the stream load to promote horizontal scaling. Partitions are grouped in abstractions called topics to which applications either publish or consume transactions.
+
+By publishing the transactions to the streaming platform, you’ve published it to all subscribers who may want to consume it. This is called a publish and subscribe model, and it’s critical to allow multiple disparate consumers to use these transactions.
+
+#### Materialized Views
+
+In typical OLTP databases, materialized views are special types of database objects that store the results of a precomputed query or aggregation. Unlike regular views, which are virtual and dynamically generate their results based on the underlying data, materialized views store the actual data, making them physically stored in the database.
+
+The purpose of materialized views is to improve the performance of complex queries or aggregations by precomputing and storing the results. When a query references a materialized view, the database can quickly retrieve the precomputed data from the materialized view instead of recalculating it from the base data tables. This can significantly reduce the query execution time and improve overall database performance, especially for large and resource-intensive queries
+
+By enabling materialized views to be updated, the stored data will always be fresh; that is, the stored data is real-time data. This characteristic makes materialized views fit naturally into streaming frameworks.
+
+![image](https://github.com/user-attachments/assets/918f7791-fd99-4f05-99de-0b9b962902b4)
+
+### Connectors
+
+In streaming, there are two types of connectors:
+
+#### Source Connectors
+Source connectors read data from a data source system (e.g., a database) and make that data available as an event stream.
+
+#### Sink Connectors
+Sink connectors consume data from an event stream and write that data to a sink system (a database again, a data warehouse, a data lake, etc.).
+
+The two connector types are below. In most cases, source connectors either transform data at rest into streaming data (aka data in motion), whereas sink connectors transform streaming data into data at rest.
+
+![image](https://github.com/user-attachments/assets/4bc8b880-b5eb-43aa-973a-21d39f0fb484)
+
+With data at rest, the data is sitting in a database or a filesystem and not moving. Data at rest tends to get processed using batching or microbatching techniques. A dataset batched from one source system to another has a beginning and an end. The applications that process batched data can be started using a job scheduler like cron, and the data processing ends when the dataset ends.
+
+This is the opposite of streaming, or data in motion. Data in motion implies that there is neither a beginning nor an end to the data. Applications that process streaming data are always running and listening for new data to arrive on the stream.
+
+Now let’s dive into how source and sink connectors can be implemented.
+
+### Connector Middleware
+Connector middleware solutions such as Kafka Connect, Meroxa, Striim, or StreamSets already provide a large number of connectors out of the box and are often extensible to serve further sources and sinks. Connector middlewares also offer horizontal scaling, monitoring, and other required features, especially for production deployments.
+
+Kafka Connect is part of the Apache Kafka project. It is a distributed cluster in which Kafka connectors are deployed to run in parallel. These types of deployments create complexity in the streaming architecture. These clusters are bulky, and maintenance of them is arduous.
+
+If you have a large amount of data sources and sinks, these clusters often become costly and consume a lot of resources. Delegation of this integration is better solved by embedding the connectors into the systems themselves.
+
+#### Embedded
+An increasing number of databases offer embedded connectors to streaming platforms. CockroachDB is an example of this. An even larger set of databases has implemented embedded connectors; that is, they can consume data off the event stream themselves. Examples are Apache Druid, Apache Pinot, ClickHouse, StarRocks, Apache Doris, and Rockset.
+
+Having databases solve the integration to streaming platforms gets them closer to becoming streaming databases. If you enable databases with the ability to pull and push data into streaming platforms, streaming will naturally become a first-class citizen in the database.
+
+#### Custom-Built
+Connectors can be custom-built, for example, by implementing a dedicated microservice. The advantage of this approach is its flexibility; the downside is clearly the need to “reinvent the wheel”—it often doesn’t make sense to implement connectors from scratch, especially in light of the plethora of existing powerful and scalable open source connectors (e.g., the Debezium source connectors for the Kafka Connect middleware).
+
+![image](https://github.com/user-attachments/assets/2ba6e1cd-927a-4b04-a4a7-8f6e536168a6)
+
+## Chapter 2: Stream Processing Platforms
+
+Events tend to need cleansing and preparation before they undergo analytical processing. Events also need to be enriched with context for them to be useful enough to derive insights. Analytical processing heavily relies on the accuracy and reliability of the data. By addressing issues such as missing values, inconsistencies, duplicates, and outliers, data quality is improved, leading to more reliable and accurate analytical results.
+
+Event data preparation can also significantly impact the performance of analytical queries. By optimizing the data layout, indexing, and partitioning, the efficiency of data retrieval and processing can be improved. This includes techniques such as data denormalization, columnar storage, and indexing strategies tailored for the analytical workload. Well-prepared data can reduce the processing time and enable faster insights.
+
+Event data preparation also plays a crucial role in ensuring data governance and compliance. This involves enforcing data security measures, anonymizing sensitive information, and adhering to privacy regulations. By properly preparing the data, organizations can maintain data integrity, protect privacy, and comply with legal and ethical requirements.
+
+![image](https://github.com/user-attachments/assets/a3b37183-2f46-485a-aa60-86825a2af847)
+
+Above, the dotted line is the processing stage.
+
+Transformation tasks tend to be resource consuming and process intensive. It’s best to complete transformations as part of a preprocessing step in a data pipeline before writing to a data store that serves it to consumers. The earlier the transformations can be done, the better—as executing them incrementally, seeing only small amounts of data at once, is much less resource consuming than having to scour through large amounts of data already at rest later.
+
+The destination data stores are typically online analytical processing, or OLAP, data stores. OLAP data stores allow users to invoke analytical queries on the data. Real-time OLAP (RTOLAP) data stores are also OLAP but are optimized for serving real-time analytical data.
+
+It’s best practice to avoid process-intensive tasks in the OLAP that serve analytics to the consumers. OLAPs need to reserve their resources for quickly responding to analytical queries that answer questions related to the business or provide fast insights to customers—another reason for executing the preprocessing as early as possible in the pipeline.
+
+### Stateful Transformations
+
+- Streaming platforms like Kafka:
+  - Do not transform data in a way that allows them to hold state.
+  - Cannot perform complex transformations like aggregations and joins directly.
+  - Require additional components like Kafka Streams for complex transformations.
+
+- Data Pipelines:
+  - Follow two common patterns for data integration and processing:
+    - ETL: Data is extracted, transformed, then loaded into the destination.
+    - ELT: Data is extracted, loaded, and then transformed.
+
+- ETL and State Management:
+  - Stateful transformations in ETL require data to be temporarily stored to maintain and update information between multiple input data elements.
+  - Examples of stateful transformations:
+    - Rolling averages: Maintain sum and count of data elements to calculate averages over a sliding window.
+    - Sessionization: Group related events into sessions based on criteria like user activity within a time threshold.
+    - Deduplication: Compare incoming events with previously stored events to filter out duplicates.
+    - Windowed aggregations: Perform aggregations (e.g., sum, count) over data windows by holding state.
+    - Machine learning models: Continuously update and refine models using streaming data, requiring state to hold learned parameters.
+
+- Stateless Transformations:
+  - Some streaming platforms offer stateless transformations, called functions or single message transforms.
+  - These include simple operations like string transformations or basic math operations, not qualifying as stateful transformations in ETL/ELT.
+
+- Limitations of Pure Streaming Platforms:
+  - Kafka and similar platforms only handle data extraction ("E").
+  - Loading data does not occur until stateful transformations are applied.
+  - Pure streaming platforms cannot directly serve user-facing applications due to the lack of cleansing, preparation, and enrichment of events.
+
+- Role of Stream Processing Platforms:
+  - Stream processing platforms (e.g., Kafka Streams) hold state and perform complex transformations.
+  - These platforms are essential for streaming ETL pipelines, enabling real-time analytics.
+  - Data pipelines collect data from various sources, transform it for analysis, and deliver it to analytics tools in real-time.
+
+### Data Pipelines
+
+One of the roles of the data pipeline is to move the data from the operational data plane to the analytical data plane. The operational data plane is where the applications live, including microservices and OLTP databases. The analytical data plane is where the analytical systems live—like data warehouses, data lakes, lakehouses—all kinds of OLAP data stores. Data pipelines live in between the operational and analytical data planes, transforming and integrating data from one to the other, respectively.
+
+![image](https://github.com/user-attachments/assets/135e42b3-3e04-44e1-bba1-27fda19a36f1)
+
+Once the data is extracted, it undergoes transformation tasks previously mentioned to convert it into a suitable format for the target system or data warehouse, or OLAP data store. The transformed data is typically stored temporarily in an intermediate store.
+
+![image](https://github.com/user-attachments/assets/a75bf450-a94f-49d7-803d-7074deac8f01)
+
+### Stream Processors
+
+Popular ones:
+
+1. Apache Kafka Streams is a stream processing library for JVM-based programming languages that is part of the Apache Kafka project. It allows developers to build real-time applications and microservices that consume, process, and produce streams of data from/to Kafka.
+
+2. Apache Flink is a stream processor that supports both batch and stream processing and can be connected to a large variety of sources and sinks, including Kafka, Pulsar, and Kinesis, as well as databases like MongoDB and Elasticsearch. Flink views batch as a special case of streaming (streaming with bounded data). Contrary to Kafka Streams, Flink is not a library but runs on its own cluster.
+
+3. Spark Structured Streaming is a component of Apache Spark that enables stream processing. It supports a large variety of connectors and is also cluster based. It is set apart from other stream processors like Kafka Streams, Flink, and Samza by making use of mini-batching instead of native stream processing—for Spark, streaming is a special case of batch rather than the other way around, as in Flink.
+
+4. Apache Samza is a stream processor developed by LinkedIn. It supports Kafka, Azure Event Hubs, Kinesis, and HDFS and is also cluster based like Flink.
+
+5. Apache Beam is not a stream processor itself but a unified programming model and set of software development kits (SDKs) for building data processing pipelines. It provides an abstraction that allows developers to write data-processing jobs that can be executed on various distributed processing engines, such as Apache Flink, Apache Spark, Apache Samza, and Google Cloud Dataflow.
+
+### Two Types of Streams
+
+There are two types of real-time streams of data that flow through stream processors—change and append-only streams
+
+![image](https://github.com/user-attachments/assets/0a2fc902-b49d-45b0-b335-307a9c493f2a)
+
+Above, the topics are highlighted: Click event, Product, and Customer. The click event stream is append-only, which means it contains discrete, distinct events. Why do we know this? Clicks are always unique. Every click event is different, even if it’s done on the same T-shirt by the same customer. The only difference is the time it was clicked. In an append-only stream, the events are associated with a timestamp that indicates when each event occurred. The timestamp provides temporal information, allowing for the ordering and sequencing of events within the stream. This also applies to other types of events, not just click events.
+
+If clicks were written to an OLTP database, the database would run out of space very quickly because every click event is a unique event; that is, we would trigger an insert into the OLTP table for every click on the application. Hence, using an OLTP database for click events would be bad design—it’s clearly better to use an append-only stream of events here.
+
+Conversely, change stream data almost always comes from CDC events. These events are the transactions applied to a table in the OLTP database. These transactions are inserts, updates, and deletes. These records will not grow as fast as clickstream data because some events will include updates or deletes of existing records in a table.
+
+Also, customer data may not change a lot. Customers may change their names, emails, or phone numbers, but probably not that often. Products would have a lot more changes, but still not many compared to clicks on an application. Also, change streams tend to represent changes to dimensional data. Dimensions change slowly in a database because they typically represent properties that do not frequently or rapidly change for the entities they describe.
+
+## Chapter 3: Serving Real-Time Data
+
+![image](https://github.com/user-attachments/assets/5b38e7e7-90f7-4fb2-8bc5-305f0b09e2be)
+
+This chapter will talk about delivering enriched real-time data to the end user. This stage of the real-time data pipeline is the last mile streaming data takes before it’s presented to the end user.
+
+### Real-Time Expectations
+
+since we want to serve analytics in real time, we should consider some metrics:
+
+* Latency: Measures the time it takes for an analytics query or computation to complete and return results. In real-time analytics, low latency is crucial to provide near-instantaneous insights to users. SLA metrics may define acceptable latency thresholds, such as average response time or maximum response time, to ensure the timely delivery of analytics results.
+
+* Throughput and concurrency: Measures the number of analytics queries or computations that can be processed within a given time frame. It indicates the system’s capacity to handle concurrent requests and is especially important for high-volume scenarios. SLA metrics may specify a target throughput, such as queries per second or computations per minute, to ensure sufficient capacity for real-time analytics workloads.
+
+* Data freshness: Indicates how up-to-date the analytics results are in relation to the underlying data streams. It measures the delay between when the data is generated and when it becomes available for analysis. SLA metrics may specify acceptable data freshness requirements, such as a maximum delay in seconds or minutes, to ensure that users have access to the most recent information
+
+* Accuracy: Measures the correctness and precision of the analytics results. SLA metrics may define acceptable error rates, confidence intervals, or validation criteria to ensure the accuracy of real-time analytics.
+
+* Other: availability, consistency, scalability, security and privacy
+
+### Choosing an Analytical Data Store
+
+Here are some examples of data stores that can satisfy these strict real-time SLAs:
+
+* In-memory databases, such as Redis, SingleStore (formerly MemSQL), Hazelcast, or Apache Ignite, store data in memory for fast access and processing. These databases offer extremely low latency and high throughput, making them suitable for real-time analytics that require near-instantaneous responses.
+
+* RTOLAP data stores, such as Apache Pinot, Apache Druid, ClickHouse, StarRocks, and Apache Doris. These data stores tend to be column-oriented distributed data stores. They organize data by column rather than row, enabling efficient analytics and query processing. These databases can handle large volumes of data and offer excellent scalability and high availability, making them well-suited for real-time analytics at scale.
+
+* Hybrid transactional/analytical processing (HTAP) data stores such as TiDB or SingleStore (formerly MemSQL) support both real-time transactional processing and analytics within a single system. These databases provide the capability to serve real-time analytics directly on operational data, minimizing data movement and reducing latency.
+
+### Sourcing from a Topic
+
+![image](https://github.com/user-attachments/assets/a0964497-5ed2-4c43-a7cd-5a41f6414259)
+
+Streaming platforms provide a publish and subscribe model for distributing streaming data, which means other systems, domains, and use cases can be consuming the same data from the sink topic. The preprocessing work done in the stream processing platform may have prepared the data only to the point where all consumers can use the data, in a data format that corresponds to a common denominator for them. This leaves any consumer-specific data preparation up to the consumer to implement.
+
+for example, Domain consumer 1 needs the timestamps in the data to be seconds while Domain consumer 2 needs the data to be in the YEAR-MONTH-DAY format. The engineer who implemented the transformation in the stream processing platform could decide to serve the timestamp in milliseconds so that both consumers can derive their preferred timestamp values. This would also satisfy any future consumer who needs the data in milliseconds.
+
+These scenarios require queries to implement the transformation to get the timestamp value to their required format. Unfortunately, this transformation will slow the query down. It would also need to be executed for every run of the query and could eventually break the SLA for latency. This would eventually cause the application or dashboard to slow down and create an unsatisfactory experience for the data consumers. Any consumer-specific data preparation left for the consumer to implement will always affect the SLAs for serving real-time analytics.
+
+![image](https://github.com/user-attachments/assets/f2d725ce-2556-4204-91e6-f9c194acf4bd)
+
+### Ingestion Transformations
+
+In RTOLAP systems, streaming data can be preprocessed before being stored, allowing data transformations (e.g., timestamp conversion) in-flight. Most RTOLAPs do stateless preprocessing, but some, like Apache Pinot, can perform stateful transformations during ingestion, which can optimize query performance. This stateful transformation could allow for creating denormalized views, making analytical queries faster by reducing the need for real-time joins. Ingestion transformations also optimize data for analytical queries by using formats like columnar storage, which is more efficient for OLAP systems than row-based formats used in OLTP databases.
+
+### OLTP Versus OLAP
+
+**OLTP Databases**  
+- Used for transactional workloads on the operational plane.
+- Designed for real-time transactional operations, supporting CRUD (Create, Retrieve, Update, Delete) operations.
+- Optimized for handling a large number of concurrent transactions, ensuring data integrity and consistency.
+- Typically have normalized data structures to minimize redundancy and maintain consistency.
+- Support ACID properties:
+  - Atomicity: Ensures transactions are all-or-nothing.
+  - Consistency: Maintains database rules and constraints.
+  - Isolation: Prevents transaction interference.
+  - Durability: Guarantees permanent data storage post-transaction.
+
+**OLAP Data Stores**  
+- Used for analytical workloads on the analytical plane.
+- Employ different optimization techniques compared to OLTP databases.
+- Optimized for fast analytical queries, often using columnar storage for better performance.
+
+#### Row- vs. Column-Based Optimization
+
+**Row-Based Format**
+- Data is stored row by row, similar to spreadsheets.
+- Optimized for transactional processing, with better performance in retrieving and updating entire rows.
+- Offers some compression techniques but can lead to higher disk I/O during queries.
+
+**Column-Based Format**
+- Data is stored column by column, ideal for analytical queries that focus on specific columns.
+- Provides better compression ratios, reducing storage needs.
+- Designed to read only the required columns, improving query efficiency and performance.
+
+#### Table: Row- vs. Column-Based Optimization Differences
+
+| Properties       | Row-Based                           | Column-Based                                 |
+|------------------|-------------------------------------|----------------------------------------------|
+| **Data Storage** | Each row is stored together, less optimized for analytical queries. | Each column is stored together, optimized for analytical queries. |
+| **Query Performance** | Suited for transactional operations. | Excels in analytical operations. |
+| **Compression**  | Offers some compression techniques with overhead. | Provides better compression ratios. |
+| **Query Efficiency** | Higher disk I/O for partial queries. | Reads only necessary columns, improving performance. |
+
+### Queries Per Second and Concurrency
+
+Queries per second (QPS) is the measure of how well an OLAP data store can return the results of a query. This in turn indicates the volume of queries the OLAP can process within a second and, ultimately, the number of concurrent queries that can be invoked.
+
+In our clickstream use case, we did not specify how many end users may be viewing the analytics. The assumption is for every end user, there will be at least one query. We did indicate that this is a real-time use case. Hence the data freshness on the dashboard is expected to be real time. Real-time dashboards tend to require a high rate of refreshes to keep the charts in the dashboard as real time as possible. There is a query for every refresh of the dashboard.
+
+For 1,000 users viewing clickstream analytics with a 5-second dashboard refresh rate, you will get the following formula:
+
+<img width="547" alt="image" src="https://github.com/user-attachments/assets/d1005cbd-c3f0-4470-9c22-72d2a523b495">
+
+This means the query needs to have a latency of 5 milliseconds. We can convert 200 queries/second to milliseconds/query by taking the inverse:
+
+<img width="311" alt="image" src="https://github.com/user-attachments/assets/5bb70e6d-7223-4c78-9da0-125f1a653643">
+
+But this might not be the case for every type of query, so it needs to be tested.
+
+### Serving Analytical Results
+
+QPS is a metric that indicates good performance for queries that are invoked by users or applications. Especially for real-time use cases, analytical queries will have a high refresh rate. High refresh rates can strain the OLAP system. This is also called polling.
+
+#### Synchronous vs. Asynchronous Queries
+
+**Synchronous (Pull) Queries**
+- Also known as pull queries, where the client submits a query and pulls data from the OLAP system.
+- Follows a request-response pattern using a SQL dialect specific to the OLAP system.
+- Provides flexibility, allowing the client to retrieve data on demand, reducing unnecessary network traffic.
+- More resource-efficient as the server doesn't need to continuously monitor and push data, but frequent pull queries can increase resource consumption.
+
+**Asynchronous (Push) Queries**
+- Data is pushed to clients when updates occur, eliminating the need for polling.
+- Often served from topics in streaming platforms where clients subscribe to real-time updates.
+- Enables real-time communication, aligning well with event-driven architectures.
+- Typically uses Server-Sent Events (SSEs) for unidirectional updates or WebSockets for bidirectional communication.
+  - SSEs: Simpler, unidirectional, event-driven updates using HTTP protocol.
+  - WebSockets: Supports bidirectional, low-latency communication using a specialized protocol.
+
+#### Push vs. Pull Queries
+
+- Push Queries: Provide real-time updates with lower latency, suitable for event-driven architectures.
+- Pull Queries: Offer flexibility and control over data retrieval, beneficial for reducing unnecessary network traffic and in cases where data changes infrequently.
+- Hybrid Approach: Combines both methods by initially fetching data with a pull query and then subscribing to changes for real-time updates.
+
+#### Limitations in OLAP Data Stores
+- OLAP data stores typically do not support asynchronous push queries due to their focus on serving aggregated analytical results rather than raw data.
+
+
+## [Advanced RAG designs](https://learning.oreilly.com/course/master-retrieval-augmented-generation/9781836646075/)
+
+Found this course in my recommended on the O'Reilly platform so decided to give it a go.
+
+The instructor introduced RAG, and a naive RAG system
+
+![image](https://github.com/user-attachments/assets/3fc373c6-cac1-4dc2-a116-32db146227ec)
+
+And its drawbacks:
+* limited contextual understanding: it focuses on keyword matching or basic semantic search (retrieving irrelevant or partially relevant docs)
+* inconsistent relevance and quality of retrieved docs: varying in quality and relevant docs (poor quality inputs for the gen model)
+* poor integration between retrieval and generation: the retriever and generator might not work in sync
+* inefficient handling of large-scale data: scaling issues (taking too long to retrieve relevant docs, or miss critical info due to bad indexing)
+* lack of robustness and adaptability: not adaptable to changing contexts or user needs without significant manual intervention
+
+### Advanced RAG benefits
+
+1. Pre-retrieval
+  * improvement of the indexing structure and user's query
+  * improvement of data details, organising indexes better, adding extra info, aligning things correctly
+2. Post-retrieval
+  * combine pre-retrieval data with the original query
+    * re-ranking to highlight the most important content
+   
+#### Query expansion with generated answers
+
+Generate partial answers to the query to get relevant context
+
+
+<img width="829" alt="image" src="https://github.com/user-attachments/assets/e956f585-feb2-4ca8-b281-a2e15e986398">
+
+Sample code:
+
+```python
+from helper_utils import project_embeddings, word_wrap
+from pypdf import PdfReader
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+
+from pypdf import PdfReader
+import umap
+
+
+# Load environment variables from .env file
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_API_KEY")
+client = OpenAI(api_key=openai_key)
+
+reader = PdfReader("data/microsoft-annual-report.pdf")
+pdf_texts = [p.extract_text().strip() for p in reader.pages]
+
+# Filter the empty strings
+pdf_texts = [text for text in pdf_texts if text]
+# print(
+#     word_wrap(
+#         pdf_texts[0],
+#         width=100,
+#     )
+# )
+
+# split the text into smaller chunks
+
+
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter,
+    SentenceTransformersTokenTextSplitter,
+)
+
+character_splitter = RecursiveCharacterTextSplitter(
+    separators=["\n\n", "\n", ". ", " ", ""], chunk_size=1000, chunk_overlap=0
+)
+character_split_texts = character_splitter.split_text("\n\n".join(pdf_texts))
+
+# print(word_wrap(character_split_texts[10]))
+# print(f"\nTotal chunks: {len(character_split_texts)}")
+
+token_splitter = SentenceTransformersTokenTextSplitter(
+    chunk_overlap=0, tokens_per_chunk=256
+)
+token_split_texts = []
+for text in character_split_texts:
+    token_split_texts += token_splitter.split_text(text)
+
+# print(word_wrap(token_split_texts[10]))
+# print(f"\nTotal chunks: {len(token_split_texts)}")
+
+
+import chromadb
+from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
+
+embedding_function = SentenceTransformerEmbeddingFunction()
+# print(embedding_function([token_split_texts[10]]))
+
+chroma_client = chromadb.Client()
+chroma_collection = chroma_client.create_collection(
+    "microsoft-collection", embedding_function=embedding_function
+)
+
+# extract the embeddings of the token_split_texts
+ids = [str(i) for i in range(len(token_split_texts))]
+chroma_collection.add(ids=ids, documents=token_split_texts)
+chroma_collection.count()
+
+query = "What was the total revenue for the year?"
+
+
+results = chroma_collection.query(query_texts=[query], n_results=5)
+retrieved_documents = results["documents"][0]
+
+# for document in retrieved_documents:
+#     print(word_wrap(document))
+#     print("\n")
+
+
+def augment_query_generated(query, model="gpt-3.5-turbo"):
+    prompt = """You are a helpful expert financial research assistant. 
+   Provide an example answer to the given question, that might be found in a document like an annual report."""
+    messages = [
+        {
+            "role": "system",
+            "content": prompt,
+        },
+        {"role": "user", "content": query},
+    ]
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+    )
+    content = response.choices[0].message.content
+    return content
+
+
+original_query = "What was the total profit for the year, and how does it compare to the previous year?"
+hypothetical_answer = augment_query_generated(original_query)
+
+joint_query = f"{original_query} {hypothetical_answer}"
+print(word_wrap(joint_query))
+
+results = chroma_collection.query(
+    query_texts=joint_query, n_results=5, include=["documents", "embeddings"]
+)
+retrieved_documents = results["documents"][0]
+
+
+# for doc in retrieved_documents:
+#     print(word_wrap(doc))
+#     print("")
+
+embeddings = chroma_collection.get(include=["embeddings"])["embeddings"]
+umap_transform = umap.UMAP(random_state=0, transform_seed=0).fit(embeddings)
+projected_dataset_embeddings = project_embeddings(embeddings, umap_transform)
+
+
+retrieved_embeddings = results["embeddings"][0]
+original_query_embedding = embedding_function([original_query])
+augmented_query_embedding = embedding_function([joint_query])
+
+projected_original_query_embedding = project_embeddings(
+    original_query_embedding, umap_transform
+)
+projected_augmented_query_embedding = project_embeddings(
+    augmented_query_embedding, umap_transform
+)
+projected_retrieved_embeddings = project_embeddings(
+    retrieved_embeddings, umap_transform
+)
+
+import matplotlib.pyplot as plt
+
+# Plot the projected query and retrieved documents in the embedding space
+plt.figure()
+
+plt.scatter(
+    projected_dataset_embeddings[:, 0],
+    projected_dataset_embeddings[:, 1],
+    s=10,
+    color="gray",
+)
+plt.scatter(
+    projected_retrieved_embeddings[:, 0],
+    projected_retrieved_embeddings[:, 1],
+    s=100,
+    facecolors="none",
+    edgecolors="g",
+)
+plt.scatter(
+    projected_original_query_embedding[:, 0],
+    projected_original_query_embedding[:, 1],
+    s=150,
+    marker="X",
+    color="r",
+)
+plt.scatter(
+    projected_augmented_query_embedding[:, 0],
+    projected_augmented_query_embedding[:, 1],
+    s=150,
+    marker="X",
+    color="orange",
+)
+
+plt.gca().set_aspect("equal", "datalim")
+plt.title(f"{original_query}")
+plt.axis("off")
+plt.show()  # display the plot
+```
+
+Visualisation:
+
+<img width="755" alt="image" src="https://github.com/user-attachments/assets/2541c9b6-9096-4b69-b02b-a85c5920db92">
+
+Red cross: original query
+
+Orange cross: modified
+
+Circles: retrieved documents
+
+#### Query expansion with multiple queries
+
+Use an LLM to generate additional queries that might help getting the most relevant answer
+
+<img width="833" alt="image" src="https://github.com/user-attachments/assets/d2cb7579-29d6-4fca-84ae-0e04bab650df">
+
+Python code:
+
+```python
+from helper_utils import project_embeddings, word_wrap
+from pypdf import PdfReader
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+
+
+from pypdf import PdfReader
+import numpy as np
+import umap
+
+
+# Load environment variables from .env file
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_API_KEY")
+client = OpenAI(api_key=openai_key)
+
+reader = PdfReader("data/microsoft-annual-report.pdf")
+pdf_texts = [p.extract_text().strip() for p in reader.pages]
+
+# Filter the empty strings
+pdf_texts = [text for text in pdf_texts if text]
+
+
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter,
+    SentenceTransformersTokenTextSplitter,
+)
+
+character_splitter = RecursiveCharacterTextSplitter(
+    separators=["\n\n", "\n", ". ", " ", ""], chunk_size=1000, chunk_overlap=0
+)
+character_split_texts = character_splitter.split_text("\n\n".join(pdf_texts))
+
+
+token_splitter = SentenceTransformersTokenTextSplitter(
+    chunk_overlap=0, tokens_per_chunk=256
+)
+
+token_split_texts = []
+for text in character_split_texts:
+    token_split_texts += token_splitter.split_text(text)
+
+# now we import chromadb and the SentenceTransformerEmbeddingFunction
+import chromadb
+from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
+
+
+embedding_function = SentenceTransformerEmbeddingFunction()
+# print(embedding_function([token_split_texts[10]]))
+
+# we then instantiate the Chroma client and create a collection called "microsoft-collection"
+chroma_client = chromadb.Client()
+chroma_collection = chroma_client.create_collection(
+    "microsoft-collection", embedding_function=embedding_function
+)
+
+# extract the embeddings of the token_split_texts
+ids = [str(i) for i in range(len(token_split_texts))]
+
+chroma_collection.add(ids=ids, documents=token_split_texts)
+chroma_collection.count()
+
+query = "What was the total revenue for the year?"
+
+results = chroma_collection.query(query_texts=[query], n_results=5)
+retrieved_documents = results["documents"][0]
+
+# for document in retrieved_documents:
+#     print(word_wrap(document))
+#     print("\n")
+
+
+def generate_multi_query(query, model="gpt-3.5-turbo"):
+
+    prompt = """
+    You are a knowledgeable financial research assistant. 
+    Your users are inquiring about an annual report. 
+    For the given question, propose up to five related questions to assist them in finding the information they need. 
+    Provide concise, single-topic questions (withouth compounding sentences) that cover various aspects of the topic. 
+    Ensure each question is complete and directly related to the original inquiry. 
+    List each question on a separate line without numbering.
+                """
+
+    messages = [
+        {
+            "role": "system",
+            "content": prompt,
+        },
+        {"role": "user", "content": query},
+    ]
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+    )
+    content = response.choices[0].message.content
+    content = content.split("\n")
+    return content
+
+
+original_query = (
+    "What details can you provide about the factors that led to revenue growth?"
+)
+aug_queries = generate_multi_query(original_query)
+
+# 1. First step show the augmented queries
+for query in aug_queries:
+    print("\n", query)
+
+# 2. concatenate the original query with the augmented queries
+joint_query = [
+    original_query
+] + aug_queries  # original query is in a list because chroma can actually handle multiple queries, so we add it in a list
+
+# print("======> \n\n", joint_query)
+
+results = chroma_collection.query(
+    query_texts=joint_query, n_results=5, include=["documents", "embeddings"]
+)
+retrieved_documents = results["documents"]
+
+# Deduplicate the retrieved documents
+unique_documents = set()
+for documents in retrieved_documents:
+    for document in documents:
+        unique_documents.add(document)
+
+
+# output the results documents
+for i, documents in enumerate(retrieved_documents):
+    print(f"Query: {joint_query[i]}")
+    print("")
+    print("Results:")
+    for doc in documents:
+        print(word_wrap(doc))
+        print("")
+    print("-" * 100)
+
+embeddings = chroma_collection.get(include=["embeddings"])["embeddings"]
+umap_transform = umap.UMAP(random_state=0, transform_seed=0).fit(embeddings)
+projected_dataset_embeddings = project_embeddings(embeddings, umap_transform)
+
+# 4. We can also visualize the results in the embedding space
+original_query_embedding = embedding_function([original_query])
+augmented_query_embeddings = embedding_function(joint_query)
+
+
+project_original_query = project_embeddings(original_query_embedding, umap_transform)
+project_augmented_queries = project_embeddings(
+    augmented_query_embeddings, umap_transform
+)
+
+retrieved_embeddings = results["embeddings"]
+result_embeddings = [item for sublist in retrieved_embeddings for item in sublist]
+
+projected_result_embeddings = project_embeddings(result_embeddings, umap_transform)
+
+import matplotlib.pyplot as plt
+
+
+# Plot the projected query and retrieved documents in the embedding space
+plt.figure()
+plt.scatter(
+    projected_dataset_embeddings[:, 0],
+    projected_dataset_embeddings[:, 1],
+    s=10,
+    color="gray",
+)
+plt.scatter(
+    project_augmented_queries[:, 0],
+    project_augmented_queries[:, 1],
+    s=150,
+    marker="X",
+    color="orange",
+)
+plt.scatter(
+    projected_result_embeddings[:, 0],
+    projected_result_embeddings[:, 1],
+    s=100,
+    facecolors="none",
+    edgecolors="g",
+)
+plt.scatter(
+    project_original_query[:, 0],
+    project_original_query[:, 1],
+    s=150,
+    marker="X",
+    color="r",
+)
+
+plt.gca().set_aspect("equal", "datalim")
+plt.title(f"{original_query}")
+plt.axis("off")
+plt.show()  # display the plot
+```
+
+Visualisation 
+
+<img width="699" alt="image" src="https://github.com/user-attachments/assets/15c9855a-e8ac-4748-950a-117b24eba9b9">
+
+We can see again the original(orange) and modified(red) queries and the retrieved docs.
+
+However some downsides are clear:
+
+* retrieving lots of documents - generated queries might not be always relevant or useful
+* results might not always be relevant or useful
+
+If we want to have many retrieved results, we can use re-ranking to handle that.
+
+#### Re-ranking
+
+Apply a sophisticated process to re-evaluate and reorder the initial retrieved documents
+
+<img width="827" alt="image" src="https://github.com/user-attachments/assets/5ca14240-c524-4440-9606-7757318c887b">
+
+Python code:
+
+```python
+from helper_utils import word_wrap, load_chroma
+from pypdf import PdfReader
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+
+
+from pypdf import PdfReader
+import numpy as np
+
+from langchain_community.document_loaders import PyPDFLoader
+
+
+# Load environment variables from .env file
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_API_KEY")
+client = OpenAI(api_key=openai_key)
+
+
+import chromadb
+from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
+
+embedding_function = SentenceTransformerEmbeddingFunction()
+
+
+reader = PdfReader("data/microsoft-annual-report.pdf")
+pdf_texts = [p.extract_text().strip() for p in reader.pages]
+
+# Filter the empty strings
+pdf_texts = [text for text in pdf_texts if text]
+
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter,
+    SentenceTransformersTokenTextSplitter,
+)
+
+character_splitter = RecursiveCharacterTextSplitter(
+    separators=["\n\n", "\n", ". ", " ", ""], chunk_size=1000, chunk_overlap=0
+)
+character_split_texts = character_splitter.split_text("\n\n".join(pdf_texts))
+
+token_splitter = SentenceTransformersTokenTextSplitter(
+    chunk_overlap=0, tokens_per_chunk=256
+)
+
+token_split_texts = []
+for text in character_split_texts:
+    token_split_texts += token_splitter.split_text(text)
+
+chroma_client = chromadb.Client()
+chroma_collection = chroma_client.get_or_create_collection(
+    "microsoft-collect", embedding_function=embedding_function
+)
+
+# extract the embeddings of the token_split_texts
+ids = [str(i) for i in range(len(token_split_texts))]
+
+chroma_collection.add(ids=ids, documents=token_split_texts)
+
+count = chroma_collection.count()
+
+query = "What has been the investment in research and development?"
+
+results = chroma_collection.query(
+    query_texts=query, n_results=10, include=["documents", "embeddings"]
+)
+
+retrieved_documents = results["documents"][0]
+
+for document in results["documents"][0]:
+    print(word_wrap(document))
+    print("")
+
+from sentence_transformers import CrossEncoder
+
+cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+
+pairs = [[query, doc] for doc in retrieved_documents]
+scores = cross_encoder.predict(pairs)
+
+print("Scores:")
+for score in scores:
+    print(score)
+
+print("New Ordering:")
+for o in np.argsort(scores)[::-1]:
+    print(o + 1)
+
+original_query = (
+    "What were the most important factors that contributed to increases in revenue?"
+)
+
+generated_queries = [
+    "What were the major drivers of revenue growth?",
+    "Were there any new product launches that contributed to the increase in revenue?",
+    "Did any changes in pricing or promotions impact the revenue growth?",
+    "What were the key market trends that facilitated the increase in revenue?",
+    "Did any acquisitions or partnerships contribute to the revenue growth?",
+]
+
+# concatenate the original query with the generated queries
+queries = [original_query] + generated_queries
+
+
+results = chroma_collection.query(
+    query_texts=queries, n_results=10, include=["documents", "embeddings"]
+)
+retrieved_documents = results["documents"]
+
+# Deduplicate the retrieved documents
+unique_documents = set()
+for documents in retrieved_documents:
+    for document in documents:
+        unique_documents.add(document)
+
+unique_documents = list(unique_documents)
+
+pairs = []
+for doc in unique_documents:
+    pairs.append([original_query, doc])
+
+scores = cross_encoder.predict(pairs)
+
+print("Scores:")
+for score in scores:
+    print(score)
+
+print("New Ordering:")
+for o in np.argsort(scores)[::-1]:
+    print(o)
+# ====
+top_indices = np.argsort(scores)[::-1][:5]
+top_documents = [unique_documents[i] for i in top_indices]
+
+# Concatenate the top documents into a single context
+context = "\n\n".join(top_documents)
+
+
+# Generate the final answer using the OpenAI model
+def generate_multi_query(query, context, model="gpt-3.5-turbo"):
+
+    prompt = f"""
+    You are a knowledgeable financial research assistant. 
+    Your users are inquiring about an annual report. 
+    """
+
+    messages = [
+        {
+            "role": "system",
+            "content": prompt,
+        },
+        {
+            "role": "user",
+            "content": f"based on the following context:\n\n{context}\n\nAnswer the query: '{query}'",
+        },
+    ]
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+    )
+    content = response.choices[0].message.content
+    content = content.split("\n")
+    return content
+
+
+res = generate_multi_query(query=original_query, context=context)
+print("Final Answer:")
+print(res)
+```
+
+For a next RAG project I will definitely consider these techniques ^^
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From afff7219a0ee530a5e4e7aead1a0a7f87bf1f00d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:45:59 +0900
Subject: [PATCH 406/638] 245-save

---
 _posts/2024-09-01-day-245.md | 331 +++++++++++++++++++++++++++++++++++
 1 file changed, 331 insertions(+)
 create mode 100644 _posts/2024-09-01-day-245.md

diff --git a/_posts/2024-09-01-day-245.md b/_posts/2024-09-01-day-245.md
new file mode 100644
index 000000000..5ec788bd9
--- /dev/null
+++ b/_posts/2024-09-01-day-245.md
@@ -0,0 +1,331 @@
+---
+layout: post
+title: (Day 245) 
+categories: []
+---
+
+# Hello :) Today is Day 245!
+A quick summary of today:
+
+
+## Streaming Databases Chapter 4: Materialized Views
+
+### Views, Materialized Views, and Incremental Updates
+
+Materialized views separate the processing logic for generating query results from the main processing pipeline, leading to more modular and manageable code in stream processing systems.
+
+#### Traditional Views vs. Materialized Views
+
+- Traditional Views:
+  - Defined by SQL statements executed upon client selection.
+  - Results aren't stored, leading to higher query latency as data is not preprocessed.
+  - Analogy: A smart chipmunk, Simon, counts nuts in a yard every time he's asked, resulting in delays.
+
+- Materialized Views:
+  - Store precomputed results, reducing latency.
+  - Analogy: Simon counts the nuts and stores the total, while a less-smart chipmunk, Alvin, reads and returns the stored value
+  - Incremental changes are monitored, avoiding full recomputation.
+
+#### Incremental Changes
+
+Incremental updates refer to small, targeted changes to data instead of recomputing everything from scratch. This approach keeps data up-to-date with less computational overhead. Simon continuously tracks changes, similar to streaming settings.
+
+### Change Data Capture
+
+Materialized views and Change Data Capture (CDC) are closely related. Materialized views handle precomputing by monitoring incremental changes and storing the results, while CDC provides these incremental changes by capturing data modifications from the Write-Ahead Log (WAL) in an OLTP database. This allows materialized views to preprocess the incremental changes captured by CDC.
+
+#### Extended Chipmunk Analogy
+
+- Attributes of Nuts:
+  - Color
+  - Location (latitude, longitude)
+- Nuts can change color, move, or be removed by other animals. Simon updates the list of nuts in the box to reflect these changes, ensuring that any client querying the list sees the latest nut status.
+
+- WAL Replication:
+  - The WAL from the primary/OLTP database is replicated to create a database replica.
+  - A CDC connector writes the WAL to a topic in a streaming platform, allowing other systems to subscribe.
+- Consumption and Replication:
+  - Sink connectors consume from the topic and create replicas in other database systems.
+  - Stream processors can replicate the database in their cache.
+
+#### Use Case Focus
+
+This technique supports real-time processing by allowing the replication of the original OLTP database in downstream data stores or stream processing engines without relying on batch processing.
+
+### Push Versus Pull Queries
+
+#### Chipmunk Analogy Extension
+
+- Push Query (Simon): Simon watches for changes in the nut count asynchronously and updates the box with the results. This represents a push query that runs without client intervention.
+- Pull Query (Alvin): Alvin retrieves the results from the box synchronously when queried by a client, representing a pull query.
+
+#### Trade-offs Between Push and Pull Queries
+
+- Push Queries:
+  - Lower latency, ideal for applications needing real-time results.
+  - Limited flexibility; clients can't ask complex questions (e.g., averages, joins).
+
+- Pull Queries:
+  - More flexible, allowing for complex queries and deeper insights.
+  - Higher latency due to the additional processing required.
+
+#### Balancing Push and Pull Queries
+
+- Materialized Views:
+  - Serve as a balance between the heavy lifting of push queries and the flexibility of pull queries.
+  - Adjusting the balance:
+    - Moving towards push queries improves performance but reduces flexibility.
+    - Moving towards pull queries increases flexibility but at the cost of higher latency.
+
+![image](https://github.com/user-attachments/assets/33143fd4-99ce-48cc-8638-7293c64a9d5b)
+![image](https://github.com/user-attachments/assets/d9a95a03-d850-4196-be11-7a024b43402c)
+
+#### Combining Flexibility and Low Latency
+
+- Materialized Views with WAL Emission:
+  - A client submits a push query, which creates a materialized view.
+  - The client subscribes to changes in the materialized view, receiving updates in real-time.
+  - This approach allows for ad hoc queries with low latency, without needing separate push and pull queries.
+
+#### Challenges and Solution
+
+- Current Dilemma: Push and pull queries often run in separate systems and are authored by different teams (streaming engineers vs. analysts).
+- Solution: Streaming databases that combine stream processing and databases, using the same SQL engine for both real-time and stored data.
+
+![image](https://github.com/user-attachments/assets/c28456ff-5b1b-4e89-98cb-4f3c34bcf100)
+
+The above shows the path by which data in an OLTP database travels to an RTOLAP system for serving to a client. Let’s look closer at this architecture:
+
+1. The entities are represented as tables in the OLTP database following domain-driven design.
+2. The application inserts, updates, or deletes records in the table. These changes are recorded in the database WAL.
+3. A CDC connector reads the WAL and writes the changes to a topic in a streaming platform. The streaming platform externalizes the OLTP WAL by publishing the changes into topics/partitions that mimic the WAL construct. These can be read by consumers to build replicas of the tables from the original OLTP database.
+4. The stream processor is one such system that reads the topic and builds internal replicas of tables by using materialized views. As the materialized view gets updated asynchronously, it outputs its changes into another topic.
+5. The RTOLAP data store reads the topic that contains the output of the materialized view and optimizes the data for analytical queries.
+
+### CDC and Upsert
+
+The term upsert is a portmanteau of the words update and insert to describe the logic an application employs when inserting and/or updating a database table. Upsert describes a logic that involves an application checking to see if a record exists in a database table. If the record exists by searching for its primary key, the record then invokes an update statement. Otherwise, if the record does not exist, the application invokes an insert statement to add the record to the table.
+
+Upsert operations can indirectly improve select query performance and accuracy in certain scenarios. While upserts themselves are primarily focused on data modification, they can have positive impacts on select query performance and accuracy by maintaining data integrity and optimizing data storage. Here’s how upserts can contribute to these improvements:
+
+Data integrity and accuracy
+* Upserts help maintain data integrity by preventing duplicate records and ensuring the data is accurate and consistent. When select queries retrieve data from a database with proper upsert operations, they are more likely to return accurate and reliable information.
+
+Simplified pull queries
+* Selecting from a table with proper upsert operations simplifies the queries upon lookup. Having to perform deduplication or filtering for the latest records complicates the SQL and adds latency to its execution.
+
+Upsert operations, by definition, only support inserts and updates, deletes tend to be omitted. Some systems will implement upsert to also include delete logic. Others, like Apache Pinot, will only flag a deleted record so that its previous versions can be recovered. In these cases, it’s important to use the RTOLAP implementation of upsert, which requires the RTOLAP to read directly from the output topic. Some RTOLAPs may not expose the delete feature, and the work would have to be done in the stream processor.
+
+In summary, there are two locations where the upsert logic can be implemented—in the RTOLAP system or the stream processor. The simpler and preferred approach is to have the RTOLAP read from the output topic and apply the upsert logic itself. The output topic also provides a buffer in cases where the stream processor produces data faster than the RTOLAP can consume.
+
+Upsert highlights the pain of having two real-time systems grapple over or dodge ownership of such complex logic. These pains will create further contention between data engineers and analytical end users.
+
+### Joining Streams
+
+#### Stream Types and Constructs
+
+- Append-Only Streams:
+  - Streams where only inserts are allowed (e.g., click events).
+  - Represented as append tables in streaming systems, not backed by a state store.
+  - Not stored in materialized views due to potential storage issues from continuous growth.
+
+- Change Streams:
+  - Streams containing change events (e.g., inserts, updates, deletes).
+  - Represented as change tables in streaming systems, backed by a state store (materialized view).
+
+#### Streaming Platforms: Topic Types
+
+- Append Topics:Contain append-only data.
+- Change Topics: Contain change events or CDC events (sometimes referred to as "table topics" in Kafka).
+
+#### Joining Streams and SQL
+
+- SQL is used to define joins and transformations, making it easier to combine streams and databases.
+- Using the same SQL engine for both data in motion (streams) and data at rest (databases) integrates stream processing and database functionalities, leading to the concept of a streaming database
+
+#### Clickstream Use Case
+
+![image](https://github.com/user-attachments/assets/39bff41d-620b-498b-8f6c-7901dcfee674)
+
+1. A customer updates their information.
+
+  * a: The information is saved in an OLTP database.
+
+  * b: A CDC process runs on the OLTP database, capturing changes to the CUSTOMERS table and writing them into a CDC topic. This topic is a compacted topic that can be considered a replica of the CUSTOMERS table. This will allow for other systems to build their replicas of the CUSTOMERS table.
+
+2. The same customer clicks on a product on an e-commerce application.
+
+3. The click event is written into a topic. We don’t write click events into an OLTP database because click events are only inserts. Capturing them in an OLTP database might eventually cause the database to run out of storage.
+
+4. The stream processor reads from the CDC and click topics.
+
+  * a: These are the messages from the CUSTOMERS change table topic in the stream processor. They are stored in a state store whose size depends on the window size (or, in the case of, for example, Kafka Streams or ksqlDB, fully stored in a KTable).
+
+  * b: These are the messages from the CLICK_EVENTS append table topic in the stream processor.
+
+  * c: A left-join is executed between the CLICK_EVENTS append table messages and the CUSTOMERS change table messages. The result of the join is CLICK_EVENTS enriched with their corresponding CUSTOMER information (if it exists).
+
+5. The stream processor writes its output to the topics below.
+
+  * a: This is a change topic and contains the CDC CUSTOMER changes. This would be a redundant topic since the topic in 1b contains the same data. We keep it here to keep the diagram balanced.
+
+  * b: This is an append topic that contains the original CLICK_EVENT data enriched with the CUSTOMER data.
+
+6. Topics are pulled into the RTOLAP data store for real-time serving.
+
+  * a: This is a replica of the original CUSTOMERS table in the OLTP database and built from the change topic.
+
+  * b: This contains the enriched CLICK_EVENTS data.
+
+7. The user invokes queries against the RTOLAP data store.
+
+  * a: The user can query the CUSTOMERS table directly.
+
+  * b: The user can query the enriched CLICK_EVENTS data without having to join the data themselves, as the join has already been done in the stream processor.
+
+## Chapter 5: Introduction to Streaming Databases****
+
+### Identifying the Streaming Database
+
+![image](https://github.com/user-attachments/assets/69637b39-3299-47ef-a102-2df4b9e67a28)
+
+* The database can be one of the three types of data stores that we talked about so far: OLTP, RTOLAP, and the internal state stores in a stream processor. The differences between them dictate how data is stored and queried.
+* The topic is a construct that mimics the WAL in an OLTP database. Topics publish streams of data to other databases and stream processors.
+* Stream processors are the applications that transform streams of data. They hold an internal state store.
+* The materialized view is a process that precomputes a result and stores it in a database. Materialized views are created in a database or stream processor, both of which need to have a persistence layer.
+
+### SQL Expressivity
+
+SQL expressivity refers to how well SQL can succinctly and effectively represent complex data manipulations and queries with concise syntax. It measures SQL's ability to capture the intent of a query or operation in a way that is easy to understand and maintain.
+
+#### Challenges in Merging SQL Engines
+
+Merging SQL engines between a **stream processor** and an **OLAP/OLTP database** can introduce several challenges due to differences in design, use cases, and performance characteristics:
+
+1. Performance Mismatch: Stream processors handle high-velocity, real-time data, while OLAP databases focus on complex analytical queries on historical data. A combined SQL engine might struggle to balance these conflicting requirements.
+
+2. Latency: Stream processing demands low latency, whereas OLAP databases prioritize query optimization. Achieving both in a single engine is challenging.
+
+3. Resource Allocation: Stream processors need resources for real-time data, potentially causing contention with OLAP queries, which require significant compute and memory resources.
+
+4. Data Modeling Differences: Stream processors work with raw or semistructured data, while OLAP databases need structured, preprocessed data. Merging the engines may cause conflicts in data modeling.
+
+5. Data Consistency: Stream processors operate on data in motion, and OLAP databases work with data at rest. Ensuring consistency across these states can be complex.
+
+6. Complexity: Combining stream processing and OLAP capabilities can lead to increased system complexity, affecting maintainability, debugging, and stability.
+
+7. Data Volume and Retention: Stream processors typically have shorter data retention due to high volume, while OLAP databases store data long-term. Managing retention and integration is challenging.
+
+8. Query Optimizations: OLAP databases offer advanced query optimizations that stream processors may lack, leading to suboptimal performance for analytical queries.
+
+9. Schema Evolution: Stream processors may handle schema evolution more flexibly than OLAP databases, which require well-defined schemas. This discrepancy can create issues when merging engines.
+
+10. Maintenance and Updates: Managing a combined SQL engine that handles both streaming and OLAP workloads is more challenging, as updates must cater to both use cases.
+
+#### Mitigating Pitfalls
+
+To mitigate these challenges, careful architectural planning, thorough performance testing, and a deep understanding of specific use cases are essential.
+
+#### Merging OLTP and Stream Processor SQL Engines
+
+Merging SQL engines between **OLTP** and a stream processor can be easier compared to OLAP due to the following shared characteristics:
+
+- Data Format: OLTP databases use a row-based model, aligning with the row-based format of stream processors, facilitating smoother integration.
+- Real-Time Nature: Both systems handle real-time data, simplifying SQL engine merging.
+- Transaction Handling: Both involve transactional processing, aiding in better integration for data consistency and updates.
+- Event-Driven: Stream processors' event-driven nature aligns with real-time updates in OLTP databases, making integration easier.
+
+### Streaming Debuggability
+
+Data engineers need to verify the logic of their SQL when writing data pipelines. Debugging becomes challenging when materialized views span across multiple distributed systems (stream processor, topic, and OLAP). Although theoretically possible by analyzing input and output topics alone, effective debugging requires examining both topics and external databases simultaneously.
+
+#### Advantages of Streaming Databases for Debugging
+
+Streaming databases simplify debugging compared to classical stream processors by offering advanced materialized views. These views are persisted in a row-based or column-based store, making it easier to verify results in one place. Additionally, indexed data in streaming databases allows for faster, more complex ad hoc queries. In contrast, stream processors like Flink require output to be written to a database before validating results through queries.
+
+#### Key Advantages:
+1. Familiar SQL Interface: Streaming databases often use SQL-like languages, making debugging more straightforward for those familiar with SQL.
+2. Simpler Logic: Higher-level abstractions simplify stream processing tasks, making the logic easier to debug.
+3. Integrated Ecosystem: These databases combine stream processing and storage in one system, providing a holistic view of the data pipeline and aiding in debugging.
+4. Built-in Optimizations: Common stream processing patterns are often optimized, improving performance and reducing the need for complex debugging.
+5. Easier Deployment: Designed for ease of deployment, streaming databases reduce potential deployment-related debugging issues.
+
+#### Limitations of SQL
+
+While SQL offers abstraction, it may not always be the best tool for debugging in performance-critical situations. Lower-level domain-specific languages (DSLs) like Kafka Streams and Flink's DataStream API offer more expressibility. Although SQL-based streaming databases support user-defined functions (UDFs) to extend functionality, they have limitations.
+
+Moreover, inspecting the actual execution plan derived by stream processing systems (e.g., aggregation operators in a `GROUP BY` statement) is still a challenge, as tooling for this purpose is underdeveloped.
+
+### Streaming Database Implementations
+
+| Name        | License                      | State Store Implementation        | Use Cases                             |
+|-------------|------------------------------|----------------------------------|---------------------------------------|
+| ksqlDB       | Confluent Community License   | RocksDB (LSM tree key-value storage) | CQRS, push queries                    |
+| RisingWave   | Apache 2                     | Row-based                         | CQRS, push queries, single row lookups|
+| Materialize  | Business Source License (BSL) | Row-based                         | CQRS, push queries, single row lookups|
+| Timeplus (Proton) | Apache 2                  | Column-based                      | Analytical push and pull queries      |
+
+
+### Streaming Database Architecture
+
+![image](https://github.com/user-attachments/assets/ce481e7c-e6cb-4505-bf0a-6898d2ae5ed9)
+
+### ELT with Streaming Databases
+
+ELT (extract, load, transform) data pipelines do not support real-time use cases because the transformation occurs in the destination database. The database, in this case, places the streaming data at rest, which forces batch semantics for all downstream processing.
+
+However, if the destination database using ELT is a streaming database, then the pipeline can be considered still in real time. This integration between the “loading” and “transformation” parts of ELT is mediated by a topic on a streaming platform from which the streaming database consumes the data.
+
+There is a large ecosystem that supports ELT solutions, for example, dbt. In combination with streaming databases, these tools can support real-time ELT for the first time. And because streaming databases behave, on the surface, like databases and not so much like a stream processor, ELT with streaming databases can actually be implemented by the same teams who have previously worked on ELT in a data warehouse. In this vein, a lot of ELT jobs that now run later in the pipeline (in the data warehouse or lakehouse) can be moved to the real-time streaming layer.
+
+### Summary of Streaming Databases
+
+The term **streaming database** represents a convergence of stream processing and traditional databases. While databases are typically associated with batch processing and data at rest, streaming databases integrate streaming and batching, handling both data in motion and data at rest.
+
+#### Key Concepts
+
+- Convergence: Streaming databases merge the concepts of streaming and batch processing, reintroducing elements like the Write-Ahead Log (WAL) and materialized views into the database environment.
+
+- SQL Engines: Traditional database SQL engines handle data at rest. Streaming databases allow materialized views to run asynchronously, enabling SQL to process both data at rest and in motion.
+
+- Persistence Layer:
+  - ksqlDB: Uses RocksDB with primary key indexes.
+  - Newer Databases: Implement a more database-like persistence layer for efficient querying.
+
+- Query Types:
+  - Push Queries: Handled by the streaming component of the database.
+  - Pull Queries: Executed by the database component. The type of storage (columnar vs. row-based) affects the efficiency of pull queries:
+    - Columnar Storage: Efficient for analytical queries and fast aggregations.
+    - Row-Based Storage: Suited for simple lookups and point queries.
+
+#### Spectrum of Streaming Databases
+
+- Row-Based Databases: Typically handle pull queries driven by applications or events.
+- Column-Based Databases: Often support pull queries invoked by humans or dashboards.
+
+![image](https://github.com/user-attachments/assets/510ff81d-e8d5-4770-86d1-40949f9bb06b)
+
+#### Consistency
+
+A crucial property of streaming databases is consistency. This ensures that data adheres to predefined rules and constraints, maintaining accuracy and reliability throughout transactions. Consistency guarantees that each transaction brings the database from one valid state to another without violating integrity rules.
+
+In the next chapter, we will explore the importance of consistency in both streaming databases and stream processors.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From 9fb95f878234d334837ab3c67b6227cc01f77394 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 2 Sep 2024 22:24:53 +0900
Subject: [PATCH 407/638] finish day 245

---
 _posts/2024-09-01-day-245.md | 413 ++++++++++++++++++++++++++++++++++-
 1 file changed, 408 insertions(+), 5 deletions(-)

diff --git a/_posts/2024-09-01-day-245.md b/_posts/2024-09-01-day-245.md
index 5ec788bd9..4e3cfb4ed 100644
--- a/_posts/2024-09-01-day-245.md
+++ b/_posts/2024-09-01-day-245.md
@@ -1,12 +1,13 @@
 ---
 layout: post
-title: (Day 245) 
-categories: []
+title: (Day 245) Streaming dbs + EvidentlyAI course
+categories: [mlops,data-eng,theory]
 ---
 
 # Hello :) Today is Day 245!
 A quick summary of today:
-
+* read more about Streaming Databases
+* started evidently AI's monitoring and observability course
 
 ## Streaming Databases Chapter 4: Materialized Views
 
@@ -189,7 +190,7 @@ Upsert highlights the pain of having two real-time systems grapple over or dodge
 
 ![image](https://github.com/user-attachments/assets/69637b39-3299-47ef-a102-2df4b9e67a28)
 
-* The database can be one of the three types of data stores that we talked about so far: OLTP, RTOLAP, and the internal state stores in a stream processor. The differences between them dictate how data is stored and queried.
+* The database can be one of the three types of data stores that is talked about so far: OLTP, RTOLAP, and the internal state stores in a stream processor. The differences between them dictate how data is stored and queried.
 * The topic is a construct that mimics the WAL in an OLTP database. Topics publish streams of data to other databases and stream processors.
 * Stream processors are the applications that transform streams of data. They hold an internal state store.
 * The materialized view is a process that precomputes a result and stores it in a database. Materialized views are created in a database or stream processor, both of which need to have a persistence layer.
@@ -309,23 +310,425 @@ The term **streaming database** represents a convergence of stream processing an
 
 A crucial property of streaming databases is consistency. This ensures that data adheres to predefined rules and constraints, maintaining accuracy and reliability throughout transactions. Consistency guarantees that each transaction brings the database from one valid state to another without violating integrity rules.
 
-In the next chapter, we will explore the importance of consistency in both streaming databases and stream processors.
+## Chapter 6: Consistency
+
+If you’re familiar with databases, you take consistency for granted. You know that the results of your queries are going to be consistent with the input data. Now imagine you dare to cross the bridge from the database to the streaming world. Can you bank on similar consistency guarantees here, even with the additional complexity of data arriving late and out of order, as well as the emphasis on low latency and high throughput?
+
+For classical stream processors, the answer is no. They guarantee a weaker form of consistency called eventual consistency. For classical stream processing use cases, often involving aggregations on windowed data, eventual consistency is a perfect fit, and it also enables data pipelines with ultra-low latency, with very high throughput, and at extremely large scale. The problem is, if you come from the database world, eventual consistency can turn out to be a confusing and counterintuitive experience—especially in combination with nonwindowed data.
+
+Some more recent stream processing systems support a stronger form of consistency, where every output is the correct output for a subset of the inputs: internal consistency.1 Of these stream processing systems, RisingWave, Materialize, and Pathway are put on the same job to see whether they allow us to solve the toy example in a way that more closely mirrors the intuitions of a typical database engineer.
+
+### Consistency Versus Latency
+
+#### Key Concepts
+- Processing Time Latency: Time required for a stream processing system to produce any answer to a query.
+- End-to-End Latency: Time required for a stream processing system to produce a consistent answer to a query.
+
+#### Latency Trade-offs
+- Internally consistent stream processing systems generally have higher processing time latency than eventually consistent ones.
+- End-to-end latency is often more important for use cases involving nonwindowed data (e.g., JOINs), where consistency is crucial. In these scenarios, internally consistent systems can achieve consistency in less than a second, unlike eventually consistent systems, which may never reach consistency unless external actions are taken.
+
+#### Proposed Solution
+- Switchable Consistency: Future versions of Flink SQL, ksqlDB, and Proton should allow users to toggle internal consistency:
+  - On: For nonwindowed/unbounded data where ultra-low latency isn’t critical.
+  - Off: For windowed data and classical stream processing use cases requiring ultra-low latency.
+
+This would balance ultra-low latency processing with the need for consistency in different scenarios, easing the transition between database and streaming environments.
+
+### Summary
+- Drawbacks of Eventual Consistency:
+  - Difficulty with nonwindowed data.
+  - Incompatibility with traditional SQL intuitions.
+  
+  These issues hinder the broader adoption of stream processing and streaming technology.
+
+- Benefits of Internally Consistent Systems:
+  - Provide stronger consistency guarantees (e.g., RisingWave, Materialize, Pathway).
+  - Solve complex consistency issues without needing expert intervention, thus democratizing stream processing for users familiar with databases.
+
+While they can’t match eventually consistent systems in processing time latency, internally consistent systems often perform better in end-to-end latency, which is more crucial for many applications.
+
+## [ML monitoring with Evidently](https://learn.evidentlyai.com/)
+
+Some of this material, I learned from the MLOps Zoomcamp, but I will cover/re-watch it for completeness.
+
+### Module 1: Introduction
+
+#### What can go wrong with ML in prod?
+
+![image](https://github.com/user-attachments/assets/4babd1be-79ee-4113-b030-bc70d0b04771)
+![image](https://github.com/user-attachments/assets/40dab51a-1a59-44d5-9a49-51ae7e88a917)
+
+Many things can go wrong once you deploy an ML model to the real world. Here are some examples.
+
+Training-serving skew. Model degrades if training data is very different from production data.
+
+Data quality issues. In most cases, when something is wrong with the model, this is due to data quality and integrity issues. These can be caused by:
+
+* Data processing issues, e.g., broken pipelines or infrastructure updates.
+
+* Data schema changes in the upstream system, third-party APIs, or catalogs.
+
+* Data loss at source when dealing with broken sensors, logging errors, database outages, etc.
+
+**Concept drift.** Gradual concept drift occurs when the target function continuously changes over time, leading to model degradation. If the change is sudden – like the recent pandemic – you’re dealing with sudden concept drift.
+
+**Data drift.** Distribution changes in the input features may signal data drift and potentially cause ML model performance degradation. For example, a significant number of users coming from a new acquisition channel can negatively affect the model trained on user data. Chances are that users from different channels behave differently. To get back on track, the model needs to learn new patterns.
+
+**Underperforming segments.** A model might perform differently on diverse data segments. It is crucial to monitor performance across all segments.
+
+![image](https://github.com/user-attachments/assets/fc8627de-250d-4240-bc25-b163c82ba7e8)
+
+#### What is ML monitoring and observability?
+
+What is ML monitoring?
+
+Machine learning model monitoring is a series of techniques to track and analyze the performance of ML models in production. It helps measure ongoing model quality, detect potential issues, and resolve them on time.
+
+When to monitor?
+
+There are three main scenarios when you need ML model monitoring:
+
+* Models in production. Upon deploying ML models to production, you need to keep tabs on the ongoing performance and business impact.
+
+* Models in shadow deployment. In shadow mode, you track the behavior of a candidate model when predictions are generated but not acted upon (in other words, ML models generate outputs, but these outputs are not used in downstream systems).
+
+* During A/B testing. In this case, you track and compare the results of active candidate models.
+
+_Challenges_
+
+![image](https://github.com/user-attachments/assets/f5a590ad-58e4-4027-9500-fbcb77b4679b)
+
+**ML monitoring:**
+
+* tracks a pre-defined set of metrics,
+
+* helps detect issues (“What happened?”, “Is the system working?”),
+
+* is more reactive (helps to find “known unknowns”).
+
+**ML observability:**
+
+* gives visibility into the system behavior,
+
+* helps understand and analyze root causes (“Why it happened? Where exactly?”),
+
+* is more proactive (helps to find “unknown unknowns”).
+
+![image](https://github.com/user-attachments/assets/bfdf6fe1-6f10-4470-90a6-4671d3fd72ef)
+
+_Who should care about observability and monitoring?_
+
+![image](https://github.com/user-attachments/assets/d9c20844-f173-4577-84b4-0d59bf612b98)
+
+#### ML monitoring metrics
+
+**Software system health**
+
+It doesn’t matter how excellent your model is when the whole ML system is down. To track the overall system health, you can reuse existing monitoring schemes from other production services. Standard software performance metrics include latency, error rate, memory usage, disk usage, etc.
+
+**Data quality and data integrity**
+
+In many cases, model issues stem from issues with the input data. To monitor data quality and integrity, you can keep tabs on metrics like the share of missing values, type mismatch, or range violations for important features. The goal here is to ensure the stability of data pipelines.
+
+**Business KPIs**
+
+![image](https://github.com/user-attachments/assets/438d0589-be96-46d7-8112-d8c80c3b516c)
+
+#### ML monitoring setup
+
+![image](https://github.com/user-attachments/assets/ca0b7441-ea14-4f11-a299-c59a19de8a3f)
+
+![image](https://github.com/user-attachments/assets/3d74a176-ceba-404d-b133-a081f41a34ae)
+
+* Ensure the monitoring setup aligns with the complexity of your use case.
+
+* Consider binding retraining to monitoring, if relevant.
+
+* Use reference datasets to simplify the monitoring process but make sure they are carefully curated.
+
+* Define custom metrics that fit your problem statement and data properties.
+
+#### ML monitoring architecture
+
+![image](https://github.com/user-attachments/assets/a2c06e4e-c16e-40f5-a71b-7c1467662910)
+
+![image](https://github.com/user-attachments/assets/f7d7af61-34fb-4607-beaf-b75fb942c008)
+
+![image](https://github.com/user-attachments/assets/720ff5c9-d7cd-4c6f-b835-4a5ade92e735)
+
+### Module 2: ML monitoring metrics
+
+ML monitoring challenges
+
+![image](https://github.com/user-attachments/assets/de1820f8-f88d-4fb0-80ce-d1c768319a40)
+
+![image](https://github.com/user-attachments/assets/9975bc70-0726-4d9c-b1c0-eb23bcd326e3)
+
+#### Overview of ML quality metrics
+
+![image](https://github.com/user-attachments/assets/0329454f-030e-4e2b-8647-1deef2489e6d)
+
+* Classification quality metrics (accuracy, recall, precision, f1, roc-auc, confusion matrix)
+* Regression quality metrics (mse, mae, mape, etc)
+
+#### Ranking quality metrics
+
+Ranking focuses on the relative order of items rather than their absolute values. Popular examples of ranking problems are search engines and recommender systems.
+
+We need to estimate the order of objects to measure quality in ranking tasks. Some commonly used ranking quality metrics are:
 
+* Cumulative gain helps estimate the cumulative value of recommendations and does not take into account the position of a result in the list.
+* Discounted Cumulative Gain (DCG) gives a penalty when a relevant result is further in the list.
+* Normalized DCG (NDCG) normalizes the evaluation irrespective of the list length.
+* Precision @k is a share of the relevant objects in top-K results.
+* Recall @k is a coverage of all relevant objects in top-K results.
+* Lift @k reflects an improvement over random ranking.
 
+If we work on a recommender system, you might want to consider additional – “beyond accuracy” – metrics that reflect RecSys behavior. Some examples are:
 
+* Serendipity
+* Novelty
+* Diversity
+* Coverage
+* Popularity bias
 
+#### [Practice] Evaluating ML model quality
 
+**Imports**
 
+```python
+import pandas as pd
+import numpy as np
 
+from sklearn import datasets
+from sklearn import ensemble
 
+from evidently import ColumnMapping
+from evidently.report import Report
+from evidently.metric_preset import ClassificationPreset, RegressionPreset
+from evidently.metrics import *
+```
 
+**Sample datasets**
 
+```python
+#Dataset for binary probabilistic classifcation
+bcancer_data = datasets.load_breast_cancer(as_frame=True) #auto
+bcancer = bcancer_data.frame
 
+bcancer_ref = bcancer.sample(n=300, replace=False)
+bcancer_cur = bcancer.sample(n=200, replace=False)
 
+model = ensemble.RandomForestClassifier(random_state=1, n_estimators=10)
+model.fit(bcancer_ref[bcancer_data.feature_names.tolist()], bcancer_ref.target)
 
+bcancer_ref['prediction'] = model.predict_proba(bcancer_ref[bcancer_data.feature_names.tolist()])[:, 1]
+bcancer_cur['prediction'] = model.predict_proba(bcancer_cur[bcancer_data.feature_names.tolist()])[:, 1]
 
+#Dataset for regression
+housing_data = datasets.fetch_california_housing(as_frame=True) #auto 
+housing = housing_data.frame
 
+housing.rename(columns={'MedHouseVal': 'target'}, inplace=True)
+housing['prediction'] = housing_data['target'].values + np.random.normal(0, 3, housing.shape[0])
 
+housing_ref = housing.sample(n=5000, replace=False)
+housing_cur = housing.sample(n=5000, replace=False)
+```
+
+**Classification report**
+
+```python
+classification_performance_report = Report(metrics=[
+    ClassificationPreset(probas_threshold=0.7),
+])
+
+classification_performance_report.run(reference_data=bcancer_ref, current_data=bcancer_cur)
+
+classification_performance_report.show(mode='inline')
+```
+
+<img width="1386" alt="image" src="https://github.com/user-attachments/assets/5b10fb4d-c9e0-43e1-a99e-72ca62582127">
+
+<img width="1385" alt="image" src="https://github.com/user-attachments/assets/04446745-829a-4572-be03-7ed44732b359">
+
+<img width="1362" alt="image" src="https://github.com/user-attachments/assets/99f935cf-c94d-4cfd-9250-730aabaf4de9">
+
+<img width="1350" alt="image" src="https://github.com/user-attachments/assets/1289b3f0-c3fd-40ff-9625-e9392980be78">
+
+We can get the data from the automatically created report as html, dict, or json.
+
+**Regression report**
+
+<img width="1376" alt="image" src="https://github.com/user-attachments/assets/19912456-dd96-44a8-bdfa-8e28776024fc">
+
+<img width="1368" alt="image" src="https://github.com/user-attachments/assets/7bf963e5-c368-498f-8bba-3fe67800e495">
+
+<img width="1336" alt="image" src="https://github.com/user-attachments/assets/92226303-bfbc-463b-b40a-78613fed9d5e">
+
+<img width="1350" alt="image" src="https://github.com/user-attachments/assets/cd60c56c-6cd5-45b7-aa0f-8e838bb05645">
+
+<img width="1388" alt="image" src="https://github.com/user-attachments/assets/5ece3bf2-b621-4d61-871a-f93558d125ac">
+
+#### Data quality in machine learning
+
+What can go wrong with the input data?
+
+Some common data processing issues are:
+
+* Wrong source. E.g., a pipeline points to an older version of the table.
+
+* Lost access. E.g., permissions are not updated.
+
+* Bad SQL. Or not SQL. E.g., a query breaks when a user comes from a different time zone and makes an action “tomorrow."
+
+* Infrastructure update. E.g., change in computation based on a dependent library.
+
+* Broken feature code. E.g., feature computation breaks at a corner case like a 100% discount.
+
+![image](https://github.com/user-attachments/assets/c6b187e0-93ba-4600-a2a7-38e3efe6d6b0)
+
+Data profiling is a good starting point for monitoring data quality metrics. Based on the data type, you can come up with basic descriptive statistics for your dataset. For example, for numerical features, you can calculate:
+
+* Min and Max values
+
+* Quantiles
+
+* Unique values
+
+* Most common values
+
+* Share of missing values, etc.
+
+Then, you can visualize and compare statistics and data distributions of the current data batch and reference data to ensure data stability.
+
+![image](https://github.com/user-attachments/assets/6ede8749-ee98-4f45-afea-9c686cc2345e)
+
+**Manual Thresholds for Data Quality**
+
+If reference data is unavailable, we can establish thresholds manually using domain knowledge. Consider the following characteristics for "general ML data quality":
+
+- No/low share of missing values
+- No duplicate columns/rows
+- No constant (or almost constant) features
+- No highly correlated features
+- No target leaks (i.e., high correlation between feature and target)
+- No range violations (e.g., negative values in age or sales)
+
+Manually setting up these conditions can be tedious, so having reference data is often beneficial.
+
+**Data Quality with Reference Data**
+
+If we have reference data, you can compare it with the current data to autogenerate test conditions. Monitor for the following based on the training data or previous batches:
+
+- Expected data schema and column types
+- Expected data completeness (e.g., 90% non-empty)
+- Expected batch size (e.g., number of rows)
+- Expected patterns for specific columns:
+  - Non-unique (features) or unique (IDs)
+  - Specific data distribution types (e.g., normality)
+  - Expected ranges based on observed values
+- Descriptive statistics: averages, median, quantiles, min-max (using point estimation or statistical tests with confidence intervals)
+
+**Sample data test report**
+
+<img width="1339" alt="image" src="https://github.com/user-attachments/assets/b19447cd-3873-4908-9930-1efa499f7ce3">
+
+We can also set tests only on specific columns, to test for specific statistics
+
+```python
+#column-level tests
+data_quality_column_tests = TestSuite(tests=[
+    TestColumnValueMean(column_name='education-num'),
+    TestColumnValueMedian(column_name='education-num'),
+    TestNumberOfUniqueValues(column_name='education'),
+    TestMostCommonValueShare(column_name='education'),
+    TestValueRange(column_name='education-num'),
+    TestShareOfOutRangeValues(column_name='education-num'),
+    TestNumberOfOutListValues(column_name='education'),
+    TestColumnQuantile(column_name='education-num', quantile=0.25),
+    TestColumnShareOfMissingValues(column_name='education'),
+    TestColumnRegExp(column_name='education',reg_exp='^[0..9]'),
+    TestCategoryShare(column_name='education', category='Some-college', lt=0.5),
+])
+
+data_quality_column_tests.run(reference_data=adult_ref, current_data=adult_cur)
+data_quality_column_tests.show(mode='inline')
+```
+
+<img width="1389" alt="image" src="https://github.com/user-attachments/assets/47f16260-7c4a-4c04-97d8-caa4aff5a0dd">
+
+#### Data and prediction drift in ML
+
+**What is data drift, and why evaluate it?**
+When ground truth is unavailable or delayed, we cannot calculate ML model quality metrics directly. Instead, we can use proxy metrics like feature and prediction drift.
+
+**Prediction drift** shows changes in the distribution of model outputs over time. Without target values, this is the best proxy of the model behavior. Detected changes in the model outputs may be an early signal of changes in the model environment, data quality bugs, pipeline errors, etc.
+
+![image](https://github.com/user-attachments/assets/4e7de754-12bc-49a7-b679-a0a4fc0e1c90)
+
+**Feature drift** demonstrates changes in the distribution of input features over time. When we train the model, we assume that if the input data remains reasonably similar, we can expect similar model quality. Thus, data distribution drift can be an early warning about model quality decay, important changes in the model environment or user behavior, unannounced changes to the modeled process, etc.
+
+![image](https://github.com/user-attachments/assets/eb60baa8-7c8b-48d7-8f1a-4441a404b661)
+
+Some key considerations about data drift to keep in mind:
+
+* Prediction drift is usually more important than feature drift. If you monitor one thing, look at the outputs.
+* Data drift in ML is a heuristic. There is no “objective” drift; it varies based on the specific use case and data.
+* Not all distribution drift leads to model performance decay. Consider the use case, the meaning of specific features, their importance, etc.
+* You don’t always need to monitor data drift. It is useful for business-critical models with delayed feedback. But often you can wait.
+* Data drift helps with debugging. Even if you do not alert on feature drift, it might help troubleshoot the decay.
+* Drift detection might be valuable even if you have the labels. Feature drift might appear before you observe the model quality drop.
+
+**How to detect data drift?**
+
+To detect distribution drift, you need to pick:
+
+* Drift detection method: statistical tests, distance metrics, rules, etc.
+
+* Drift detection threshold: e.g., confidence levels for statistical tests or numeric threshold for distance metrics.
+
+* Reference dataset: what an exemplary distribution is.
+
+* Alert conditions: e.g., based on feature importance and the share of the drifting features.
+
+**Data drift detection methods**
+
+![image](https://github.com/user-attachments/assets/e177b96c-8e38-46c6-9633-b9eff4b13854)
+
+![image](https://github.com/user-attachments/assets/6ccdaae8-c491-489d-8dbe-f88e409b2025)
+
+* And rule-based checks
+
+**Univariate vs. multivariate drift**
+
+The univariate drift detection approach looks at drift in each feature individually. It returns drift/no drift for each feature and can be easily interpretable.
+
+The multivariate drift detection approach looks at the complete dataset (e.g., using PCA and certain methods like domain classifier). It returns drift/no drift for the dataset and may be useful for systems with many features.
+
+You can still use the univariate approach to detect drift in a dataset by:
+
+* Tracking the share (%) of drifting features to get a dataset drift decision.
+
+* Tracking distribution drift only in the top model features.
+
+* Combining both solutions.
+
+**Tips for calculating drift**
+
+Here are some tips to keep in mind when calculating data drift:
+
+* Data quality is a must. Calculate data quality metrics first and then monitor for drift. Otherwise, you might detect “data drift” that is caused by data quality issues.
+
+* Mind the feature set. The approach to drift analysis varies based on the type and importance of features.
+
+* Mind the segments. Consider segment-based drift monitoring when you have clearly defined segments in your data. For example, in manufacturing, you might have different suppliers of raw materials and need to monitor distribution drift separately for each of them.
+
+---
 
+So far the evidently course is great, but also the material in the first two modules is what I learned in the MLOps zoomcamp. Seems like the next modules will be something new, but that will be left for tomorrow.
 
+That is all for today!
 
+See you tomorrow :)

From b876a2e402d38f7a27c16774f57de60db117c77c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:01:37 +0900
Subject: [PATCH 408/638] Rename 2024-09-01-day-245.md to 2024-09-02-day-245.md

---
 _posts/{2024-09-01-day-245.md => 2024-09-02-day-245.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2024-09-01-day-245.md => 2024-09-02-day-245.md} (100%)

diff --git a/_posts/2024-09-01-day-245.md b/_posts/2024-09-02-day-245.md
similarity index 100%
rename from _posts/2024-09-01-day-245.md
rename to _posts/2024-09-02-day-245.md

From 0ff2366c2dab203ace23eba7b7d1b216c8884827 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 3 Sep 2024 22:51:33 +0900
Subject: [PATCH 409/638] Create 2024-08-03-day-246.md

---
 _posts/2024-08-03-day-246.md | 478 +++++++++++++++++++++++++++++++++++
 1 file changed, 478 insertions(+)
 create mode 100644 _posts/2024-08-03-day-246.md

diff --git a/_posts/2024-08-03-day-246.md b/_posts/2024-08-03-day-246.md
new file mode 100644
index 000000000..334be02af
--- /dev/null
+++ b/_posts/2024-08-03-day-246.md
@@ -0,0 +1,478 @@
+---
+layout: post
+title: (Day 246) First deployment on Kubernetes
+categories: [mlops,data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 246!
+A quick summary of today:
+* deploying a simple model on K8s
+* finishing the Streaming databases book
+
+## [Deploying a simple model on Kubernetes (K8s)](https://github.com/divakaivan/k8s_deploy)
+
+Who else to explain K8s in an easy and intuitive way better than DataTalksClub... I am not surprised haha. In their ML zoomcamp, [module 10](https://github.com/DataTalksClub/machine-learning-zoomcamp/tree/master/10-kubernetes) is about kubernetes, and I watched the videos where Alexey (the instructor) explains k8s and goes into a practical example. Below is my understanding of how k8s works and a deployed example model. Everything is on my repo.
+
+### Kubernetes intro
+
+* Node ~ server/computer (i.e. EC2 instance)
+* Pod ~ docker container, runs on a node
+* Deployment ~ group of pods with the image & config
+* Service ~ the entry point of an application, routes requests to pods
+    - external = LoadBalancer
+	- internal = ClusterIP
+* Ingress ~ the entrypoint to the cluster
+* HPA ~ Horizontal pod autoscaler - ensures the app scales up and down to match the required demand
+
+![image](https://github.com/user-attachments/assets/8ea73ed6-7f5d-452d-8b38-105b262166b5)
+
+* Ingress routes traffic to services within the cluster
+* a sample request comes in from a user and is first sent to the LoadBalancer (the external-facing service), which balances the load of incoming requests across multiple Pods within the Gateway Deployment. This means the LoadBalancer routes the request to a Pod in the Gateway Deployment on any Node in the cluster (e.g., Node 1 or Node 2)
+* from there, the Gateway Pod processes the request (e.g., handling authentication, routing, etc.) and forwards it to the internal Model Service
+* the Model Service then assigns the request to one of the Pods in the Model Deployment, which may be running on a different Node within the cluster (e.g., Node 1 or Node 2)
+* finally, once the Model Pod processes the request and generates a response, the response is sent back through the Gateway Pod and ultimately back to the user
+
+The example in this repo and below only has 1 service and deployment. The above is used as a more complicated example to show routing
+
+### Create k8s cluster
+
+```
+kind create cluster
+```
+
+- next, build and run our docker app
+
+### Create deployment
+
+```yaml
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  name: app-deployment  # name of the deployment, used to identify the deployment object
+spec:
+  replicas: 1  # number of pod replicas that should be running at any time
+  selector:
+    matchLabels:
+      app: app  # identifies the pods managed by this deployment using the "app: app" label
+  template:
+    metadata:
+      labels:
+        app: app  # the label applied to the pods, used by the deployment to manage the pods
+    spec:
+      containers:
+      - name: app-pod  # name of the container inside the pod.
+        image: test-model:v1  # container image to use
+        resources:
+          limits:
+            memory: "128Mi"
+            cpu: "500m"
+        ports:
+        - containerPort: 9696  # exposed port of the container
+```
+
+- create deployment.yaml, and run
+
+```
+kubectl apply -f deployment.yaml
+```
+
+![image](https://github.com/user-attachments/assets/ef6b1659-e3fb-4b86-96af-a11eb86ed662)
+
+k8s does not know about the test-model:v1 image, so we should register it first. Load it with
+
+```
+kind load docker-image test-model:v1
+``` 
+
+The deployment keeps retrying to load the image, and once the image is loaded, we get
+
+`app-deployment-7bfd99c878-fnbxl   1/1     Running   0          4m32s`
+
+when I run `kubectl get all`
+
+![image](https://github.com/user-attachments/assets/319b6240-3bc7-4c04-8c04-393d6fd15032)
+
+#### Test the deployment
+
+![image](https://github.com/user-attachments/assets/520273bd-6899-4f5d-823a-5c3b5e956081)
+
+We can use port-forwarding:
+
+We specify, the name of the pod, and then the port
+
+```
+kubectl port-forward app-deployment-58c8855fbd-xknbf 9696:9696
+```
+
+And I can use curl to test this:
+
+```
+curl -X POST http://localhost:9696/predict -H "Content-Type: application/json" -d '{"features": [5.1, 3.5, 1.4, 0.2]}'
+```
+
+And I get: `{"prediction":0}`  
+
+### Create service.yaml
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: app # name of the service
+spec:
+  type: LoadBalancer # LoadBalancer means expose the service outside the cluster
+  selector:
+    app: app # selects the pods with the label "app: app"
+  ports:
+  - port: 80 # port in the service
+    targetPort: 9696 # port on the pod
+```
+
+The service will forward requests to the pods. We as users will communicate with the service. 
+
+Apply service:
+
+```
+kubectl apply -f service.yaml 
+```
+
+We can see it with `kubectl get svc`
+
+```
+NAME         TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
+kubernetes   ClusterIP      10.96.0.1      <none>        443/TCP        29m
+app         LoadBalancer   10.96.94.146   <pending>     80:30966/TCP   2m15s
+```
+
+The EXTERNAL-IP is always pending unless we use some service like GKE and EKS
+
+
+#### Test the k8s service
+
+```
+kubectl port-forward service/app 8080:80 
+```
+
+Terminal:
+
+```
+Forwarding from 127.0.0.1:8080 -> 9696
+Forwarding from [::1]:8080 -> 9696
+```
+
+And if I do (also change the port to 8080 now)
+
+```
+curl -X POST http://localhost:8080/predict -H "Content-Type: application/json" -d '{"features": [5.1, 3.5, 1.4, 0.2]}'
+```
+
+I get back: `{"prediction":0}` 
+
+### Overall
+
+Definitely feel more confident about my k8s knowledge ^^
+
+## [MLOps engineer for 4 years shares their experience](https://mburaksayici.com/blog/2024/08/29/what-ive-learned-building-mlops-systems-for-four-years.html)
+
+The author reflects on their journey in machine learning and MLOps, beginning with their first major project in 2021: day-ahead electricity consumption forecasting for multiple cities. Initially, they focused on predicting daily electricity use 24 hours in advance using tree-based models like LightGBM and XGBoost. The challenges of model and data drift, as well as external factors like weather and holidays, complicated predictions. To address these, they built automated systems to generate features, select models, and make daily predictions, though manual oversight was still necessary.
+
+Their experience led to a realization of the importance of MLOps in maintaining models and adapting to changes. After this project, they transitioned to working on an MLOps platform in healthcare where they faced new challenges in building production-level software. They had to learn traditional software engineering skills and navigate the complexities of integrating client-specific needs into a scalable platform.
+
+Throughout their career they grappled with the evolving roles in ML and MLOps, questioning their identity as an engineer (MLOps vs MLE vs SE vs DS). They juggled various responsibilities, from research and model development to platform design and deployment. Despite these struggles, having a wide range of skills is of course a benefit, but it is important to balance wide knowledge and expertise in 1 field. 
+
+## Streaming databases: Chapter 7. Emergence of Other Hybrid Data Systems
+
+### Data Planes
+
+Emerging real-time systems can be better understood through a Venn diagram that illustrates where these systems fit within different data planes. This approach helps clarify the use cases and deployment models in real-time analytical scenarios.
+
+The diagram considers both streaming and database perspectives. From a streaming database perspective, it involves systems that can consume and emit streams, while executing materialized views asynchronously. This view hints at the potential characteristics of next-generation databases.
+
+#### The Streaming Plane
+
+The Venn diagram outlines three data planes:
+
+1. **Operational Plane**: Houses OLTP databases, which are consistent and use row-based storage. It also includes the applications that utilize these databases.
+
+2. **Analytical Plane**: Contains OLAP databases, optimized for analytical queries, featuring columnar storage and eventual consistency.
+
+3. **Streaming Plane**: Acknowledged as the only data plane where data is mostly in motion. It bridges the operational and analytical planes, enabling real-time data flow into the analytical plane. This integration allows for quicker, data-driven decisions by combining real-time and historical data.
+
+   - The streaming plane includes source connectors that bring static data into motion and sink connectors that write streaming data into RTOLAP databases for low-latency access.
+   - Platforms like Kafka and Kafka Connect, along with stateful stream processors and streaming databases, reside in this plane.
+
+#### The Overlaps
+
+- The overlap between the operational and streaming planes, and between the streaming and analytical planes, represents areas where both data at rest and data in motion coexist. This is where streaming databases operate.
+- The overlap between the operational and analytical planes will be further explored later in the chapter.
+
+#### Consistency Spectrum
+
+![image](https://github.com/user-attachments/assets/4f794673-c2cf-4aeb-a2fd-baabb32be602)
+
+The above image illustrates the consistency spectrum of stream processors, showing a division between strictly consistent and eventually consistent processors, as well as various storage types. Streaming data flows from the operational plane to the analytical plane, leveraging connectors and streaming platforms like Kafka along the way.
+
+The overlapping areas in these diagrams highlight the crucial interactions between different data planes, essential for understanding the role of streaming systems in the broader analytical ecosystem.
+
+### Hybrid Transactional/Analytical Database (HTAP)
+
+- Streaming OLTP Databases: These databases (e.g., RisingWave, Materialize) combine stream processing with OLTP features using row-based storage.
+
+- Streaming OLAP Databases: These databases (e.g., Proton) merge OLAP features with stream processing using column-based storage, optimized for complex queries with eventual consistency.
+
+- HTAP Databases: Positioned between OLTP and OLAP without streaming, HTAP databases can handle both workloads, enabling real-time decision-making. They leverage in-memory and persistent storage, allowing analytical queries on in-flight transactions, though they lack stream processing features.
+
+- Market Examples:
+  - **Unistore (Snowflake)**: Hybrid storage with both row and columnar data.
+  - **SingleStoreDB (SingleStore)**: Supports in-memory row-based and on-disk column-based tables.
+  - **TiDB (PingCAP)**: Combines transactional key-value store (TiKV) with columnar analytics (TiFlash).
+  - **HydraDB**: Offers transactional row-based and column-based storage.
+
+- Limitations: HTAP databases are monolithic and not suited for large-scale historical data, thus requiring separate OLAP systems for deep history. They excel in operational plane analytics but may still need data aggregation from OLAP systems.
+
+- Hybrid Databases: The evolution of HTAP and hybrid databases addresses real-time analytical challenges but often necessitates additional infrastructure for scalability and optimization.
+
+### Motivations for Hybrid Systems
+
+Stream processing systems aim to reduce the complexity often associated with real-time data processing by providing a more user-friendly experience similar to traditional databases. Vendors are working to make stream processing more accessible by simplifying APIs, interfaces, and tools, thereby lowering the barrier to entry for businesses.
+
+A significant challenge in stream processing adoption is the scarcity of skilled data engineers. By integrating a database-like environment, organizations can leverage real-time analytics without needing specialized expertise.
+
+On the other hand, OLTP databases are adopting OLAP and streaming features to better handle analytical queries and meet the demand for real-time data analytics. This helps avoid complex infrastructure typically needed for separate analytical systems. Streaming features also simplify data replication and synchronization in distributed environments by ensuring real-time data consistency.
+
+Both stream processing and OLTP databases are converging toward the goal of providing real-time analytics with less infrastructure and greater accessibility. Hybrid systems often use OLTP databases due to their proximity to user-facing applications, emphasizing the need for data analytics closer to the operational plane. Streaming systems must recognize these needs to improve their reputation and ease of implementation.
+
+## Streaming dbs: Chapter 8. Zero-ETL or Near-Zero-ETL
+
+### ETL Model
+
+![image](https://github.com/user-attachments/assets/bfa8409a-dc74-4ff0-a0d2-943cb0a73f9b)
+
+The above shows existing ETL solutions from no ETL in HTAP databases at the top to the turn-the-database-inside-out distributed solution at the bottom. The lower the solution is on the triangle, the more distributed and complex it becomes. Likewise, at the top, solutions are more centralized and monolithic, and they become more decentralized and modular as you move to the bottom. On the left side of the triangle are transactional databases, while on the right are columnar databases. Midway down the triangle is zero-ETL.
+
+### Zero-ETL
+
+Zero-ETL is a pattern first defined by Amazon Web Services (AWS) to simplify data integration from an OLTP database to an OLAP database. In its proposal, zero-ETL is defined as follows:
+
+_A set of integrations that eliminates or minimizes the need to build ETL data pipelines. Extract, transform, and load (ETL) is the process of combining, cleaning, and normalizing data from different sources to get it ready for analytics, artificial intelligence (AI) and machine learning (ML) workloads.
+_ - AWS
+
+Zero-ETL refers to an approach or concept in data integration and analytics that aims to minimize or even eliminate the need for traditional ETL processes. The traditional ETL process involves extracting data from source systems, transforming it to meet the requirements of the target system, and then loading it into the destination.
+
+![image](https://github.com/user-attachments/assets/6a1fc402-adaf-475f-992e-6472d2df933e)
+
+Zero-ETL challenges the traditional integration approach and seeks to reduce the complexity, latency, and resource requirements associated with ETL at a cost. Key aspects of zero-ETL:
+
+| Key aspect               | Description                                                                                                                                               |
+|---------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Real-time data integration** | Minimizing or eliminating batch processing to enable real-time or near-real-time data integration. This is particularly relevant for scenarios where timely insights are crucial. |
+| **Schema-on-read**             | Adopting a schema-on-read approach, where the data is not transformed into a predefined schema during the ETL process but is instead interpreted at the time of analysis. This allows for more flexibility in handling diverse and changing data. |
+| **Data virtualization**         | Leveraging data virtualization technologies that provide a unified and virtual view of data across multiple sources without physically moving or transforming the data. This can reduce the need for creating and maintaining a separate data warehouse. |
+| **In-database processing**      | Performing transformations and analytics directly within the database systems, where the data resides, avoiding the need to extract and move large datasets for processing. |
+| **Event-driven architecture**   | Adopting event-driven architectures, where data changes trigger immediate updates, reducing the reliance on periodic batch processes. |
+| **Modern data architectures**   | Embracing modern data architectures, such as data lakes and cloud-based solutions, that provide scalable and cost-effective options for managing and analyzing data without the traditional ETL bottlenecks. |
+
+### Near-Zero-ETL
+
+Near-zero-ETL still tries to limit infrastructure for ETL components without losing the flexibility needed to support complex data integration requirements. This involves using data systems that adopt hybrid approaches.
+
+One solution is to leverage an OLTP database that has embedded features to send data to other systems, without the need for self-managing connectors running on a separate infrastructure.
+
+#### Embedded OLAP
+
+There is a trend to bring smaller analytical workloads closer to the operational plane. HTAP databases like Hydra and SingleStore provide columnar databases for analytical workloads, for example. However, due to their limited capacity, these databases cannot hold the amount of data analytical systems like Snowflake, Databricks, ClickHouse, and Pinot can.
+
+Conversely, bringing bulky analytical systems to the operational plane for faster serving of real-time analytics makes it harder for analytical systems to source historical data. These are the limitations that created the data divide between operational and analytical data planes.
+
+Alternatively, reducing the analytical data to a size fitting to the scope of the business domain and the capacity in the operational plane could provide a better solution.
+
+### Lambda Architecture
+
+The lambda architecture is a data processing model introduced by Nathan Marz in his 2011 book *Big Data: Principles and Best Practices of Scalable Realtime Data Systems*. It addresses the need for robust and scalable data processing in big data applications by combining both batch and real-time/streaming data processing.
+
+![image](https://github.com/user-attachments/assets/33490236-4dd3-4cc8-adb7-4c3cf244d1c9)
+
+#### Key Components of Lambda Architecture:
+
+- Batch layer: Processes large datasets in batch mode for complex analytics and historical queries. Technologies like Apache Hadoop are commonly used.
+- Speed layer: Handles real-time data with low-latency processing, using technologies like Apache Storm or Apache Flink. It processes recent data that hasn't been handled by the batch layer.
+- Serving layer: Combines results from both the batch and speed layers to provide a unified data view, typically using scalable NoSQL databases like Apache HBase or Cassandra.
+
+#### Strengths and Challenges:
+The architecture's strength lies in its ability to handle both batch and real-time processing. However, managing two separate processing paths introduces complexities, particularly in maintaining consistency between batch and real-time data views.
+
+#### Alternatives:
+- Kappa architecture: Proposes a unified approach to stream and batch processing to simplify system design.
+- Stream processor and OLAP database: Using tools like Apache Pinot for serving historical data combined with streaming data processed by technologies like Flink or Pathway (for Python users).
+
+## Streaming dbs: Chapter 10. Deployment Models
+
+### Consistent Streaming Database
+
+A consistent streaming database allows complex asynchronous/stream processing that integrates with application logic. It simplifies infrastructure and allows querying of asynchronous process outputs directly from the database, without requiring extensive streaming knowledge. Examples of such databases include **RisingWave** and **Materialize**.
+
+![image](https://github.com/user-attachments/assets/653635b2-1a80-4b87-8075-eb7f832d40af)
+
+Above, a consistent streaming database is shown to bridge the gap between streaming and operational planes. It consumes data from an OLTP database, performs stream processing for analytical transformations, and stores results in row-based storage. Applications can write to the OLTP database and read from the streaming database, separating read and write resources.
+
+The streaming database can also aggregate data from multiple OLTP databases and send it to a data warehouse or lakehouse for internal business analytics. However, limitations arise when user-facing applications require historical data from the warehouse, or when complex analytical queries demand columnar storage.
+
+#### Pros:
+- **Low Latency**: Provides data freshness within milliseconds.
+- **Scalability**: Separates and scales read/write resources independently.
+- **Multi-Source Aggregation**: Combines inputs from various OLTP databases.
+- **Incremental Transformations**: Processes data before sending it to the analytical plane.
+- **Versatile Querying**: Supports both push and pull queries with a single interface.
+
+#### Cons:
+- **Lacks Columnar Storage**: Less suited for complex analytical queries.
+- **Historical Data Limitations**: Challenges in sourcing historical data from the analytical plane.
+- **Scalability Issues**: Struggles with very large datasets.
+
+### Consistent Streaming Processor and RTOLAP
+
+A consistent stream processor, like **Pathway**, can output streams to a columnar database. These processors execute push queries near the operational plane and integrate with application logic. The output is sent to a streaming platform, such as **Kafka**, which a **RTOLAP** database (e.g., **Pinot**) can consume.
+
+![image](https://github.com/user-attachments/assets/30339ea8-99e2-425b-a3ab-ca9d80969ba5)
+
+
+The above illustrates the transformation of data within the stream processor, writing results to both a data warehouse/lakehouse and a RTOLAP database. The RTOLAP database can then combine real-time data with historical data from the warehouse/lakehouse, allowing low-latency pull queries at the operational plane.
+
+#### Pros:
+- **Low Latency**: Provides data freshness in milliseconds to seconds.
+- **Comprehensive Analytics**: Includes all historical context in user-facing analytics without burdening operational infrastructure.
+- **Fast Queries**: Columnar storage in the RTOLAP database optimizes analytical workloads.
+- **Topic Reuse**: The RTOLAP and stream processor can efficiently reuse topics on the streaming platform.
+- **Dual Functionality**: The stream processor can handle both real-time analytics and application logic.
+
+#### Cons:
+- **Query Separation**: Push and pull queries are handled separately by the stream processor and OLAP database, requiring coordination between engineers, which can be challenging in practice.
+
+### Eventually Consistent OLAP Streaming Database
+
+For simplified infrastructure, a **streaming OLAP database** like **Proton** can consolidate analytical workloads into one solution. However, due to its **eventual consistency**, it should not participate in application logic. The below shows that data flows one way from the OLTP to the streaming database, leveraging the OLAP's columnar storage for low-latency queries.
+
+![image](https://github.com/user-attachments/assets/d54c2268-d8a5-42cc-8865-49b792ab7e98)
+
+
+Proton, being a streaming database, can output its stream processing results to **Kafka**, allowing other databases to consume the analytical stream and build replicas. Since Proton integrates **ClickHouse** (an RTOLAP), it has built-in columnar storage for low-latency analytical queries. Kafka can also distribute analytics globally in real-time.
+
+#### Pros:
+- **Low Latency**: Provides data freshness in milliseconds to seconds.
+- **Comprehensive Historical Data**: Offers more context by combining real-time and historical data.
+- **Replication**: Can emit analytical changes for building replicas.
+- **Simplified Infrastructure**: Merges stream processing and OLAP technologies.
+- **Unified Query Engine**: Balances push and pull queries within a single SQL engine.
+- **Efficiency**: Less bulky solution.
+
+#### Cons:
+- **Eventual Consistency**: Not suitable for application logic.
+
+This solution reduces infrastructure complexity and provides more historical data access for user-facing analytics, simplifying the engineering effort.
+
+### Eventually Consistent Stream Processor and RTOLAP
+
+![image](https://github.com/user-attachments/assets/af69b077-44bf-4400-a08f-eed2303735d6)
+
+This solution, commonly used for real-time analytics, typically involves **Flink** as the stream processor and an **RTOLAP** database like **Pinot** (see **Figure 10-4**). It is widely adopted in high-scale, real-time applications.
+
+#### Pros:
+- **Low Latency**: Provides data freshness in milliseconds to seconds.
+- **Comprehensive Analytics**: Combines historical and real-time data for a complete view.
+
+#### Cons:
+- **Complexity**: The solution is complex and bulky, leading to higher costs.
+- **Limited Application Logic**: Since Flink is eventually consistent, it should not be used in application logic.
+- **Separated Query Engines**: Lacks a unified SQL engine for push and pull queries, increasing engineering and organizational complexity.
+
+This solution is ideal when your application requires more historical data for user-facing analytics, and consistency isn't a critical concern when enriching fact streams with dimensional data.
+
+### Eventually Consistent Stream Processor and HTAP
+
+This solution leverages an **HTAP database** alongside an **OLTP database** to keep analytical workloads close to the operational plane. An **eventually consistent stream processor** captures and sends historical data to a data warehouse or lakehouse.
+
+![image](https://github.com/user-attachments/assets/8dd4212d-28ce-4ea6-a0ce-81fb2644fe4a)
+
+
+The stream processor sources limited historical data for the HTAP database, which, due to its columnar format, supports low-latency analytical queries. However, the HTAP database typically holds limited historical data, often with retention constraints.
+
+#### Pros:
+- **Low Latency**: Provides data freshness in milliseconds.
+- **Fast Queries**: HTAP databases use columnar storage for quick analytical queries.
+- **Simplified Infrastructure**: Low infrastructural complexity.
+
+#### Cons:
+- **Limited Historical Data**: Holds only a small subset of historical data.
+- **Retention Complexity**: Managing retention for historical data in the HTAP database increases complexity.
+- **Application Logic Limitations**: The stream processor cannot participate in application logic.
+
+This solution is ideal for use cases that require millisecond-level freshness and a small amount of historical data for user-facing analytics.
+
+### ksqlDB
+
+**ksqlDB** offers **continuous refinement** (similar to eventual consistency) and is built on the **Kafka Streams** library, designed for deployment in microservices within the operational plane of an application backend.
+
+**ksqlDB** is best suited for **simpler stream processing operations** on Kafka, as it only supports Kafka as its data source and sink. Complex operations, especially involving JOINs, can be difficult to implement correctly due to limited SQL syntax and semantics (e.g., no self-JOINs, no nested JOINs). A high level of stream processing expertise is required to avoid inconsistent logic.
+
+**Use Cases**: Ideal for preparing data for analytical destinations (e.g., data warehouses or lakes) and enabling point queries through materialized views (TABLEs) for batch-only systems.
+
+#### Pros:
+- Provides data freshness.
+- Supports stream processing capabilities.
+- Enables point queries using materialized views.
+
+#### Cons:
+- Limited to Kafka as the source and sink.
+- Requires significant stream processing expertise.
+- Complex stream processing operations are challenging.
+- Limited SQL syntax and semantics support.
+
+### Incremental View Maintenance (IVM)
+IVM solutions like **Feldera, PeerDB,** and **Epsio** support **batch-based point queries** on preprocessed, fresh data, offering **full SQL syntax and semantics**. Unlike ksqlDB, they integrate more closely with operational databases like PostgreSQL and do not rely on Kafka. However, these solutions are more **inflexible** and limited to the data sources and sinks supported by the respective vendor.
+
+**Pros:**
+- Data freshness
+- Full SQL syntax and semantics
+- Consistency
+- Enables point queries
+- Introduces streaming aspects into the database world
+
+**Cons:**
+- Restricted to vendor-supported sources/sinks
+
+### Postgres Multicorn Foreign Data Wrapper (FDW)
+**Multicorn** is a PostgreSQL extension that facilitates foreign data wrapper (FDW) development using Python. It enables access to non-Postgres databases and integrates operational and analytical planes without relying on Kafka. However, it’s **impractical for large organizations** that use a variety of databases, as it requires centralizing around PostgreSQL.
+
+**Pros:**
+- Full SQL syntax and semantics
+- Consistency
+- Access to both OLTP and OLAP databases within Postgres
+
+**Cons:**
+- Requires Postgres as the central database technology, which is impractical at large scale
+
+### Code-Based Stream Processors
+Traditional stream processors like **Kafka Streams** and **Flink** are still valuable, especially for complex streaming use cases such as fraud detection in microservices. Newer technologies like **Quix Streams, Bytewax,** and **Pathway** (Python-based) provide alternatives, with frameworks like **Deephaven** and **GlassFlow** also emerging.
+
+### Lakehouse/Streamhouse Technologies
+**Lakehouse** technologies (e.g., **Databricks' Delta Tables, Apache Iceberg,** and **Apache Hudi**) increasingly incorporate streaming features. The **Streamhouse** architecture (e.g., **Ververica's Apache Paimon**) integrates stream and batch processing more closely, with **Confluent’s Tableflow** enabling similar functionality by exposing Kafka data as Iceberg tables. **Streambased.io** offers optimized SQL querying on Kafka topics using Bloom filters.
+
+These advancements blur the lines between **operational, analytical,** and **streaming planes**, moving more processing closer to data in motion and reducing reliance on data warehouses/lakes.
+
+### Caching Technologies
+For **extremely low latency**, tools like **Redis** or **Hazelcast** provide caching solutions, which can be integrated with streaming platforms.
+
+### General Processing and Querying Considerations
+When determining where to process and query data, consider the **location of your use case, data origin, processing location,** and **querying location**. While processing and querying on the **analytical plane** is common, it can lead to challenges like **data freshness, sharing,** and **incremental processing**.
+
+**Streaming databases** simplify stream processing and querying, reducing the need for specialized expertise and making streaming more accessible. **Materialized views** in streaming databases allow for fresher data and cost-effective incremental processing.
+
+As **streaming technologies** continue to evolve, the **data gravity of the streaming plane** increases, blurring the boundaries between **operational, analytical,** and **streaming planes**.
+
+
+---
+
+
+That is all for today!
+
+See you tomorrow :)

From c2e3fddd3b56c7c96ff7734557a3b82403f681b9 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 3 Sep 2024 22:52:22 +0900
Subject: [PATCH 410/638] Rename 2024-08-03-day-246.md to 2024-09-03-day-246.md

---
 _posts/{2024-08-03-day-246.md => 2024-09-03-day-246.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2024-08-03-day-246.md => 2024-09-03-day-246.md} (100%)

diff --git a/_posts/2024-08-03-day-246.md b/_posts/2024-09-03-day-246.md
similarity index 100%
rename from _posts/2024-08-03-day-246.md
rename to _posts/2024-09-03-day-246.md

From 4d63bd83c522c71ac1af5064b72234f1000dda5a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 4 Sep 2024 23:22:38 +0900
Subject: [PATCH 411/638] Create 2024-09-04-day-247.md

---
 _posts/2024-09-04-day-247.md | 289 +++++++++++++++++++++++++++++++++++
 1 file changed, 289 insertions(+)
 create mode 100644 _posts/2024-09-04-day-247.md

diff --git a/_posts/2024-09-04-day-247.md b/_posts/2024-09-04-day-247.md
new file mode 100644
index 000000000..05e5ebf34
--- /dev/null
+++ b/_posts/2024-09-04-day-247.md
@@ -0,0 +1,289 @@
+---
+layout: post
+title: (Day 247) Continuing with AI monitoring using EvidentlyAI
+categories: [mlops,theory]
+---
+
+# Hello :) Today is Day 247!
+A quick summary of today:
+* monitoring unstructured data (text) using evidently
+
+Today I continued with EvidentlyAI's course on monitoring.
+
+## [Evidently AI: Module 3: ML monitoring for unstructured data](https://learn.evidentlyai.com/ml-observability-course/module-3-ml-monitoring-for-unstructured-data)
+
+### 3.1. Introduction to NLP and LLM monitoring
+
+NLP and LLM models are widely used for both predictive and generative tasks.
+
+Predictive applications include:
+
+* Classification tasks, e.g., spam detection, classification of support tickets, evaluating text sentiment, etc.
+
+* Search (ranking) tasks are common in e-commerce website search, content recommendations, etc.
+
+* Information extraction, like extracting structured information (names, dates, etc.) from text, etc.
+
+Generative applications cover such tasks as translation, summarization, and text generation, e.g., conversational interfaces, article generation, code generation, etc.
+
+
+#### What can go wrong with NLP and LLMs?
+
+There are standard issues that apply to both unstructured and tabular data:
+
+* Technical errors such as wrong encoding or data processing bugs.
+
+* Data shifts like new topics or unexpected usage scenarios.
+
+However, there are also issues specific to working with unstructured data. For example:
+
+* Model attacks, e.g., prompt injection and adversarial usage.
+
+* Model behavior shift. For example, if we rely on third-party models, we can face changes in their properties due to retraining.
+
+* Hallucinations arise when the model starts to generate factually incorrect or unrelated answers.
+
+![image](https://github.com/user-attachments/assets/358aed90-5f96-4f16-be51-1ed9ac436672)
+
+#### ML monitoring metrics for NLP and LLMs
+
+![image](https://github.com/user-attachments/assets/d2c27ff2-b6e7-45df-94e2-1df96bb48de1)
+
+![image](https://github.com/user-attachments/assets/dcc5bce0-fea0-4547-8d5a-7b22b666f236)
+
+### 3.2. Monitoring data drift on raw text data
+
+#### Challenges of monitoring raw text data
+
+Handling raw text data is more complex than dealing with structured tabular data. With structured data, we can usually define “good” or “expected” data, e.g., particular feature distributions or statistical values can signal the data quality. For unstructured data, there is no straightforward way to define data quality or extract a signal for raw text data.
+
+When it comes to data drift detection, we can use two strategies that rely on raw text data: domain classifier and topic modeling
+
+![image](https://github.com/user-attachments/assets/bc4d741b-eb5a-487e-8e0e-cf6f7e998bf8)
+
+#### Domain classifier
+
+Domain classifier method or model-based drift detection uses a classifier model to compare distributions of reference and current datasets by training a model that predicts to which dataset a specific text belongs. If the model can confidently identify which text samples belong to the current or reference dataset, the two datasets are probably sufficiently different.
+
+![image](https://github.com/user-attachments/assets/497bc334-5145-411e-ab70-451510d598f0)
+
+![image](https://github.com/user-attachments/assets/d9217c9b-3af2-4524-ae1a-8115d227aada)
+
+#### Topic modeling
+
+Another strategy for evaluating raw data quality is topic modeling. The goal here is to categorize text into interpretable topic clusters, so instead of a binary classification model, we use a clustering model.
+
+How it works:
+
+* Apply the clustering model to new batches of data.
+
+* Monitor the size and share of different topics over time.
+
+* Changes in topics can indicate data drift.
+
+Using this method can be challenging due to difficulties in building a good clustering model:
+
+* There is no ideal structure in clustering.
+
+* Typically, it requires manual tuning to build accurate and interpretable clusters.
+
+![image](https://github.com/user-attachments/assets/c1b3c8c5-0ff2-47a5-8499-add8a4471f05)
+
+### 3.3. Monitoring text data quality and data drift with descriptors
+
+What is a text descriptor?
+A text descriptor is any feature we can derive or calculate from raw text. Text descriptor transforms unstructured data (e.g., text) into structured data (e.g., numeric or categorical descriptors).
+
+Depending on wer goal and problem statement, there are various groups of descriptors we can calculate:
+
+* Text data quality. Example descriptors are text length, the share of out-of-vocabulary words, the share of non-letter characters, regular expressions match, etc.
+
+* Text contents. For instance, we can monitor the presence of trigger words like mentions of specific brands or competitors or text sentiment. Collaborating with product managers or business teams is recommended to determine the meaningful text descriptors for wer problem statement and domain area.
+
+
+#### Monitoring text descriptors
+
+Once you have a structured representation of the unstructured data – in our case, text descriptors – you can use monitoring techniques applied to structured data. For example, you can:
+
+Measure descriptor distribution drift.
+
+Run rule-based checks, such as min-max ranges for text length, the expected share of non-letter symbols, or responses that match a regular expression.
+
+Track any statistics calculable for tabular data, e.g., correlation changes between descriptors and model target.
+
+![image](https://github.com/user-attachments/assets/f71be3dd-911e-4a76-9a2f-f82fe4e30749)
+
+### 3.4. Monitoring embeddings drift
+
+![image](https://github.com/user-attachments/assets/8ca1e07c-bc7d-4dd5-8b5a-4cc899ca204f)
+
+Embedding drift detection methods
+
+Since embeddings are numerical values, we can use many methods to monitor embedding distribution drift.
+
+Distance metrics Each object in the dataset, represented as an embedding, is a numerical vector. Distance metrics allow calculating distances between these vectors. For example, you can use Euclidean distance or Cosine distance (to assess the angle between vectors). We can detect shifts in datasets by measuring the distance between centroids of reference data and current data.
+
+![image](https://github.com/user-attachments/assets/708fc051-e408-42d3-b532-1588dd994be9)
+
+Model-based drift detection This approach uses embeddings to build a domain classifier that distinguishes between reference and current data. The idea is similar to model-based drift detection on raw text data: you get an estimation of the model's ability to distinguish between datasets. However, with embeddings, this approach has a limitation: you cannot use the information about the strongest features or best objects to determine the root cause/source of drift.
+
+![image](https://github.com/user-attachments/assets/9db418b5-f204-417c-9be1-448d712bee68)
+
+Share of drifted components You can also use the share of drifted components to monitor embedding drift. This approach treats each embedding component independently and uses drift detection methods that can be applied to numerical values. For each component, drift size or score is assessed. You can then aggregate these individual scores into the number of drifted components or share of drifted components.
+
+![image](https://github.com/user-attachments/assets/d57feee3-19ba-4cc7-a2a5-708aeef60406)
+
+### Practical
+
+Setup
+
+```python
+import pandas as pd
+import numpy as np
+
+from sklearn import datasets, ensemble, model_selection
+
+from evidently import ColumnMapping
+from evidently.report import Report
+
+from evidently.metric_preset import TextOverviewPreset
+
+from evidently.metrics import ColumnSummaryMetric
+from evidently.metrics import ColumnCorrelationsMetric
+from evidently.metrics import ColumnDistributionMetric
+from evidently.metrics import ColumnDriftMetric
+from evidently.metrics import ColumnValueRangeMetric
+from evidently.metrics import DatasetCorrelationsMetric
+from evidently.metrics import DatasetDriftMetric
+from evidently.metrics import DataDriftTable
+from evidently.metrics import EmbeddingsDriftMetric
+
+from evidently.metrics.data_drift.embedding_drift_methods import model, distance, ratio, mmd
+
+from evidently.descriptors import TextLength, TriggerWordsPresence, OOV, NonLetterCharacterPercentage
+from evidently.descriptors import SentenceCount, WordCount, Sentiment, RegExp
+```
+
+Import data
+
+```python
+import nltk
+nltk.download('words')
+nltk.download('wordnet')
+nltk.download('omw-1.4')
+nltk.download('vader_lexicon')
+
+reviews_data = datasets.fetch_openml(name='Womens-E-Commerce-Clothing-Reviews', version=2, as_frame='auto')
+reviews = reviews_data.frame
+
+reviews['prediction'] = reviews['Rating']
+reviews_ref = reviews[reviews.Rating > 3].sample(n=5000, replace=True, ignore_index=True, random_state=42)
+reviews_cur = reviews[reviews.Rating < 3].sample(n=5000, replace=True, ignore_index=True, random_state=42)
+
+column_mapping = ColumnMapping(
+    numerical_features=['Age', 'Positive_Feedback_Count'],
+    categorical_features=['Division_Name', 'Department_Name', 'Class_Name'],
+    text_features=['Review_Text', 'Title']
+)
+```
+
+Text data overview report
+
+```python
+text_overview_report = Report(metrics=[
+    TextOverviewPreset(column_name="Review_Text")
+])
+
+text_overview_report.run(reference_data=reviews_ref, current_data=reviews_cur, column_mapping=column_mapping)
+text_overview_report.show(mode='inline')
+```
+
+Sample graphs in the report:
+
+<img width="1260" alt="image" src="https://github.com/user-attachments/assets/a8840c52-fcee-4b2f-9872-d454cb220f41">
+
+<img width="1325" alt="image" src="https://github.com/user-attachments/assets/0ddfda0e-9968-479f-9e27-ed707d8cb3f2">
+
+<img width="1330" alt="image" src="https://github.com/user-attachments/assets/0af43081-aa49-403d-a212-9ef575af4f39">
+
+<img width="1316" alt="image" src="https://github.com/user-attachments/assets/62c0c45d-9bae-435b-abdf-2a3f2d55f62a">
+
+<img width="1306" alt="image" src="https://github.com/user-attachments/assets/c38c6682-4147-4883-a177-ed18535bd58f">
+
+Text data drift with descriptors
+
+```python
+table_column_metrics_report = Report(metrics=[
+    DatasetDriftMetric(columns=["Age", "Review_Text"]),
+    DataDriftTable(columns=["Age", "Review_Text"]),
+
+])
+
+table_column_metrics_report.run(reference_data=reviews_ref, current_data=reviews_cur, column_mapping=column_mapping)
+table_column_metrics_report.show(mode='inline')
+```
+
+<img width="1366" alt="image" src="https://github.com/user-attachments/assets/a6f0d2a7-6241-447e-a5a1-04f4d249dff6">
+
+Extra info in each dropdown
+
+<img width="1289" alt="image" src="https://github.com/user-attachments/assets/d59f98bf-12ee-4419-bed0-eca02b6cd0ce">
+
+Embedding Drift Detection
+
+```python
+
+embeddings_data = datasets.fetch_lfw_people()
+embeddings_data = pd.DataFrame(embeddings_data['data'])
+embeddings_data.columns = ['col_' + str(x) for x in embeddings_data.columns]
+
+embeddings_data = embeddings_data.iloc[:5100, :31]
+
+embeddings_data_shifted = embeddings_data.copy()
+embeddings_data_shifted.iloc[2500:5000, :5] = 0
+
+column_mapping = ColumnMapping(
+    embeddings={'small_subset': embeddings_data.columns[:10], 'big_subset': embeddings_data.columns[10:29]},
+    target=embeddings_data.columns[30]
+)
+
+report = Report(metrics=[
+    EmbeddingsDriftMetric('small_subset')
+])
+
+report.run(reference_data = embeddings_data[:2500], current_data = embeddings_data[2500:5000], 
+           column_mapping = column_mapping)
+report.show(mode='inline')
+```
+
+<img width="1371" alt="image" src="https://github.com/user-attachments/assets/a57221f1-23f2-4aaf-b715-bccaf0a7607e">
+
+Use a specific drift detection method:
+
+```python
+report = Report(metrics = [
+    EmbeddingsDriftMetric('small_subset', 
+                          drift_method = ratio(
+                              component_stattest = 'wasserstein',
+                              component_stattest_threshold = 0.1,
+                              threshold = 0.2,
+                              pca_components = None,
+                          )
+                         )
+])
+
+report.run(reference_data = embeddings_data_shifted[:2500], current_data = embeddings_data_shifted[2500:5000],  
+           column_mapping = column_mapping)
+report.show(mode='inline')
+```
+
+<img width="1376" alt="image" src="https://github.com/user-attachments/assets/66f4a1cc-a16f-4495-b905-8700cd6052ed">
+
+
+This course/tutorial/explanation series is amazing. AI monitoring is crucial and I am glad I found this great course to improve upon this skill. There are 3 more modules to cover (hopefully in the next days).
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From a24d113a7fcb822c58ee2c0bbb8c5f592e9b9622 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 6 Sep 2024 01:30:16 +0900
Subject: [PATCH 412/638] Create 2024-09-05-day-248.md

---
 _posts/2024-09-05-day-248.md | 46 ++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 _posts/2024-09-05-day-248.md

diff --git a/_posts/2024-09-05-day-248.md b/_posts/2024-09-05-day-248.md
new file mode 100644
index 000000000..58e11cf50
--- /dev/null
+++ b/_posts/2024-09-05-day-248.md
@@ -0,0 +1,46 @@
+---
+layout: post
+title: (Day 248) New LLM project from my advisor
+categories: [nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 248!
+A quick summary of today:
+* create a fake movie review dataset
+* use LoRA to fine-tune llama3
+
+So ~ I cannot share what the exact project is at this moment, or the process. However, to learn more about it I created the below dataset and learned how to fine-tune Llama3.
+
+Who is my advising professor? Song Hee-Seok (송희석) of the Management Information Systems department at my university. His latest interests are also AI, ML and LLMs and their applications. Today I went to meet with him for the 1st time this semester and he shared with me about the project. As mentioned at the start I will not share the exact details as the project, and even the dataset is still being created.
+
+Nevertheless, to prepare myself for the project I created some basic dataset and learned how to fine-tune Llama3. Below is all the info.
+
+### [Creating a sample movie review dataset](https://huggingface.co/datasets/divakaivan/fake_movie_review_kr)
+
+The dataset is in Korean as the project data will be in Korean too. Below is info about the type of data in the dataset. 
+
+Using an online movie review dataset, I modified it so that each row of data consists of 50 individual reviews for 1 movie. Each review is some short piece of comment about a movie.
+
+Once I did that, I passed each row of data (contained 50 movie reviews) to gpt-4o-mini (cost me ~$0.25) using the following prompt:
+
+```
+You are a helpful assistant that helps me evaluate Korean reviews. For each movie you are given 50 reviews. Analyze the reviews, and for the movie itself return a score(1 to 10) and explanation for each of the following criteria: Emotional, Characters, Plot, Visuals, Pacing. Return the review in Korean.
+```
+
+So the resulting data is all in the link above, uploaded to HuggingFace. There are two columns: 1 for the reviews, and a 2nd where the model evaluating the movie based on the factors in the prompt. I cannot say exactly why I did it this way, but it will be helpful later for the actual project data. 
+
+### [Using Parameter-Efficient Fine-Tuning (PEFT) to fine-tune Llama3](https://huggingface.co/divakaivan/llama3-finetuned-reviewer-kr)
+
+The above link will take you to the fine-tuned llama3-8b that I uploaded to HuggingFace. 
+
+On HF, I found this company Unsloth that has a [nice notebook](https://colab.research.google.com/drive/135ced7oHytdxu3N2DNe1Z0kqjyYIkDXp) for PEFT on popular models, and llama3-8b is one of them.
+
+After I created the dataset, I started following the code in the notebook to fine-tune llama3-8b on my own data and get a fine-tuned model (which I uploaded to HF as well). 
+
+---
+
+Very excited for this project, but the official data that will come from the customer (again I cannot share any details) will come after around a month. During that time I will keep learning, and explore other PEFT methods and try them out on my sample dataset.
+
+That is all for today!
+
+See you tomorrow :)

From c7eef187bf39568d6210e67c637f9381c843ca44 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 6 Sep 2024 22:21:39 +0900
Subject: [PATCH 413/638] Create 2024-09-06-day-249.md

---
 _posts/2024-09-06-day-249.md | 77 ++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 _posts/2024-09-06-day-249.md

diff --git a/_posts/2024-09-06-day-249.md b/_posts/2024-09-06-day-249.md
new file mode 100644
index 000000000..7fcd6d8c8
--- /dev/null
+++ b/_posts/2024-09-06-day-249.md
@@ -0,0 +1,77 @@
+---
+layout: post
+title: (Day 249) Seoul Tech Impact hackathon tomorrow
+categories: [nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 249!
+A quick summary of today:
+* Seoul tech impact team
+* joined a team project to build a robot
+
+## Found a team for the [Seoul Tech Impact hackathon](https://www.seoultechimpact.com/) 🥳
+
+I don't think I have talked about it yet, but some time ago I applied to be part of Seoul Tech Impact - an annual hackathon organised in Seoul for devs from any corner of software/hardware dev and any experience level. I applied, and got in. The hackathon has a discord channel for all the participants, I introduced myself and have been looking for teams to join. Thankfully, today another participant messaged me, proposing they want to make some kind of an LLM that uses free models. They also said that their main goal is to learn. And I was definitely on board as my main goal is to learn as well, practice my soft skills (teamwork, communication, facing challenges, fast-pace) and have fun in my 1st hackathon. 
+
+The hackathon is tomorrow, so this is great timing. I checked how to load some free LLMs, using Ollama, HuggingFace, and LangChain.
+
+For ollama, first I pulled a model, like phi3 or llama3.1
+
+```bash
+ollama pull llama3.1
+```
+
+And this runs on `http://localhost:11434/v1/`, which I can then use in my code.
+
+Using OpenAI:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url='http://localhost:11434/v1/',
+    api_key='ollama',
+)
+
+def llm(prompt):
+    response = client.chat.completions.create(
+        model='llama3.1',
+        messages=[{"role": "user", "content": prompt}]
+    )
+    
+    return response.choices[0].message.content
+
+llm('How are you?')
+```
+
+Or, using LangChain:
+
+```python
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_ollama.llms import OllamaLLM
+
+template = """Question: {question}
+"""
+
+prompt = ChatPromptTemplate.from_template(template)
+
+model = OllamaLLM(model="llama3.1")
+
+chain = prompt | model
+
+chain.invoke({"question": "What is LangChain?"})
+```
+
+## Innovative Product Design course
+
+Today was the 1st day of IPD in the [Hannam Design Factory](http://df.hannam.ac.kr/kor/sub02/menu_02.html). It was from 9am to 6pm. 9 company and potential projects were presented to the students, and I selected a company called [Rastech ](http://www.rastech.co.kr/) who is looking for a robot that can help with security and patrolling. Me, in a group to make a robot? Well, the main idea of this course is to gather students from different backgrounds - finance, business, design, media, engineering, AI, and develop a project. So this is where I fit in - finance and AI. We will be working this as if it is a real project, with reports, a prototype, brochures, presentations. It is not only an opportunity to apply my skills to a real project (and also go deeper into computer vision), but also I will have the opportunity to work in a team with people from very different academic backgrounds and knowledge, and I can learn about different aspects of product development, while also helping my teammates by sharing my knowledge. 
+
+We took photos, discussed the project, and did some team building exercises.
+
+---
+
+Tomorrow, the hackathon is a 24 hour event, so I will see how I will post for day 250. Of course I can sleep, but I will try to maximise my dev hours and commit to the real hackathon spirit of non-stop coding. 
+
+That is all for today!
+
+See you tomorrow :)

From eced0d2b77817f0231fb9777299924a2b147b896 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 8 Sep 2024 01:00:15 +0900
Subject: [PATCH 414/638] Create 2024-09-07-day-250.md

---
 _posts/2024-09-07-day-250.md | 83 ++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 _posts/2024-09-07-day-250.md

diff --git a/_posts/2024-09-07-day-250.md b/_posts/2024-09-07-day-250.md
new file mode 100644
index 000000000..9f8f0a09f
--- /dev/null
+++ b/_posts/2024-09-07-day-250.md
@@ -0,0 +1,83 @@
+---
+layout: post
+title: (Day 250) Seoul Tech Impact Day 1
+categories: [nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 250!
+A quick summary of today:
+* creating a language learning chatbot for expressions in different real-life situations
+
+
+I am awake, 'hacking' and I am writing this blog quickly so that I can keep working on our app. 
+
+Here are some pics from today:
+
+![image](https://github.com/user-attachments/assets/e9f18268-b55d-4be9-9e11-a722052ca58a)
+![image](https://github.com/user-attachments/assets/a7b566d3-2a95-45db-9f6d-6cb6435a72ef)
+![image](https://github.com/user-attachments/assets/14519e2b-8484-4949-98af-5b294fbc4fc3)
+
+As I mentioned, we are creating a chatbot. 
+
+This is our Figma graph right now:
+
+<img width="767" alt="image" src="https://github.com/user-attachments/assets/33996b50-6d25-477a-8a1f-0a08c31b5c3a">
+
+I am working on implementing chat history and feedback. I have setup postgres using Docker, and set up saving the chat history in the db. But I have been struggling to save user feedback. The reason is, it needs to be done using some buttons/submit form. But any time a button is clicked after a response comes from the RAG, the UI resets and nothing is executed. 
+
+Here is the problematic code:
+
+```python
+if authentication_status:
+    st.write(f'Welcome *{name}*')
+    authenticator.logout('Logout', 'main')
+    user_query = st.text_input("Enter your question:")
+
+    if st.button("Search"):
+        if user_query:
+            st.session_state.last_query = user_query  # Store the last query
+            with st.spinner("Searching..."):
+                answer, token_usage, response_time, search_results = rag(user_query)
+                st.success("Query Completed!")
+                answer_data = {
+                    'answer': answer,
+                    'response_time': response_time,
+                    'token_usage': token_usage,
+                    'search_results': str(search_results)
+                }
+                st.write(answer)
+                save_conversation(st.session_state.conversation_id, user_query, answer_data, username)
+                
+                # Display feedback options
+                st.session_state.feedback = st.radio(
+                    "Was this answer helpful?", 
+                    options=[0, 1], 
+                    format_func=lambda x: "👍" if x == 1 else "👎", 
+                    key='feedback_radio'
+                )
+################## HERE
+                if st.button("Submit Feedback"):
+                    if st.session_state.feedback is not None:
+                        feedback_value = 1 if st.session_state.feedback == 1 else -1  # Map radio button value to -1, +1
+                        save_feedback(username, st.session_state.conversation_id, feedback_value)
+                        st.success("Feedback submitted!")
+                    else:
+                        st.warning("Please provide feedback before submitting.")
+#######################
+        else:
+            st.warning("Please enter a query.")
+```
+
+This is what I have right now. It does not work, but I am taking a 10 min break to write this blog. Other options with just buttons don't work either.
+
+Anyway ~ hopefully I figure it out.
+
+Meanwhile, my teammate is working on prompt engineering and setting up the UI.
+
+This is kind of quick but I still wanted to share something. 
+
+I will update if I survived not sleeping for 24 hrs.
+
+That is all for today!
+
+See you tomorrow :)

From 37592d1de1e68b44659b3a1336880cf3e9fcafcf Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 9 Sep 2024 01:11:48 +0900
Subject: [PATCH 415/638] Create 2024-09-08-day-251.md

---
 _posts/2024-09-08-day-251.md | 56 ++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)
 create mode 100644 _posts/2024-09-08-day-251.md

diff --git a/_posts/2024-09-08-day-251.md b/_posts/2024-09-08-day-251.md
new file mode 100644
index 000000000..53b173ca6
--- /dev/null
+++ b/_posts/2024-09-08-day-251.md
@@ -0,0 +1,56 @@
+---
+layout: post
+title: (Day 251) LingoMate and first hackathon completed
+categories: [nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 251!
+A quick summary of today:
+* finished LingoMate - our project for the Seoul Tech Impact hackathon
+
+
+I have not slept in more than 24 hours now, so this one will be quite short.
+
+[Here](https://github.com/divakaivan/LingoMate) is the repo for our project. In here, is all the info and code of our hackathon project.
+
+I also recorded a 1 min [video](https://youtu.be/CVNY7Xi1Png) as a demo (which was part of our ppt)
+
+And here is some info from our ppt
+
+### What it does
+One key to learning a new language is consistent interaction with native speakers and exposure to diverse expressions. LingoMate offers user-friendly, interactive apps that make language learning simple and engaging. Compared to books and video courses, LingoMate is easy to use, simple, and accessible.
+
+### How we built it
+- **Ollama 3.1:** One of the LLMs we have used.
+- **Grafana:** Used for monitoring and tracking RAG performance/response.
+- **ElasticSearch:** Used as the vector database.
+- **Postgres:** Used to save user feedback and conversation histories.
+- **Streamlit:** Implemented the interface with limited resources.
+
+### Challenges we ran into
+- Optimizing prompts to yield appropriate/consistent answers.
+- Finding/preparing datasets.
+- Enhancing RAG tracing.
+
+### Accomplishments that we're proud of
+- I was new to AI, and I am proud that I could absorb/experience LLM in such a short time.
+- It was good to put myself in a learner's shoes (who I used to be) and explore how we can help language learners.
+
+### What's next for LingoMate
+- Expand into multiple languages. Offer more combinations of input/output languages to scale it up.
+- Scale up datasets, cover broader topics, and enhance model performance.
+- Offer features to track histories of conversations for a user.
+- Enhance UI.
+- With data-driven recommendations, we are aiming to let users personalize their learning.
+
+Finally, below is a pic from the stream when we were presenting:
+
+![image](https://github.com/user-attachments/assets/8b6944db-5fd2-4c4b-8232-de43c8a901ff)
+
+---
+
+Time to sleep 😴
+
+That is all for today!
+
+See you tomorrow :)

From 1a1d614c6156a4f35d900419dd4139cdd493c241 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 10 Sep 2024 00:39:00 +0900
Subject: [PATCH 416/638] Create 2024-09-09-day-252.md

---
 _posts/2024-09-09-day-252.md | 127 +++++++++++++++++++++++++++++++++++
 1 file changed, 127 insertions(+)
 create mode 100644 _posts/2024-09-09-day-252.md

diff --git a/_posts/2024-09-09-day-252.md b/_posts/2024-09-09-day-252.md
new file mode 100644
index 000000000..9a9a6ba62
--- /dev/null
+++ b/_posts/2024-09-09-day-252.md
@@ -0,0 +1,127 @@
+---
+layout: post
+title: (Day 252) Designing effective ML monitoring with EvidentlyAI
+categories: [nlp,mlops]
+---
+
+# Hello :) Today is Day 252!
+A quick summary of today:
+* covered first 3 sections of module 4
+
+Tomorrow I have an exam so today my study time for this blog was a bit limited. Nevertheless, I got to cover half of Module 4 from EvidentlyAI's course on AI monitoring.
+
+## 4.1. Logging for ML monitoring
+
+What is a good ML monitoring system?
+A good ML monitoring system consists of three key components:
+
+* Instrumentation to ensure collection and computation of useful metrics for analyzing model behavior and resolving issues.
+
+* Alerting to define unexpected model behavior through metrics and thresholds and design action policy.
+
+* Debugging to provide engineers with context to understand model issues for faster resolution.
+
+![image](https://github.com/user-attachments/assets/3ee4bbc5-d809-44b8-af53-a214ec9c70d4)
+
+### Logging and instrumentation
+
+Step 1
+
+![image](https://github.com/user-attachments/assets/cd091fcb-9a06-453f-9548-0443a6c8d193)
+
+Step 2
+
+![image](https://github.com/user-attachments/assets/a85536ff-697f-4662-b3bc-a964bb32a91b)
+
+Step 3
+
+![image](https://github.com/user-attachments/assets/226cc5f6-b296-4d7d-94d5-2a32dd84b83f)
+
+## 4.2. How to prioritize ML monitoring metrics
+
+![image](https://github.com/user-attachments/assets/2d5c3144-e56c-4c45-bba9-dd78d7a18da0)
+
+![image](https://github.com/user-attachments/assets/9015044e-f082-4523-9bd4-f965518fbf50)
+
+![image](https://github.com/user-attachments/assets/63956970-c4cd-4fee-b935-f6b0167be5a8)
+
+![image](https://github.com/user-attachments/assets/6749ce96-97df-4661-9ec7-36b7a66e8701)
+
+TLDR for metric prio
+
+![image](https://github.com/user-attachments/assets/14822511-695d-4efa-96a1-f3f136277b66)
+
+1. Service health
+
+![image](https://github.com/user-attachments/assets/85f71c24-c1be-48c4-bf1d-60ed487261f8)
+
+2. Model performance
+
+![image](https://github.com/user-attachments/assets/96626585-e38d-46d6-88f0-f73ea66ae0a2)
+![image](https://github.com/user-attachments/assets/86fe709a-aed0-48d6-be9b-f2c5f9974a56)
+
+3. Data quality and data integrity
+
+![image](https://github.com/user-attachments/assets/1abd68da-f7bd-4690-8c6c-9aee366f51a1)
+![image](https://github.com/user-attachments/assets/e6f9f5cd-3126-4fc5-8160-d4e53686ec71)
+
+4. Data and concept drift
+
+![image](https://github.com/user-attachments/assets/5947cbbb-4b0b-4bbb-832d-52d66f81dc69)
+![image](https://github.com/user-attachments/assets/be46f2b2-cb2f-4c9c-8396-f20ee2ac5fb7)
+
+### Comprehensive monitoring
+
+Depending on the problem statement and model usage scenario, we can introduce more comprehensive monitoring metrics:
+
+* Performance by segment. It can be especially useful if we deal with a diverse audience or complex object structures and want to monitor them separately.
+
+* Model bias and fairness. These metrics are crucial for sensitive domain areas like healthcare.
+
+* Outliers. Monitoring for outliers is vital when individual errors are costly.
+
+* Explainability. Explainability is important when users need to understand model decisions/outputs.
+
+## 4.3. When to retrain machine learning models
+
+### Model retraining strategies
+
+1. On Schedule
+
+![image](https://github.com/user-attachments/assets/5c1e7c3e-8db6-4d03-9d1a-b7acae7daaa0)
+
+Pro-tip for scheduled retraining: use historical data to determine the rate of model decay and the volume of new data required for effective retraining. For example, we can get a training set from our historical data and train a model on top of this dataset. Then, we can start experimenting: apply this model to new batches of data with a certain time step – daily, weekly, monthly – to measure how the model performs on the new data and define when its quality starts to degrade. Important note: we need labels to do it. If feedback/ground truth is not available yet, it makes sense to send data for labeling before we start experimenting with historical data.
+
+![image](https://github.com/user-attachments/assets/8a77cc6a-b4a6-4797-bc48-1617f2662a3b)
+
+2. Trigger-based retraining
+
+![image](https://github.com/user-attachments/assets/eec2b90d-d6c1-4690-907b-6deef85b47f7)
+
+### Model retraining tradeoffs
+
+![image](https://github.com/user-attachments/assets/614b2ac7-71a5-47f9-8ef3-a96747d5a48f)
+
+### Thinking through the retraining decision
+
+Be pragmatic. Develop a strategy considering available actions, service properties, resources, model criticality, and the cost of errors. Here is an example of a decision-making logic you can follow:
+
+![image](https://github.com/user-attachments/assets/e8499952-5cbe-428d-9fdd-8737dad9acc0)
+
+![image](https://github.com/user-attachments/assets/284ab602-e821-4113-97a7-9b1d583a6339)
+
+![image](https://github.com/user-attachments/assets/2794e2cf-8a6b-4916-ae4b-c7ffd19f9485)
+
+![image](https://github.com/user-attachments/assets/c6616793-ef24-422d-b95c-5e03eae5f9a9)
+
+![image](https://github.com/user-attachments/assets/e19ee268-e9c6-4a2f-b54b-b3e52e46c8af)
+
+![image](https://github.com/user-attachments/assets/dcd6be7c-f39a-4b6d-833e-3890ee98f39f)
+
+![image](https://github.com/user-attachments/assets/b4510b23-e0df-4639-996a-a552cd01d65e)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 8a41605b05469e3256c681d197c2ca0d4cde320d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 11 Sep 2024 01:06:29 +0900
Subject: [PATCH 417/638] Create 2024-09-10-day-253.md

---
 _posts/2024-09-10-day-253.md | 119 +++++++++++++++++++++++++++++++++++
 1 file changed, 119 insertions(+)
 create mode 100644 _posts/2024-09-10-day-253.md

diff --git a/_posts/2024-09-10-day-253.md b/_posts/2024-09-10-day-253.md
new file mode 100644
index 000000000..f6d521e7f
--- /dev/null
+++ b/_posts/2024-09-10-day-253.md
@@ -0,0 +1,119 @@
+---
+layout: post
+title: (Day 253) Designing effective ML monitoring with EvidentlyAI Part 2
+categories: [nlp,mlops]
+---
+
+# Hello :) Today is Day 253!
+A quick summary of today:
+* finished module 4 of evidenyliAI's course
+
+
+## 4.4. How to choose a reference dataset in ML monitoring
+
+### Why use a reference dataset?
+
+There are two main uses for a reference dataset.
+
+1. You can use it to derive test conditions automatically, saving time and effort in setting up tests manually.
+
+For example, you can use a reference dataset to generate conditions for data quality checks (to track feature ranges, share of missing values, etc.) and model quality checks (to keep tabs on metrics like precision and accuracy) by passing a previous batch of data as a reference.
+
+2. You can use a reference dataset as a baseline to detect data and prediction drift in production by comparing new data distributions against reference distributions.
+
+Reference dataset also can be used to detect training-serving skew as it provides a baseline to detect changes between training and production data.
+
+![image](https://github.com/user-attachments/assets/2747fea3-49e7-4554-90a8-e3d74efceeff)
+
+#### What makes a good reference dataset?
+
+Characteristics of a good reference dataset:
+
+* Reflects realistic data patterns, including cycles and seasonality.
+
+* Contains a large enough sample to derive meaningful statistics.
+
+* Includes realistic scenarios (e.g., sensor outages) to validate against new data.
+
+What a reference dataset is not:
+
+* It is not the same as a training dataset. You can sometimes choose training data to be your reference, but they are not synonymous.
+
+* It is not a “golden dataset,” which serves a different purpose.
+
+#### Using training data as a reference
+
+![image](https://github.com/user-attachments/assets/ed1a72ac-0152-4f35-ba2a-06d3bfbd67c0)
+
+![image](https://github.com/user-attachments/assets/a6947c38-8c77-4d39-b48a-551079ba505a)
+
+![image](https://github.com/user-attachments/assets/ddcb82c4-759a-433c-a922-6efb10063b90)
+
+![image](https://github.com/user-attachments/assets/8c46691f-be96-4775-9273-d181286cbfe9)
+
+![image](https://github.com/user-attachments/assets/8dbfd2b4-9452-490c-97fe-9a048017663f)
+
+#### Summing up
+
+* There is no “universal” reference dataset. It should be tailored to the specific use case and expectations of similarity to current data.
+
+* Hold-out validation data is preferred over training data for creating reference datasets, especially for drift detection. Use training data only if there is nothing else.
+
+* It is crucial to account for seasonality and historical patterns when choosing the reference dataset to ensure that it accurately represents the expected variations in data.
+
+* Historical data is a valuable resource for informing reference dataset selection.
+
+## 4.5. Custom metrics in ML monitoring
+
+### Types of custom metrics
+
+
+While there is no strict division between “standard” and “custom” metrics, there is some consensus on evaluating, for example, classification model quality using metrics like precision and recall. They are fairly “standard.”
+
+However, you often need to implement “custom” metrics to reflect specific aspects of model performance. They typically refer to business objectives or domain requirements and help capture the impact of an ML model within its operational context.
+
+Here are some examples.
+
+Business and product KPIs (or proxies). These metrics are aligned with key performance indicators that reflect the business goals and product performance.
+
+Examples include:
+
+* Manufacturing optimization: raw materials saved.
+
+* Chatbots: number of successful chat completions.
+
+* Fraud detection: number of detected fraud cases over $50,000.
+
+* Recommender systems: share of recommendation blocks without clicks.
+
+![image](https://github.com/user-attachments/assets/97a38da2-8490-480b-b198-d58311f1b0f2)
+
+![image](https://github.com/user-attachments/assets/19bc947a-b9a0-4fd9-8945-9be0d52b3495)
+
+![image](https://github.com/user-attachments/assets/87c97cc5-14fb-4a89-b59f-afc874cc582c)
+
+## 4.7. How to choose the ML monitoring deployment architecture
+
+![image](https://github.com/user-attachments/assets/1fee27fd-9faf-4fa6-b32e-ce54db8015a2)
+
+![image](https://github.com/user-attachments/assets/4a3efca2-9a69-466d-91b2-c21cff1e27a8)
+
+![image](https://github.com/user-attachments/assets/120dc5c0-b33e-4d28-a486-d3dc5e462472)
+
+![image](https://github.com/user-attachments/assets/386e1620-18c5-4652-9880-0525c1fa4751)
+
+![image](https://github.com/user-attachments/assets/97743eb8-2053-4a11-aedb-6acde07bcd64)
+
+![image](https://github.com/user-attachments/assets/d5e9f214-4fa2-475e-a58f-4dd80afdd6f7)
+
+![image](https://github.com/user-attachments/assets/94dd2464-64a2-441d-8516-ebbf30a320d7)
+
+![image](https://github.com/user-attachments/assets/b4504571-d446-4201-8b2e-dde2efe0d64a)
+
+---
+
+I had my exam today, but after it I am glad I found time to complete this module.
+
+That is all for today!
+
+See you tomorrow :)

From af0ed20ef3f4644ce02def6d97d3dad63b3c020e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 11 Sep 2024 17:25:40 +0900
Subject: [PATCH 418/638] Create 2024-09-11-day-254.md

---
 _posts/2024-09-11-day-254.md | 976 +++++++++++++++++++++++++++++++++++
 1 file changed, 976 insertions(+)
 create mode 100644 _posts/2024-09-11-day-254.md

diff --git a/_posts/2024-09-11-day-254.md b/_posts/2024-09-11-day-254.md
new file mode 100644
index 000000000..b36d69f24
--- /dev/null
+++ b/_posts/2024-09-11-day-254.md
@@ -0,0 +1,976 @@
+---
+layout: post
+title: (Day 254) save 1
+categories: []
+---
+
+# Hello :) Today is Day 254!
+A quick summary of today:
+
+## [Intro to LangGraph by LangChain](https://academy.langchain.com/courses/intro-to-langgraph)
+
+## Setup
+
+- Cloning the academy repo
+- running jupyter
+- setting up langsmith/openai env variables
+- setting up Tavily API (a search engine optimized for LLMs and RAG, aimed at efficient, quick, and persistent search results)
+- download [langgraph studio for mac](https://github.com/langchain-ai/langgraph-studio?tab=readme-ov-file#download)
+
+LangGraph is a framework developed by LangChain for building agent and multi-agent applications. Unlike LangChain's main package, LangGraph is specifically designed to give developers more control and precision in agent workflows, especially for complex real-world systems. Its goal is to help ensure that agents execute tasks more reliably by allowing developers to dictate the order in which tools are used or modify prompts based on the agent's current state. LangGraph is part of an effort to overcome the challenges of building agents that can perform tasks autonomously with high reliability.
+
+## Module 0: Using LangChain's chat models
+
+```python
+from langchain_openai import ChatOpenAI
+gpt4o_chat = ChatOpenAI(model="gpt-4o", temperature=0)
+gpt35_chat = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0)
+```
+
+Here is an example with OpenAI models. We have model and temperature. 
+
+Chat models in LangChain have a number of [default methods](https://python.langchain.com/v0.2/docs/concepts/#runnable-interface). For the most part, we'll be using:
+
+* `stream`: stream back chunks of the response
+* `invoke`: call the chain on an input
+
+Chat models take [messages](https://python.langchain.com/v0.2/docs/concepts/#messages) as input. Messages have a role (that describes who is saying the message) and a content property. We'll be talking a lot more about this later, but here let's just show the basics.
+
+```python
+from langchain_core.messages import HumanMessage
+
+# Create a message
+msg = HumanMessage(content="Hello world", name="Lance")
+
+# Message list
+messages = [msg]
+
+# Invoke the model with a list of messages 
+gpt4o_chat.invoke(messages)
+```
+
+Output:
+
+```
+AIMessage(content='Hello! How can I assist you today?', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 9, 'prompt_tokens': 11, 'total_tokens': 20}, 'model_name': 'gpt-4o-2024-05-13', 'system_fingerprint': 'fp_25624ae3a5', 'finish_reason': 'stop', 'logprobs': None}, id='run-9abd403f-f4de-4581-ba13-1c834a36c63d-0', usage_metadata={'input_tokens': 11, 'output_tokens': 9, 'total_tokens': 20})
+```
+
+## Module 1: Introduction
+
+Here is a FAQ that was provided and gave me a better idea of what's langgraph
+
+### LangGraph FAQ Summary
+
+- Do I need to use LangChain to use LangGraph?
+  No. LangGraph is a more low-level and controllable orchestration framework for complex agentic systems. LangChain, by contrast, provides a standard interface for interacting with models and components, suited for simpler tasks.
+
+- How is LangGraph different from other agent frameworks?  
+  Unlike other frameworks for simple tasks, LangGraph is designed to handle bespoke, complex tasks, offering more control without restricting users to a specific cognitive architecture.
+
+- Does LangGraph impact the performance of my app?  
+  No. LangGraph is designed for streaming workflows and will not add overhead to your code.
+
+- Is LangGraph open source? Is it free?  
+  Yes, LangGraph is an open-source library under the MIT license, and it’s free to use.
+
+- Is LangGraph Cloud open source?
+  No. LangGraph Cloud is proprietary software that will eventually become a paid service with certain tiers.
+
+- How do I enable LangGraph Cloud?  
+  LangGraph Cloud is currently in beta and accessible to LangSmith Plus and Enterprise plan users.
+
+- How are LangGraph and LangGraph Cloud different?  
+  LangGraph is a framework for building agent workflows, while LangGraph Cloud is a service for deploying, scaling, and debugging these applications. It also includes a Studio for prototyping.
+
+- How does LangGraph fit into the LangChain ecosystem? 
+  LangGraph is used to build stateful agents with streaming and human-in-the-loop support, while LangChain’s components aid development, and LangSmith assists in monitoring and optimizing production deployments. LangGraph Cloud helps turn LangGraph applications into production-ready systems.
+
+~ 
+
+A solitary llm is faily limited as it does not have access to tools, external context, or multi-step workflows. There are apps that have chains (a series of step, like retrieval, augmentation, generation, etc.). Chains are nice as they are deterministic because they will always follow the steps that we outline for them. What we want to do next, is give multiple chains to an LLM system and depending on the need, the system can decide on its own which chain to invoke. 
+
+<img width="932" alt="image" src="https://github.com/user-attachments/assets/95910a04-bd64-4d53-82b9-24baba52c6c8">
+
+<img width="991" alt="image" src="https://github.com/user-attachments/assets/5a867f5d-9c03-49e3-a74b-cbeae2062cc2">
+
+Module 1 structure:
+
+<img width="1029" alt="image" src="https://github.com/user-attachments/assets/40c21248-9749-4460-a372-7fad004a2910">
+
+### Simple graph
+
+3 nodes and 1 conditional edge
+
+![image](https://github.com/user-attachments/assets/fd72ca97-ba79-4f55-880c-7b4e90eef1fc)
+
+#### State
+
+First we need to define the [State](https://langchain-ai.github.io/langgraph/concepts/low_level/#state) of the graph. The State schema serves as the input schema for all Nodes and Edges in the graph.
+
+We can use TypedDict, and we give it 1 key: graph_state
+
+```python
+from typing_extensions import TypedDict
+
+class State(TypedDict):
+    graph_state: str
+```
+
+Then, we define Nodes
+
+#### Nodes
+
+[Nodes](https://langchain-ai.github.io/langgraph/concepts/low_level/#nodes) are just python functions. The first positional argument is the state, as defined above. Because the state is a `TypedDict` with schema as defined above, each node can access the key, `graph_state`, with `state['graph_state']`. Each node returns a new value of the state key `graph_state`. By default, the new value returned by each node [will override](https://langchain-ai.github.io/langgraph/concepts/low_level/#reducers) the prior state value.
+
+```python
+def node_1(state):
+    print("---Node 1---")
+    return {"graph_state": state['graph_state'] +" I am"}
+
+def node_2(state):
+    print("---Node 2---")
+    return {"graph_state": state['graph_state'] +" happy!"}
+
+def node_3(state):
+    print("---Node 3---")
+    return {"graph_state": state['graph_state'] +" sad!"}
+```
+
+#### Edges
+
+[Edges](https://langchain-ai.github.io/langgraph/concepts/low_level/#edges) connect the nodes. Normal Edges are used if you want to *always* go from, for example, `node_1` to `node_2`. [Conditional Edges](https://langchain-ai.github.io/langgraph/reference/graphs/?h=conditional+edge#langgraph.graph.StateGraph.add_conditional_edges) are used want to *optionally* route between nodes. Conditional edges are implemented as functions that return the next node to visit based upon some logic.
+
+```python
+import random
+from typing import Literal
+
+def decide_mood(state) -> Literal["node_2", "node_3"]:
+    
+    # Often, we will use state to decide on the next node to visit
+    user_input = state['graph_state'] 
+    
+    # Here, let's just do a 50 / 50 split between nodes 2, 3
+    if random.random() < 0.5:
+
+        # 50% of the time, we return Node 2
+        return "node_2"
+    
+    # 50% of the time, we return Node 3
+    return "node_3"
+```
+
+#### Graph construction
+
+Now, we build the graph from our [components](
+https://langchain-ai.github.io/langgraph/concepts/low_level/) defined above.
+
+The [StateGraph class](https://langchain-ai.github.io/langgraph/concepts/low_level/#stategraph) is the graph class that we can use.
+ 
+First, we initialize a StateGraph with the `State` class we defined above.
+ 
+Then, we add our nodes and edges.
+
+We use the [`START` Node, a special node](https://langchain-ai.github.io/langgraph/concepts/low_level/#start-node) that sends user input to the graph, to indicate where to start our graph.
+ 
+The [`END` Node](https://langchain-ai.github.io/langgraph/concepts/low_level/#end-node) is a special node that represents a terminal node. 
+
+Finally, we [compile our graph](https://langchain-ai.github.io/langgraph/concepts/low_level/#compiling-your-graph) to perform a few basic checks on the graph structure. 
+
+We can visualize the graph as a [Mermaid diagram](https://github.com/mermaid-js/mermaid).
+
+```python
+from IPython.display import Image, display
+from langgraph.graph import StateGraph, START, END
+
+# Build graph
+builder = StateGraph(State)
+builder.add_node("node_1", node_1)
+builder.add_node("node_2", node_2)
+builder.add_node("node_3", node_3)
+
+# Logic
+builder.add_edge(START, "node_1")
+builder.add_conditional_edges("node_1", decide_mood)
+builder.add_edge("node_2", END)
+builder.add_edge("node_3", END)
+
+# Add
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="219" alt="image" src="https://github.com/user-attachments/assets/f01208ab-0cbc-49ce-9d52-9e64ffee3a4d">
+
+#### Graph Invocation
+
+The compiled graph implements the [runnable](https://python.langchain.com/v0.1/docs/expression_language/interface/) protocol.
+
+This provides a standard way to execute LangChain components. 
+ 
+`invoke` is one of the standard methods in this interface.
+
+The input is a dictionary `{"graph_state": "Hi, this is lance."}`, which sets the initial value for our graph state dict.
+
+When `invoke` is called, the graph starts execution from the `START` node.
+
+It progresses through the defined nodes (`node_1`, `node_2`, `node_3`) in order.
+
+The conditional edge will traverse from node `1` to node `2` or `3` using a 50/50 decision rule. 
+
+Each node function receives the current state and returns a new value, which overrides the graph state.
+
+The execution continues until it reaches the `END` node.
+
+```python
+graph.invoke({"graph_state" : "Hi, this is Lance."})
+```
+
+Output: `{'graph_state': 'Hi, this is Lance. I am sad!'}`
+
+`invoke` runs the entire graph synchronously.
+
+This waits for each step to complete before moving to the next.
+
+It returns the final state of the graph after all nodes have executed.
+
+In this case, it returns the state after `node_3` has completed: 
+
+```
+{'graph_state': 'Hi, this is Lance. I am sad!'}
+```
+
+Node 1 appended `I am`, and then with a 50% chance we go to node 2 or 3, and append `happy` or `sad`
+
+I can open this graph in LangGraph studio as well
+
+![image](https://github.com/user-attachments/assets/89072172-2002-4471-8f87-80a97fa50030)
+
+In the provided academy repo, there is a studio folder with the following files that define the graph's logic:
+
+![image](https://github.com/user-attachments/assets/eec1fab6-8fe6-4417-9954-4841ad292d9b)
+
+and all I have to do is drag the folder in LangGraph Studio.
+
+I can interact with the graph, like in the notebook, and also see what each node did:
+
+![image](https://github.com/user-attachments/assets/0a3fd060-2e35-44f5-b02d-7647507ae253)
+
+### Chain
+
+Next, let's build up to a simple chain that combines 4 [concepts](https://python.langchain.com/v0.2/docs/concepts/):
+
+* Using [chat messages](https://python.langchain.com/v0.2/docs/concepts/#messages) as our graph state
+* Using [chat models](https://python.langchain.com/v0.2/docs/concepts/#chat-models) in graph nodes
+* [Binding tools](https://python.langchain.com/v0.2/docs/concepts/#tools) to our chat model
+* [Executing tool calls](https://python.langchain.com/v0.2/docs/concepts/#functiontool-calling) in graph nodes 
+
+<img width="943" alt="image" src="https://github.com/user-attachments/assets/a0220be4-717d-4567-98e2-8eb7bf1630cf">
+
+#### Messages
+
+Chat models can use [`messages`](https://python.langchain.com/v0.2/docs/concepts/#messages), which capture different roles within a conversation. 
+
+LangChain supports various message types, including `HumanMessage`, `AIMessage`, `SystemMessage`, and `ToolMessage`. 
+
+These represent a message from the user, from chat model, for the chat model to instruct behavior, and from a tool call. 
+
+Let's create a list of messages. 
+
+Each message can be supplied with a few things:
+
+* `content` - content of the message
+* `name` - optionally, a message author 
+* `response_metadata` - optionally, a dict of metadata (e.g., often populated by model provider for `AIMessages`)
+
+```python
+from pprint import pprint
+from langchain_core.messages import AIMessage, HumanMessage
+
+messages = [AIMessage(content=f"So you said you were researching ocean mammals?", name="Model")]
+messages.append(HumanMessage(content=f"Yes, that's right.",name="Lance"))
+messages.append(AIMessage(content=f"Great, what would you like to learn about.", name="Model"))
+messages.append(HumanMessage(content=f"I want to learn about the best place to see Orcas in the US.", name="Lance"))
+
+for m in messages:
+    m.pretty_print()
+```
+
+Output:
+
+```
+================================== Ai Message ==================================
+Name: Model
+
+So you said you were researching ocean mammals?
+================================ Human Message =================================
+Name: Lance
+
+Yes, that's right.
+================================== Ai Message ==================================
+Name: Model
+
+Great, what would you like to learn about.
+================================ Human Message =================================
+Name: Lance
+
+I want to learn about the best place to see Orcas in the US.
+```
+
+#### Chat models
+
+They can use a sequence of messages as input and support message types (like above). Below, OpenAI is used.
+
+```python
+from langchain_openai import ChatOpenAI
+llm = ChatOpenAI(model="gpt-4o")
+result = llm.invoke(messages)
+type(result)
+```
+
+Out:
+
+```
+AIMessage(content="Orcas, also known as killer whales, can be observed in several locations in the United States, but one of the best places to see them is the Pacific Northwest, particularly around the San Juan Islands in Washington State. \n\n### Key Locations:\n\n1. **San Juan Islands, Washington:**\n   - **Best Time:** The peak season is from May to September, but they can be seen year-round.\n   - **Why It's Great:** The nutrient-rich waters attract a variety of marine life, making it an ideal habitat for orcas. There are both resident and transient orca pods in the area.\n\n2. **Puget Sound, Washington:**\n   - **Best Time:** Summer months, primarily from May to September.\n   - **Why It's Great:** Puget Sound is home to the Southern Resident orcas, which are a unique and endangered population.\n\n3. **Monterey Bay, California:**\n   - **Best Time:** Spring and fall are the best times to see transient orcas.\n   - **Why It's Great:** The nutrient-rich waters of Monterey Bay attract a variety of marine life, including orcas, which come to prey on seals and sea lions.\n\n4. **Southeast Alaska:**\n   - **Best Time:** Summer months, from May to September.\n   - **Why It's Great:** The Inside Passage is a prime location for spotting orcas, along with humpback whales and other marine animals.\n\n### Tips for Whale Watching:\n- **Tours:** Consider booking a whale-watching tour with a reputable company that follows responsible wildlife viewing guidelines.\n- **Binoculars and Camera:** Bring binoculars for a closer look and a camera with a good zoom lens to capture the experience.\n- **Weather:** Dress in layers and prepare for variable weather conditions, especially if you're going out on a boat.\n\n### Conservation Note:\nOrcas face various threats, including habitat destruction, pollution, and reduced prey availability. Supporting conservation efforts and choosing eco-friendly tour operators can help protect these magnificent creatures for future generations.\n\nIs there anything more specific you’d like to know about orcas or whale watching?", additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 427, 'prompt_tokens': 67, 'total_tokens': 494}, 'model_name': 'gpt-4o-2024-05-13', 'system_fingerprint': 'fp_157b3831f5', 'finish_reason': 'stop', 'logprobs': None}, id='run-98914d63-e629-401b-a74e-dea0e68eff03-0', usage_metadata={'input_tokens': 67, 'output_tokens': 427, 'total_tokens': 494})
+```
+
+Metadata `result.response_metadata`:
+
+```
+{'token_usage': {'completion_tokens': 427,
+  'prompt_tokens': 67,
+  'total_tokens': 494},
+ 'model_name': 'gpt-4o-2024-05-13',
+ 'system_fingerprint': 'fp_157b3831f5',
+ 'finish_reason': 'stop',
+ 'logprobs': None}
+```
+
+#### Tools
+
+Tools are useful whenever you want a model to interact with external systems.
+
+External systems (e.g., APIs) often require a particular input schema or payload, rather than natural language. 
+
+When we bind an API, for example, as a tool we given the model awareness of the required input schema.
+
+The model will choose to call a tool based upon the natural language input from the user. 
+
+And, it will return an output that adheres to the tool's schema. 
+
+[Many LLM providers support tool calling](https://python.langchain.com/v0.1/docs/integrations/chat/) and [tool calling interface](https://blog.langchain.dev/improving-core-tool-interfaces-and-docs-in-langchain/) in LangChain is simple. 
+ 
+You can simply pass any Python `function` into `ChatModel.bind_tools(function)`.
+
+<img width="898" alt="image" src="https://github.com/user-attachments/assets/1c579133-7545-43bb-ba32-41cc6e305275">
+
+
+Here is an example function as a tool:
+
+```python
+def multiply(a: int, b: int) -> int:
+    """Multiply a and b.
+
+    Args:
+        a: first int
+        b: second int
+    """
+    return a * b
+
+llm_with_tools = llm.bind_tools([multiply])
+```
+
+If we pass an input - e.g., `"What is 2 multiplied by 3"` - we see a tool call returned. 
+
+The tool call has specific arguments that match the input schema of our function along with the name of the function to call.
+
+```
+{'arguments': '{"a":2,"b":3}', 'name': 'multiply'}
+```
+
+#### Using messages as state
+
+With these foundations in place, we can now use [`messages`](https://python.langchain.com/v0.2/docs/concepts/#messages) in our graph state.
+
+We can define our state `MessagesState` as a `TypedDict` with a single key: `messages`.
+
+`messages` is simply a list of messages, as we defined above (e.g., `HumanMessage`, etc).
+
+```python
+from typing_extensions import TypedDict
+from langchain_core.messages import AnyMessage
+
+class MessagesState(TypedDict):
+    messages: list[AnyMessage]
+```
+
+#### Reducers
+
+Now, we have a minor problem! 
+
+As discussed, each node will return a new value for our state key `messages`.
+
+But, this new value will [override](https://langchain-ai.github.io/langgraph/concepts/low_level/#reducers) the prior `messages` value.
+ 
+As our graph runs, we want to **append** messages to to our `messages` state key.
+ 
+We can use [reducer functions](https://langchain-ai.github.io/langgraph/concepts/low_level/#reducers) address this.
+
+Reducers allow us to specify how state updates are performed.
+
+If no reducer function is specified, then it is assumed that updates to the key should *override it* as we saw before.
+ 
+But, to append messages, we can use the pre-built `add_messages` reducer.
+
+This ensures that any messages are appended to the existing list of messages.
+
+We annotate simply need to annotate our `messages` key with the `add_messages` reducer function as metadata.
+
+
+```python
+from typing import Annotated
+from langgraph.graph.message import add_messages
+
+class MessagesState(TypedDict):
+    messages: Annotated[list[AnyMessage], add_messages]
+```
+
+Since having a list of messages in graph state is so common, LangGraph has a pre-built [`MessagesState`](https://langchain-ai.github.io/langgraph/concepts/low_level/#messagesstate)! 
+
+`MessagesState` is defined: 
+
+* With a pre-build single `messages` key
+* This is a list of `AnyMessage` objects 
+* It uses the `add_messages` reducer
+
+We'll usually use `MessagesState` because it is less verbose than defining a custom `TypedDict`, as shown above.
+
+```python
+from langgraph.graph import MessagesState
+
+class MessagesState(MessagesState):
+    # Add any keys needed beyond messages, which is pre-built 
+    pass
+```
+
+#### The graph
+
+```python
+from IPython.display import Image, display
+from langgraph.graph import StateGraph, START, END
+    
+# Node
+def tool_calling_llm(state: MessagesState):
+    return {"messages": [llm_with_tools.invoke(state["messages"])]}
+
+# Build graph
+builder = StateGraph(MessagesState)
+builder.add_node("tool_calling_llm", tool_calling_llm)
+builder.add_edge(START, "tool_calling_llm")
+builder.add_edge("tool_calling_llm", END)
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="141" alt="image" src="https://github.com/user-attachments/assets/07985b1f-4178-4708-b183-3264ff8f1f4a">
+
+If we pass in `Hello!`, the LLM responds without any tool calls.
+
+```python
+messages = graph.invoke({"messages": HumanMessage(content="Hello!")})
+for m in messages['messages']:
+    m.pretty_print()
+```
+
+```
+================================ Human Message =================================
+
+Hello!
+================================== Ai Message ==================================
+
+Hi there! How can I assist you today?
+```
+
+The LLM chooses to use a tool when it determines that the input or task requires the functionality provided by that tool.
+
+```python
+messages = graph.invoke({"messages": HumanMessage(content="Multiply 2 and 3!")})
+for m in messages['messages']:
+    m.pretty_print()
+```
+
+```
+================================ Human Message =================================
+
+Multiply 2 and 3!
+================================== Ai Message ==================================
+Tool Calls:
+  multiply (call_Er4gChFoSGzU7lsuaGzfSGTQ)
+ Call ID: call_Er4gChFoSGzU7lsuaGzfSGTQ
+  Args:
+    a: 2
+    b: 3
+```
+
+### Router
+
+Above, we saw that the graph can:
+
+* Return a tool call
+* Return a natural language response
+
+We can think of a router as a tool where the chat model routes between a direct response or a tool call based upon the user input.
+
+This is an simple example of an agent, where the LLM is directing the control flow either by calling a tool or just responding directly. 
+
+<img width="950" alt="image" src="https://github.com/user-attachments/assets/47806154-1182-4b68-8c1e-762ec4a25cf0">
+
+Below, we will extend the graph to work with either output
+
+First, we create the tool again:
+
+```python
+from langchain_openai import ChatOpenAI
+
+def multiply(a: int, b: int) -> int:
+    """Multiply a and b.
+
+    Args:
+        a: first int
+        b: second int
+    """
+    return a * b
+
+llm = ChatOpenAI(model="gpt-4o")
+llm_with_tools = llm.bind_tools([multiply])
+```
+
+ We use the [built-in `ToolNode`](https://langchain-ai.github.io/langgraph/reference/prebuilt/?h=tools+condition#toolnode) and simply pass a list of our tools to initialize it. 
+ 
+ We use the [built-in `tools_condition`](https://langchain-ai.github.io/langgraph/reference/prebuilt/?h=tools+condition#tools_condition) as our conditional edge.
+
+```python
+from IPython.display import Image, display
+from langgraph.graph import StateGraph, START, END
+from langgraph.graph import MessagesState
+from langgraph.prebuilt import ToolNode
+from langgraph.prebuilt import tools_condition
+
+# Node
+def tool_calling_llm(state: MessagesState):
+    return {"messages": [llm_with_tools.invoke(state["messages"])]}
+
+# Build graph
+builder = StateGraph(MessagesState)
+builder.add_node("tool_calling_llm", tool_calling_llm)
+builder.add_node("tools", ToolNode([multiply]))
+builder.add_edge(START, "tool_calling_llm")
+builder.add_conditional_edges(
+    "tool_calling_llm",
+    # If the latest message (result) from assistant is a tool call -> tools_condition routes to tools
+    # If the latest message (result) from assistant is a not a tool call -> tools_condition routes to END
+    tools_condition,
+)
+builder.add_edge("tools", END)
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="154" alt="image" src="https://github.com/user-attachments/assets/3c190ed0-8e0b-4cf6-86cd-4b7df1876abf">
+
+```python
+messages = graph.invoke({"messages": ("user", "Multiply 3 and 2")})
+for m in messages['messages']:
+    m.pretty_print()
+```
+
+```
+================================ Human Message =================================
+
+Multiply 3 and 2
+================================== Ai Message ==================================
+Tool Calls:
+  multiply (call_om27lpEIRsGdjQ9K78JUZtps)
+ Call ID: call_om27lpEIRsGdjQ9K78JUZtps
+  Args:
+    a: 3
+    b: 2
+================================= Tool Message =================================
+Name: multiply
+
+6
+```
+
+Here is doing the same thing but using LangGraph Studio:
+
+![image](https://github.com/user-attachments/assets/123f3e3f-0c80-434d-baf0-e2a97e029ba5)
+
+### Agent
+
+we can extend the above into a generic agent architecture.
+
+In the above router, we invoked the model and, if it chose to call a tool, we returned a `ToolMessage` to the user.
+ 
+But, what if we simply pass that `ToolMessage` *back to the model*?
+
+We can let it either (1) call another tool or (2) respond directly.
+
+This is the intuition behind [ReAct](https://react-lm.github.io/), a general agent architecture.
+  
+* `act` - let the model call specific tools 
+* `observe` - pass the tool output back to the model 
+* `reason` - let the model reason about the tool output to decide what to do next (e.g., call another tool or just respond directly)
+
+This [general purpose architecture](https://blog.langchain.dev/planning-for-agents/) can be applied to many types of tools. 
+
+<img width="870" alt="image" src="https://github.com/user-attachments/assets/fb39188c-9dd7-4977-b372-e4f9cb73ac31">
+
+
+Lets add some tools
+
+
+```python
+from langchain_openai import ChatOpenAI
+
+def multiply(a: int, b: int) -> int:
+    """Multiply a and b.
+
+    Args:
+        a: first int
+        b: second int
+    """
+    return a * b
+
+# This will be a tool
+def add(a: int, b: int) -> int:
+    """Adds a and b.
+
+    Args:
+        a: first int
+        b: second int
+    """
+    return a + b
+
+def divide(a: int, b: int) -> float:
+    """Adds a and b.
+
+    Args:
+        a: first int
+        b: second int
+    """
+    return a / b
+
+tools = [add, multiply, divide]
+llm = ChatOpenAI(model="gpt-4o")
+llm_with_tools = llm.bind_tools(tools)
+```
+
+Create an LLM
+
+```python
+from langgraph.graph import MessagesState
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# System message
+sys_msg = SystemMessage(content="You are a helpful assistant tasked with performing arithmetic on a set of inputs.")
+
+# Node
+def assistant(state: MessagesState):
+   return {"messages": [llm_with_tools.invoke([sys_msg] + state["messages"])]}
+```
+
+As before, we use `MessagesState` and define a `Tools` node with our list of tools.
+
+The `Assistant` node is just our model with bound tools.
+
+We create a graph with `Assistant` and `Tools` nodes.
+
+We add `tools_condition` edge, which routes to `End` or to `Tools` based on  whether the `Assistant` calls a tool.
+
+Now, we add one new step:
+
+We connect the `Tools` node *back* to the `Assistant`, forming a loop.
+
+* After the `assistant` node executes, `tools_condition` checks if the model's output is a tool call.
+* If it is a tool call, the flow is directed to the `tools` node.
+* The `tools` node connects back to `assistant`.
+* This loop continues as long as the model decides to call tools.
+* If the model response is not a tool call, the flow is directed to END, terminating the process.
+
+
+```python
+from langgraph.graph import START, StateGraph
+from langgraph.prebuilt import tools_condition
+from langgraph.prebuilt import ToolNode
+from IPython.display import Image, display
+
+# Graph
+builder = StateGraph(MessagesState)
+
+# Define nodes: these do the work
+builder.add_node("assistant", assistant)
+builder.add_node("tools", ToolNode(tools))
+
+# Define edges: these determine how the control flow moves
+builder.add_edge(START, "assistant")
+builder.add_conditional_edges(
+    "assistant",
+    # If the latest message (result) from assistant is a tool call -> tools_condition routes to tools
+    # If the latest message (result) from assistant is a not a tool call -> tools_condition routes to END
+    tools_condition,
+)
+builder.add_edge("tools", "assistant")
+react_graph = builder.compile()
+
+# Show
+display(Image(react_graph.get_graph(xray=True).draw_mermaid_png()))
+```
+
+<img width="205" alt="image" src="https://github.com/user-attachments/assets/515565d5-8400-4806-9441-9379cf4bde48">
+
+```python
+messages = [HumanMessage(content="Add 3 and 4, then multiply by 2, and finally divide by 5")]
+messages = react_graph.invoke({"messages": messages})
+for m in messages['messages']:
+    m.pretty_print()
+```
+
+```
+================================ Human Message =================================
+
+Add 3 and 4, then multiply by 2, and finally divide by 5
+================================== Ai Message ==================================
+Tool Calls:
+  add (call_s056B7Zg4pWzrBeb5t0ZqTM7)
+ Call ID: call_s056B7Zg4pWzrBeb5t0ZqTM7
+  Args:
+    a: 3
+    b: 4
+================================= Tool Message =================================
+Name: add
+
+7
+================================== Ai Message ==================================
+Tool Calls:
+  multiply (call_Wh2R0c53q9Kin7AeCtFcD5ng)
+ Call ID: call_Wh2R0c53q9Kin7AeCtFcD5ng
+  Args:
+    a: 7
+    b: 2
+================================= Tool Message =================================
+Name: multiply
+
+14
+================================== Ai Message ==================================
+Tool Calls:
+  divide (call_U9eDdbIfW9N2DbCPCKi0i6TA)
+ Call ID: call_U9eDdbIfW9N2DbCPCKi0i6TA
+  Args:
+    a: 14
+    b: 5
+================================= Tool Message =================================
+Name: divide
+
+2.8
+================================== Ai Message ==================================
+
+The result of the calculation is \(2.8\).
+```
+
+We can check it out in LangGraph Studio as well:
+
+![image](https://github.com/user-attachments/assets/5fd6ed05-a63d-458e-a283-5c71b870cd20)
+
+Now, the system responds with a natural language, rather than a tool_call
+
+In LangSmith, we can even see extra tracing:
+
+![image](https://github.com/user-attachments/assets/a59d3f71-6de6-4100-874a-0f5e06e1b7cc)
+
+### Agent with memory
+
+Lets introduce memory to our agent
+
+Setup for the agent is like the above section, but for memory, here is how to set it up:
+
+```python
+from langgraph.checkpoint.memory import MemorySaver
+memory = MemorySaver()
+react_graph_memory = builder.compile(checkpointer=memory)
+```
+
+When we use memory, we need to specify a `thread_id`.
+
+This `thread_id` will store our collection of graph states.
+
+Here is a cartoon:
+
+* The checkpointer write the state at every step of the graph
+* These checkpoints are saved in a thread 
+* We can access that thread in the future using the `thread_id`
+
+
+<img width="964" alt="image" src="https://github.com/user-attachments/assets/a0248c3c-2f2b-4cbb-aafa-6f3891192209">
+
+```python
+# Specify a thread
+config = {"configurable": {"thread_id": "1"}}
+
+# Specify an input
+messages = [HumanMessage(content="Add 3 and 4.")]
+
+# Run
+messages = react_graph_memory.invoke({"messages": messages},config)
+for m in messages['messages']:
+    m.pretty_print()
+```
+
+```
+================================ Human Message =================================
+
+Add 3 and 4.
+================================== Ai Message ==================================
+Tool Calls:
+  add (call_23tmdvhv1VAfMxIZnFBFHJD7)
+ Call ID: call_23tmdvhv1VAfMxIZnFBFHJD7
+  Args:
+    a: 3
+    b: 4
+================================= Tool Message =================================
+Name: add
+
+7
+================================== Ai Message ==================================
+
+The sum of 3 and 4 is 7.
+```
+
+If we pass the same `thread_id`, then we can proceed from from the previously logged state checkpoint! 
+
+In this case, the above conversation is captured in the thread.
+
+The `HumanMessage` we pass (`"Multiply that by 2."`) is appended to the above conversation.
+
+So, the model now know that `that` refers to the `The sum of 3 and 4 is 7.`.
+
+
+```python
+messages = [HumanMessage(content="Multiply that by 2.")]
+messages = react_graph_memory.invoke({"messages": messages}, config)
+for m in messages['messages']:
+    m.pretty_print()
+```
+
+```
+================================ Human Message =================================
+
+Add 3 and 4.
+================================== Ai Message ==================================
+Tool Calls:
+  add (call_23tmdvhv1VAfMxIZnFBFHJD7)
+ Call ID: call_23tmdvhv1VAfMxIZnFBFHJD7
+  Args:
+    a: 3
+    b: 4
+================================= Tool Message =================================
+Name: add
+
+7
+================================== Ai Message ==================================
+
+The sum of 3 and 4 is 7.
+================================ Human Message =================================
+
+Multiply that by 2.
+================================== Ai Message ==================================
+Tool Calls:
+  multiply (call_aFKM1qZxhnsW9YICm8xSeLpG)
+ Call ID: call_aFKM1qZxhnsW9YICm8xSeLpG
+  Args:
+    a: 7
+    b: 2
+================================= Tool Message =================================
+Name: multiply
+
+14
+================================== Ai Message ==================================
+
+The result of multiplying 7 by 2 is 14.
+```
+
+### Deployment
+
+There are a few central concepts to understand -
+
+`LangGraph` —
+- Python and JavaScript library 
+- Allows creation of agent workflows 
+
+`LangGraph API` —
+- Bundles the graph code 
+- Provides a task queue for managing asynchronous operations
+- Offers persistence for maintaining state across interactions
+
+`LangGraph Cloud` --
+- Hosted service for the LangGraph API
+- Allows deployment of graphs from GitHub repositories
+- Also provides monitoring and tracing for deployed graphs
+- Accessible via a unique URL for each deployment
+
+`LangGraph Studio` --
+- Integrated Development Environment (IDE) for LangGraph applications
+- Uses the API as its back-end, allowing real-time testing and exploration of graphs
+- Can be run locally or with cloud-deployment
+
+`LangGraph SDK` --
+- Python library for programmatically interacting with LangGraph graphs
+- Provides a consistent interface for working with graphs, whether served locally or in the cloud
+- Allows creation of clients, access to assistants, thread management, and execution of runs
+
+The tutorial showed how to deploy the above agent with tools using GitHub and LangSmith.
+
+But I cannot actually deploy a link at the moment. Nevertheless local is fine for now too. 
+
+![image](https://github.com/user-attachments/assets/e3cab1df-c7ce-45f2-9617-7b962c3bd3ee)
+
+--- 
+
+That is all for module 1. Tbh, extremely informative, and definitely learned a lot. Especially related to agents, and how those functions (like the multiply one) get put in the system and the LLM can use them. 
+
+
+## [Movie reviewer LLM](https://huggingface.co/divakaivan/llama3-finetuned-reviewer-kr)
+
+Last week, I mentioned about a project with one of my professors. I still cannot share anything related to it. But today I looked into evaluation metrics like ROUGE and BLEU. BLEU seems to be more for machine translation rather than our case. And seems like [ROUGE](https://huggingface.co/spaces/evaluate-metric/rouge) fits us better as it has multiple metrics inside of it.
+
+The `evaluate` module on Hugging Face provides multiple ROUGE (Recall-Oriented Understudy for Gisting Evaluation) metrics to assess the overlap between the model-generated text and the reference text. Here's the breakdown of the four different ROUGE metrics you're seeing:
+
+1. **ROUGE-1**: Measures the overlap of unigrams (individual words) between the generated and reference summaries. It gives an indication of how many single words from the reference are captured in the generated summary.
+
+2. **ROUGE-2**: Measures the overlap of bigrams (two consecutive words) between the generated and reference summaries. This metric takes into account the preservation of word sequences, reflecting how well the generated text captures small sequences of words from the reference.
+
+3. **ROUGE-L**: Measures the longest common subsequence (LCS) between the generated and reference summaries. It evaluates the longest sequence of words that appear in both texts in the same order but not necessarily consecutively. This metric captures a more flexible notion of similarity based on the overall structure of the text.
+
+4. **ROUGE-Lsum**: Specifically designed for evaluating summarization tasks, it computes the LCS at the sentence level, focusing on the structure of summaries. ROUGE-Lsum looks at how well the sentences in the generated summary align with those in the reference summary.
+
+And the value for each is between 0 and 1, 0 meaning completel different summaries, and 1 exactly the same, so it is a nice and easy metric to understand. I ran a test on my generated data, using huggingface's evaluate library (which contains all 4 ROUGE metrics from above), and I got: `{'rouge1': 0.7613263913669215, 'rouge2': 0.4409441673251364, 'rougeL': 0.6578334536498124, 'rougeLsum': 0.7542867701046572}`a
+
+However, I am concerned as in addition to text, the model outputs a score out of 10 for each movie category (storyline, characters, etc). And using metrics like ROUGE we are not evaluating the model based on these numbers. So as part of my update email to my professor I mentioned this concern and whether we should look into a specific model that outputs scores.
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From 8c9b2010eb02b2980fe8bb394bbc7617e6a30ec3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 11 Sep 2024 23:45:23 +0900
Subject: [PATCH 419/638] Update 2024-09-11-day-254.md

---
 _posts/2024-09-11-day-254.md | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/_posts/2024-09-11-day-254.md b/_posts/2024-09-11-day-254.md
index b36d69f24..62b664078 100644
--- a/_posts/2024-09-11-day-254.md
+++ b/_posts/2024-09-11-day-254.md
@@ -1,11 +1,13 @@
 ---
 layout: post
-title: (Day 254) save 1
-categories: []
+title: (Day 254) Started Intro to LangGraph by LangChain
+categories: [nlp,theory,applying-knowledge]
 ---
 
 # Hello :) Today is Day 254!
 A quick summary of today:
+* learning about agents with LangGraph
+* file reviewer LLM evaluation metrics
 
 ## [Intro to LangGraph by LangChain](https://academy.langchain.com/courses/intro-to-langgraph)
 
@@ -962,15 +964,8 @@ And the value for each is between 0 and 1, 0 meaning completel different summari
 However, I am concerned as in addition to text, the model outputs a score out of 10 for each movie category (storyline, characters, etc). And using metrics like ROUGE we are not evaluating the model based on these numbers. So as part of my update email to my professor I mentioned this concern and whether we should look into a specific model that outputs scores.
 
 
+---
 
+That is all for today!
 
-
-
-
-
-
-
-
-
-
-
+See you tomorrow :)

From d8280e715e2650352ef19a9b27f57be4282b26ed Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Sep 2024 23:49:57 +0900
Subject: [PATCH 420/638] Create 2024-09-12-day-255.md

---
 _posts/2024-09-12-day-255.md | 83 ++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 _posts/2024-09-12-day-255.md

diff --git a/_posts/2024-09-12-day-255.md b/_posts/2024-09-12-day-255.md
new file mode 100644
index 000000000..7627f765e
--- /dev/null
+++ b/_posts/2024-09-12-day-255.md
@@ -0,0 +1,83 @@
+---
+layout: post
+title: (Day 255) The Little Book of Deep Learning
+categories: [theory,deep-learning]
+---
+
+# Hello :) Today is Day 255!
+A quick summary of today:
+* found and read [this book](https://fleuret.org/francois/lbdl.html) by Francois Fleuret
+
+![image](https://github.com/user-attachments/assets/3664aa56-f583-40ea-9212-1fee5eda6ddb)
+
+I randomly saw the author of the book post about it on LinkedIn I think. As a fan of short and to the point books I immediately thought I would like it. It's aim is not to go in depth for any concept but rather provide a quick overview, and even a reminder to the reader about the concepts in deep learning.
+
+What is more, the book is completely free, there is a print version made for A5 paper so I decided to print it, read it and add it to my library.
+
+Below is the table of contents.
+
+<img width="427" alt="image" src="https://github.com/user-attachments/assets/c7351e36-16dc-4d84-9164-c456aebe87e7">
+
+<img width="426" alt="image" src="https://github.com/user-attachments/assets/43c00b13-67d4-4738-a2ce-b41a5592ce5b">
+
+<img width="423" alt="image" src="https://github.com/user-attachments/assets/7208be9d-9d48-4b20-8dbc-4006cf681443">
+
+It is around 150 pages, and today I finished it. I am glad I found this book, and after reading it I like it as it is very light and simple. Of course, for such a short book it does not go into detail about anything, but is a good overview for someone who knows DL theory, or maybe even for a beginner it can serve as a guidence for what to study.
+
+For the upcoming days, I have a few things to finish:
+
+1. [EvidentlyAI's last 3 modules](https://learn.evidentlyai.com/ml-observability-course/module-5-ml-pipelines-validation-and-testing)
+2. [Intro to LangGraph](https://academy.langchain.com/courses/take/intro-to-langgraph/lessons/58239426-lesson-1-state-schema)
+
+And today I looked into more books, and found [Graph Algorithms for Data Science](https://learning.oreilly.com/library/view/graph-algorithms-for/9781617299469/) and [Machine Learning System Design With end-to-end examples](https://www.manning.com/books/machine-learning-system-design)
+
+---
+
+That is all for today!
+
+See you tomorrow :)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From b0cddd14fa85fa861799a683d8532c5bf4aef7d4 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 14 Sep 2024 00:17:35 +0900
Subject: [PATCH 421/638] Create 2024-09-13-day-256.md

---
 _posts/2024-09-13-day-256.md | 88 ++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 _posts/2024-09-13-day-256.md

diff --git a/_posts/2024-09-13-day-256.md b/_posts/2024-09-13-day-256.md
new file mode 100644
index 000000000..3496ee909
--- /dev/null
+++ b/_posts/2024-09-13-day-256.md
@@ -0,0 +1,88 @@
+---
+layout: post
+title: (Day 256) Using, finetuning and explaining computer vision to my teammatest
+categories: [deep-learning,applying-knowledge]
+---
+
+# Hello :) Today is Day 256!
+A quick summary of today:
+* using YOLO for object and violence detection
+
+## Firstly, we met with a rep from Rastech
+
+Rastech is the company for which we are developing a project in my Hannam Design Factory course. When we met today I asked how they do computer vision and some data questions. The main thing I got out is that they use open-source models and YOLO to be specific. 
+
+## Secondly, using YOLO for violence detection
+
+There is a nice library - _ultralytics_ that makes using YOLO and fine-tuning it very easy.
+
+I found a violence image dataset online and fine tuned a basic model. I did it just for 2 epochs so that I can test it with my teammates. CV models are definitely smaller than LLMs but it still took around 30 minutes. 
+
+To finetune YOLOv8, we need a data.yaml
+
+```yaml
+
+train: train/images
+val: valid/images
+test: test/images
+
+nc: 2
+names: ['NonViolence', 'Violence']
+
+roboflow:
+  workspace: shah-xxxqs
+  project: violence-3h8pw
+  version: 1
+  license: CC BY 4.0
+  url: https://universe.roboflow.com/shah-xxxqs/violence-3h8pw/dataset/1
+```
+
+It specifies the classes, and paths to files
+
+and then we can just execute:
+
+```bash
+yolo task=detect mode=train model=yolov8s.pt data=data.yaml epochs=2 imgsz=416 plots=True device=mps mlflow=False
+```
+
+And the training starts
+
+<img width="1007" alt="image" src="https://github.com/user-attachments/assets/ffa105c7-05b4-4102-bbdc-8d28608bf120">
+
+Then, to use the model ~
+
+```python
+import cv2
+from ultralytics import YOLO
+
+model = YOLO('best.pt')
+print(model.names)
+webcamera = cv2.VideoCapture(0)
+
+while True:
+    success, frame = webcamera.read()
+    
+    results = model.track(frame, classes=[i for i in range(80)], conf=0.8, imgsz=480)
+    cv2.putText(frame, f"Total: {len(results[0].boxes)}", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv2.LINE_AA)
+    cv2.imshow("Live Camera", results[0].plot())
+
+    if cv2.waitKey(1) == ord('q'):
+        break
+
+webcamera.release()
+cv2.destroyAllWindows()
+```
+
+and the results are not bad and promising. 
+
+I also found a model from huggingface - [IDEA-Research/grounding-dino-base](https://huggingface.co/IDEA-Research/grounding-dino-base) but when I used it the camera was very laggy. I suspect because I am on a mac and even though I specified 'mps' as the device, it still uses the CPU (but who knows ... I need to test this on my nvidia gpu laptop)
+
+## Thirdly, an intro to how computer vision models work
+
+When I was writing my code today, my teammates kept looking, asking questions and just being curious about what's happening. Even though I explained a little, we had to go back to doing other work. So after finishing for the day, I decided to create a short document explaning how we go from raw images to real-time detection using cameras. [The doc is here](https://docs.google.com/document/d/1tA6rrlRJIqFYc-nHN-2EJxkepH2KsrmRdfDBzHieeV0/edit?usp=sharing)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 1e17be041895c4f750b41fb4edaa7c43e71dc566 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 14 Sep 2024 23:01:12 +0900
Subject: [PATCH 422/638] Create 2024-09-14-day-257.md

---
 _posts/2024-09-14-day-257.md | 841 +++++++++++++++++++++++++++++++++++
 1 file changed, 841 insertions(+)
 create mode 100644 _posts/2024-09-14-day-257.md

diff --git a/_posts/2024-09-14-day-257.md b/_posts/2024-09-14-day-257.md
new file mode 100644
index 000000000..a95bdb07b
--- /dev/null
+++ b/_posts/2024-09-14-day-257.md
@@ -0,0 +1,841 @@
+---
+layout: post
+title: (Day 257) LangChain's intro to LangGraph part 2
+categories: [nlp,theory]
+---
+
+# Hello :) Today is Day 257!
+A quick summary of today:
+* continued with [LangChain's Intro to LangGraph course](https://academy.langchain.com/courses/take/intro-to-langgraph/lessons/58239426-lesson-1-state-schema)
+
+## Module 2: State Schema
+
+### Schema
+
+When we define a LangGraph `StateGraph`, we use a [state schema](https://langchain-ai.github.io/langgraph/concepts/low_level/#state).
+
+The state schema represents the structure and types of data that our graph will use.
+
+All nodes are expected to communicate with that schema.
+
+LangGraph offers flexibility in how you define your state schema, accommodating various Python [types](https://docs.python.org/3/library/stdtypes.html#type-objects) and validation approaches!
+
+### TypedDict
+
+As mentioned in Module 1, we can use the `TypedDict` class from python's `typing` module.
+
+It allows you to specify keys and their corresponding value types.
+ 
+But, note that these are type hints. 
+
+They can used by static type checkers (like [mypy](https://github.com/python/mypy)) or IDEs to catch potential type-related errors before the code is run. 
+
+**But they are not enforced at runtime!**
+
+### Dataclass
+
+Python's [dataclasses](https://docs.python.org/3/library/dataclasses.html) provide [another way to define structured data](https://www.datacamp.com/tutorial/python-data-classes).
+
+Dataclasses offer a concise syntax for creating classes that are primarily used to store data.
+
+To access the keys of a `dataclass`, we just need to modify the subscripting used in `node_1`: 
+
+* We use `state.name` for the `dataclass` state rather than `state["name"]` for the `TypedDict` above
+
+You'll notice something a bit odd: in each node, we still return a dictionary to perform the state updates.
+ 
+This is possible because LangGraph stores each key of your state object separately.
+
+The object returned by the node only needs to have keys (attributes) that match those in the state!
+
+In this case, the `dataclass` has key `name` so we can update it by passing a dict from our node, just as we did when state was a `TypedDict`.
+
+The best option is:
+
+### Pydantic
+
+As mentioned, `TypedDict` and `dataclasses` provide type hints but they don't enforce types at runtime. 
+ 
+This means you could potentially assign invalid values without raising an error!
+
+For example, we can set `mood` to `mad` even though our type hint specifies `mood: list[Literal["happy","sad"]]`.
+
+[Pydantic](https://docs.pydantic.dev/latest/api/base_model/) is a data validation and settings management library using Python type annotations. 
+
+It's particularly well-suited [for defining state schemas in LangGraph](https://langchain-ai.github.io/langgraph/how-tos/state-model/) due to its validation capabilities.
+
+Pydantic can perform validation to check whether data conforms to the specified types and constraints at runtime.
+
+```python
+from pydantic import BaseModel, field_validator, ValidationError
+
+class PydanticState(BaseModel):
+    name: str
+    mood: Literal["happy", "sad"]
+
+    @field_validator('mood')
+    @classmethod
+    def validate_mood(cls, value):
+        # Ensure the mood is either "happy" or "sad"
+        if value not in ["happy", "sad"]:
+            raise ValueError("Each mood must be either 'happy' or 'sad'")
+        return value
+
+try:
+    state = PydanticState(name="John Doe", mood="mad")
+except ValidationError as e:
+    print("Validation Error:", e)
+```
+
+This returns:
+
+```
+Validation Error: 1 validation error for PydanticState
+mood
+  Input should be 'happy' or 'sad' [type=literal_error, input_value='mad', input_type=str]
+    For further information visit https://errors.pydantic.dev/2.8/v/literal_error
+```
+
+We can use `PydanticState` in our graph seamlessly. 
+
+```python
+# Build graph
+builder = StateGraph(PydanticState)
+builder.add_node("node_1", node_1)
+builder.add_node("node_2", node_2)
+builder.add_node("node_3", node_3)
+
+# Logic
+builder.add_edge(START, "node_1")
+builder.add_conditional_edges("node_1", decide_mood)
+builder.add_edge("node_2", END)
+builder.add_edge("node_3", END)
+
+# Add
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="214" alt="image" src="https://github.com/user-attachments/assets/0a98dada-cfb7-48ca-b602-22897d4201c8">
+
+
+## Module 2: State Reducers
+
+<img width="214" alt="image" src="https://github.com/user-attachments/assets/220d8268-f110-4bd1-bd87-d78c5e59e461">
+
+We might have a branching case where 2 nodes execute at the same time and they want to update the same state. What happens then by default? an error
+
+```python
+class State(TypedDict):
+    foo: int
+
+def node_1(state):
+    print("---Node 1---")
+    return {"foo": state['foo'] + 1}
+
+def node_2(state):
+    print("---Node 2---")
+    return {"foo": state['foo'] + 1}
+
+def node_3(state):
+    print("---Node 3---")
+    return {"foo": state['foo'] + 1}
+
+# Build graph
+builder = StateGraph(State)
+builder.add_node("node_1", node_1)
+builder.add_node("node_2", node_2)
+builder.add_node("node_3", node_3)
+
+# Logic
+builder.add_edge(START, "node_1")
+builder.add_edge("node_1", "node_2")
+builder.add_edge("node_1", "node_3")
+builder.add_edge("node_2", END)
+builder.add_edge("node_3", END)
+
+# Add
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+```python
+from langgraph.errors import InvalidUpdateError
+try:
+    graph.invoke({"foo" : 1})
+except InvalidUpdateError as e:
+    print(f"InvalidUpdateError occurred: {e}")
+```
+
+```
+---Node 1---
+---Node 2---
+---Node 3---
+InvalidUpdateError occurred: At key 'foo': Can receive only one value per step. Use an Annotated key to handle multiple values.
+```
+
+We see a problem! 
+
+Node 1 branches to nodes 2 and 3.
+
+Nodes 2 and 3 run in parallel, which means they run in the same step of the graph.
+
+They both attempt to overwrite the state *within the same step*. 
+
+This is ambiguous for the graph! Which state should it keep? 
+
+This is where Reducers come in
+
+### Reducers
+
+[Reducers](https://langchain-ai.github.io/langgraph/concepts/low_level/#reducers) give us a general way to address this problem.
+
+They specify how to perform updates.
+
+We can use the `Annotated` type to specify a reducer function. 
+
+For example, in this case let's append the value returned from each node rather than overwriting them.
+
+We just need a reducer that can perform this: `operator.add` is a function from Python's built-in operator module.
+
+When `operator.add` is applied to lists, it performs list concatenation.
+
+```python
+from operator import add
+from typing import Annotated
+
+class State(TypedDict):
+    foo: Annotated[list[int], add]
+
+def node_1(state):
+    print("---Node 1---")
+    return {"foo": [state['foo'][0] + 1]}
+
+# Build graph
+builder = StateGraph(State)
+builder.add_node("node_1", node_1)
+
+# Logic
+builder.add_edge(START, "node_1")
+builder.add_edge("node_1", END)
+
+# Add
+graph = builder.compile()
+```
+
+Now if we manipulate the state:
+
+```python
+graph.invoke({"foo" : [1]})
+```
+
+```
+---Node 1---
+{'foo': [1, 2]}
+```
+
+Now, our state key `foo` is a list.
+
+This `operator.add` reducer function will append updates from each node to this list. 
+
+### Messages
+
+In module 1, the course showed how to use a built-in reducer, `add_messages`, to handle messages in state.
+
+also showed that [`MessagesState` is a useful shortcut if you want to work with messages](https://langchain-ai.github.io/langgraph/concepts/low_level/#messagesstate). 
+
+* `MessagesState` has a built-in `messages` key 
+* It also has a built-in `add_messages` reducer for this key
+
+These two are equivalent. 
+
+We'll use the `MessagesState` class via `from langgraph.graph import MessagesState` for brevity.
+
+```python
+from typing import Annotated
+from langgraph.graph import MessagesState
+from langchain_core.messages import AnyMessage
+from langgraph.graph.message import add_messages
+
+# Define a custom TypedDict that includes a list of messages with add_messages reducer
+class CustomMessagesState(TypedDict):
+    messages: Annotated[list[AnyMessage], add_messages]
+    added_key_1: str
+    added_key_2: str
+    # etc
+
+# Use MessagesState, which includes the messages key with add_messages reducer
+class ExtendedMessagesState(MessagesState):
+    # Add any keys needed beyond messages, which is pre-built 
+    added_key_1: str
+    added_key_2: str
+    # etc
+```
+
+Add msgs:
+
+```python
+from langgraph.graph.message import add_messages
+from langchain_core.messages import AIMessage, HumanMessage
+
+# Initial state
+initial_messages = [AIMessage(content="Hello! How can I assist you?", name="Model"),
+                    HumanMessage(content="I'm looking for information on marine biology.", name="Lance")
+                   ]
+
+# New message to add
+new_message = AIMessage(content="Sure, I can help with that. What specifically are you interested in?", name="Model")
+
+# Test
+add_messages(initial_messages , new_message)
+```
+
+```
+[AIMessage(content='Hello! How can I assist you?', name='Model', id='79df9553-6f7e-4f79-9ec6-949d6857d89f'),
+ HumanMessage(content="I'm looking for information on marine biology.", name='Lance', id='66ca4a3a-e362-4d96-a720-e4b2d2c206a0'),
+ AIMessage(content='Sure, I can help with that. What specifically are you interested in?', name='Model', id='aa7d50d1-c4ee-4888-887e-aee79a4dae7f')]
+```
+
+Re-writing msgs
+
+If we pass a message with the same ID as an existing one in our `messages` list, it will get overwritten!
+
+```python
+# Initial state
+initial_messages = [AIMessage(content="Hello! How can I assist you?", name="Model", id="1"),
+                    HumanMessage(content="I'm looking for information on marine biology.", name="Lance", id="2")
+                   ]
+
+# New message to add
+new_message = HumanMessage(content="I'm looking for information on whales, specifically", name="Lance", id="2")
+
+# Test
+add_messages(initial_messages , new_message)
+```
+
+```
+[AIMessage(content='Hello! How can I assist you?', name='Model', id='1'),
+ HumanMessage(content="I'm looking for information on whales, specifically", name='Lance', id='2')]
+```
+
+Removing msgs
+
+```python
+from langchain_core.messages import RemoveMessage
+
+# Message list
+messages = [AIMessage("Hi.", name="Bot", id="1")]
+messages.append(HumanMessage("Hi.", name="Lance", id="2"))
+messages.append(AIMessage("So you said you were researching ocean mammals?", name="Bot", id="3"))
+messages.append(HumanMessage("Yes, I know about whales. But what others should I learn about?", name="Lance", id="4"))
+
+# Isolate messages to delete
+delete_messages = [RemoveMessage(id=m.id) for m in messages[:-2]]
+print(delete_messages)
+```
+
+```
+[RemoveMessage(content='', id='1'), RemoveMessage(content='', id='2')]
+```
+
+## Module 2: Multiple Schemas
+
+Typically, all graph nodes communicate with a single schema. 
+
+Also, this single schema contains the graph's input and output keys / channels.
+
+But, there are cases where we may want a bit more control over this:
+
+* Internal nodes may pass information that is *not required* in the graph's input / output.
+
+* We may also want to use different input / output schemas for the graph. The output might, for example, only contain a single relevant output key.
+
+### Private State
+
+First, let's cover the case of passing [private state](https://langchain-ai.github.io/langgraph/how-tos/pass_private_state/) between nodes.
+
+This is useful for anything needed as part of the intermediate working logic of the graph, but not relevant for the overall graph input or output.
+
+We'll define an `OverallState` and a `PrivateState`.
+
+`node_2` uses `PrivateState` as input, but writes out to `OverallState`.
+
+```python
+from typing_extensions import TypedDict
+from IPython.display import Image, display
+from langgraph.graph import StateGraph, START, END
+
+class OverallState(TypedDict):
+    foo: int
+
+class PrivateState(TypedDict):
+    baz: int
+
+def node_1(state: OverallState) -> PrivateState:
+    print("---Node 1---")
+    return {"baz": state['foo'] + 1}
+
+def node_2(state: PrivateState) -> OverallState:
+    print("---Node 2---")
+    return {"foo": state['baz'] + 1}
+
+# Build graph
+builder = StateGraph(OverallState)
+builder.add_node("node_1", node_1)
+builder.add_node("node_2", node_2)
+
+# Logic
+builder.add_edge(START, "node_1")
+builder.add_edge("node_1", "node_2")
+builder.add_edge("node_2", END)
+
+# Add
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="150" alt="image" src="https://github.com/user-attachments/assets/11ccecd9-ccc7-4dd1-a920-6d0c19bc4798">
+
+```python
+graph.invoke({"foo" : 1})
+```
+
+Output:
+
+```
+---Node 1---
+---Node 2---
+{'foo': 3}
+```
+
+`baz` is only included in `PrivateState`.
+
+`node_2` uses `PrivateState` as input, but writes out to `OverallState`.
+
+So, we can see that `baz` is excluded from the graph output because it is not in `OverallState`.
+
+### Input / Output Schema
+
+By default, `StateGraph` takes in a single schema and all nodes are expected to communicate with that schema. 
+
+However, it is also possible to [define explicit input and output schemas for a graph](https://langchain-ai.github.io/langgraph/how-tos/input_output_schema/?h=input+outp).
+
+Often, in these cases, we define an "internal" schema that contains *all* keys relevant to graph operations.
+
+But, we use specific `input` and `output` schemas to constrain the input and output.
+
+First, let's just run the graph with a single schema.
+
+```python
+class OverallState(TypedDict):
+    question: str
+    answer: str
+    notes: str
+
+def thinking_node(state: OverallState):
+    return {"answer": "bye", "notes": "... his is name is Lance"}
+
+def answer_node(state: OverallState):
+    return {"answer": "bye Lance"}
+
+graph = StateGraph(OverallState)
+graph.add_node("answer_node", answer_node)
+graph.add_node("thinking_node", thinking_node)
+graph.add_edge(START, "thinking_node")
+graph.add_edge("thinking_node", "answer_node")
+graph.add_edge("answer_node", END)
+
+graph = graph.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="190" alt="image" src="https://github.com/user-attachments/assets/363e9682-1487-4aa2-b205-51ac819c8162">
+
+```python
+graph.invoke({"question":"hi"})
+```
+
+Notice that the output of invoke contains all keys in OverallState:
+
+```
+{'question': 'hi', 'answer': 'bye Lance', 'notes': '... his is name is Lance'}
+```
+
+Now, let's use a specific input and output schema.
+
+```python
+class InputState(TypedDict):
+    question: str
+
+class OutputState(TypedDict):
+    answer: str
+
+class OverallState(TypedDict):
+    question: str
+    answer: str
+    notes: str
+
+def thinking_node(state: OverallState):
+    return {"answer": "bye", "notes": "... his is name is Lance"}
+
+def answer_node(state: OutputState):
+    return {"answer": "bye Lance"}
+
+graph = StateGraph(input=InputState, output=OutputState)
+graph.add_node("answer_node", answer_node)
+graph.add_node("thinking_node", thinking_node)
+graph.add_edge(START, "thinking_node")
+graph.add_edge("thinking_node", "answer_node")
+graph.add_edge("answer_node", END)
+
+graph = graph.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+
+graph.invoke({"question":"hi"})
+```
+
+```
+{'answer': 'bye Lance'}
+```
+
+We can see the output schema constrains the output to only the answer key.
+
+## Module 2: Filtering and trimming messages for a ChatBot
+
+### Reducer
+
+A practical challenge when working with messages is managing long-running conversations. 
+
+Long-running conversations result in high token usage and latency if we are not careful, because we pass a growing list of messages to the model.
+
+We have a few ways to address this.
+
+First, recall the trick we saw using `RemoveMessage` and the `add_messages` reducer.
+
+```python
+from langchain_core.messages import RemoveMessage
+
+# Nodes
+def filter_messages(state: MessagesState):
+    # Delete all but the 2 most recent messages
+    delete_messages = [RemoveMessage(id=m.id) for m in state["messages"][:-2]] # Include only the last 2 msgs
+    return {"messages": delete_messages}
+
+def chat_model_node(state: MessagesState):    
+    return {"messages": [llm.invoke(state["messages"])]}
+
+# Build graph
+builder = StateGraph(MessagesState)
+builder.add_node("filter", filter_messages)
+builder.add_node("chat_model", chat_model_node)
+builder.add_edge(START, "filter")
+builder.add_edge("filter", "chat_model")
+builder.add_edge("chat_model", END)
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+### Filtering messages
+
+If you don't need or want to modify the graph state, you can just filter the messages you pass to the chat model.
+
+For example, just pass in a filtered list: `llm.invoke(messages[-1:])` to the model.
+
+```python
+# Node
+def chat_model_node(state: MessagesState):
+    return {"messages": [llm.invoke(state["messages"][-1:])]}
+
+# Build graph
+builder = StateGraph(MessagesState)
+builder.add_node("chat_model", chat_model_node)
+builder.add_edge(START, "chat_model")
+builder.add_edge("chat_model", END)
+graph = builder.compile()
+
+# View
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+### Trim messages
+
+Another approach is to [trim messages](https://python.langchain.com/v0.2/docs/how_to/trim_messages/#getting-the-last-max_tokens-tokens), based upon a set number of tokens. 
+
+This restricts the message history to a specified number of tokens.
+
+While filtering only returns a post-hoc subset of the messages between agents, trimming restricts the number of tokens that a chat model can use to respond.
+
+See the `trim_messages` below.
+
+```python
+from langchain_core.messages import trim_messages
+
+# Node
+def chat_model_node(state: MessagesState):
+    messages = trim_messages(
+            state["messages"],
+            max_tokens=100,
+            strategy="last",
+            token_counter=ChatOpenAI(model="gpt-4o"),
+            allow_partial=False,
+        )
+    return {"messages": [llm.invoke(messages)]}
+
+# Build graph
+builder = StateGraph(MessagesState)
+builder.add_node("chat_model", chat_model_node)
+builder.add_edge(START, "chat_model")
+builder.add_edge("chat_model", END)
+graph = builder.compile()
+```
+
+## Module 2: ChatBot with msg summarisation and memory
+
+With trimming we can lose important information from a message, so its not idea. Summarisation fixes this and still lowers the token cost
+
+Init the model
+
+```python
+from langchain_openai import ChatOpenAI
+model = ChatOpenAI(model="gpt-4o",temperature=0)
+```
+
+We'll use `MessagesState`, as before.
+
+In addition to the built-in `messages` key, we'll now include a custom key (`summary`).
+
+```python
+from langgraph.graph import MessagesState
+class State(MessagesState):
+    summary: str
+```
+
+We'll define a node to call our LLM that incorporates a summary, if it exists, into the prompt.
+
+```python
+from langchain_core.messages import SystemMessage, HumanMessage, RemoveMessage
+
+# Define the logic to call the model
+def call_model(state: State):
+    
+    # Get summary if it exists
+    summary = state.get("summary", "")
+
+    # If there is summary, then we add it
+    if summary:
+        
+        # Add summary to system message
+        system_message = f"Summary of conversation earlier: {summary}"
+
+        # Append summary to any newer messages
+        messages = [SystemMessage(content=system_message)] + state["messages"]
+    
+    else:
+        messages = state["messages"]
+    
+    response = model.invoke(messages)
+    return {"messages": response}
+```
+
+We'll define a node to produce a summary.
+
+Note, here we'll use `RemoveMessage` to filter our state after we've produced the summary.
+
+```python
+def summarize_conversation(state: State):
+    
+    # First, we get any existing summary
+    summary = state.get("summary", "")
+
+    # Create our summarization prompt 
+    if summary:
+        
+        # A summary already exists
+        summary_message = (
+            f"This is summary of the conversation to date: {summary}\n\n"
+            "Extend the summary by taking into account the new messages above:"
+        )
+        
+    else:
+        summary_message = "Create a summary of the conversation above:"
+
+    # Add prompt to our history
+    messages = state["messages"] + [HumanMessage(content=summary_message)]
+    response = model.invoke(messages)
+    
+    # Delete all but the 2 most recent messages
+    delete_messages = [RemoveMessage(id=m.id) for m in state["messages"][:-2]]
+    return {"summary": response.content, "messages": delete_messages}
+```
+
+We'll add a conditional edge to determine whether to produce a summary based on the conversation length.
+
+```python
+from langgraph.graph import END
+# Determine whether to end or summarize the conversation
+def should_continue(state: State):
+    
+    """Return the next node to execute."""
+    
+    messages = state["messages"]
+    
+    # If there are more than six messages, then we summarize the conversation
+    if len(messages) > 6:
+        return "summarize_conversation"
+    
+    # Otherwise we can just end
+    return END
+```
+
+### Adding memory
+
+Recall that [state is transient](https://github.com/langchain-ai/langgraph/discussions/352#discussioncomment-9291220) to a single graph execution.
+
+This limits our ability to have multi-turn conversations with interruptions. 
+
+As introduced at the end of Module 1, we can use [persistence](https://langchain-ai.github.io/langgraph/how-tos/persistence/) to address this! 
+ 
+LangGraph can use a checkpointer to automatically save the graph state after each step.
+
+This built-in persistence layer gives us memory, allowing LangGraph to pick up from the last state update. 
+
+As we previously showed, one of the easiest to work with is `MemorySaver`, an in-memory key-value store for Graph state.
+
+All we need to do is compile the graph with a checkpointer, and our graph has memory!
+
+```python
+from IPython.display import Image, display
+from langgraph.checkpoint.memory import MemorySaver
+from langgraph.graph import StateGraph, START
+
+# Define a new graph
+workflow = StateGraph(State)
+workflow.add_node("conversation", call_model)
+workflow.add_node(summarize_conversation)
+
+# Set the entrypoint as conversation
+workflow.add_edge(START, "conversation")
+workflow.add_conditional_edges("conversation", should_continue)
+workflow.add_edge("summarize_conversation", END)
+
+# Compile
+memory = MemorySaver()
+graph = workflow.compile(checkpointer=memory)
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+<img width="260" alt="image" src="https://github.com/user-attachments/assets/0ac54212-d6ba-4e37-b784-38ef2d8a1966">
+
+### Threads
+
+The checkpointer saves the state at each step as a checkpoint.
+
+These saved checkpoints can be grouped into a `thread` of conversation.
+
+Think about Slack as an analog: different channels carry different conversations.
+
+Threads are like Slack channels, capturing grouped collections of state (e.g., conversation).
+
+Below, we use `configurable` to set a thread ID.
+
+<img width="976" alt="image" src="https://github.com/user-attachments/assets/27a1b549-1abd-4e4d-8a11-e9ef79bbccb2">
+
+```python
+# Create a thread
+config = {"configurable": {"thread_id": "1"}}
+
+# Start conversation
+input_message = HumanMessage(content="hi! I'm Lance")
+output = graph.invoke({"messages": [input_message]}, config) 
+for m in output['messages'][-1:]:
+    m.pretty_print()
+
+input_message = HumanMessage(content="what's my name?")
+output = graph.invoke({"messages": [input_message]}, config) 
+for m in output['messages'][-1:]:
+    m.pretty_print()
+
+input_message = HumanMessage(content="i like the 49ers!")
+output = graph.invoke({"messages": [input_message]}, config) 
+for m in output['messages'][-1:]:
+    m.pretty_print()
+```
+
+```
+================================== Ai Message ==================================
+
+Hi Lance! How can I assist you today?
+================================== Ai Message ==================================
+
+You mentioned that your name is Lance. How can I help you today, Lance?
+================================== Ai Message ==================================
+
+That's awesome, Lance! The San Francisco 49ers have a rich history and a passionate fan base. Do you have a favorite player or a memorable game that stands out to you?
+```
+
+Now, we don't yet have a summary of the state because we still have < = 6 messages.
+
+This was set in `should_continue`. 
+
+```
+    # If there are more than six messages, then we summarize the conversation
+    if len(messages) > 6:
+        return "summarize_conversation"
+```
+
+We can pick up the conversation because we have the thread.
+
+
+Once we add a few more msgs (to exceed the threshold we set), we can see a summary is passed into our model
+
+```
+"Lance introduced himself and mentioned that he is a fan of the San Francisco 49ers. He specifically likes Nick Bosa and inquired if Bosa is the highest-paid defensive player. I confirmed that Nick Bosa signed a record-breaking contract extension in September 2023, making him the highest-paid defensive player at that time, and acknowledged Bosa's talent and Lance's enthusiasm for the player."
+```
+
+## Module 2: ChatBot with summarisation and external memory
+
+It is like above, but  for memory we use sqlite3
+
+```python
+import sqlite3
+# In memory
+conn = sqlite3.connect(":memory:", check_same_thread = False)
+
+# Saved to local db
+db_path = "state_db/example.db"
+conn = sqlite3.connect(db_path, check_same_thread=False)
+
+# Here is our checkpointer 
+from langgraph.checkpoint.sqlite import SqliteSaver
+memory = SqliteSaver(conn)
+```
+
+Actually after watching and taking notes for module 2, I decided to just watch modules 3 and 4 because I started to feel a bit lost in terms of practicality. I understand there's a lot of LangGraph basics that we need to cover, but it felt a bit unclear as to how we will take this from notebook to practice. I suspect in the next few months, as these LLM app libraries are moving very fast, there will be a lot of abstractions coming out to make the whole thing a bit more clear. Nevertheless understanding these basics is important as once abstractions come in, it can be a bit harder to go back.
+
+## [Exercises in ML](https://arxiv.org/abs/2206.13446)
+
+On another note, I have been keeping this paper in my to-dos. It involves only math related exercises related to ML. So today I finally went to print it and created a small booklet. 
+
+![image](https://github.com/user-attachments/assets/f8e5bc02-54e4-4334-9c94-a265e784277b)
+
+I haven't started it yet but it is something for the next few days. I checked the exercises and it is just math related to linear algebra, optimisation, directed/undirected graphical models, expressive power of graphical models, factor graphs and message passing, inference for hidden markov chains, model-based learning, sampling and monte-carlo integration, variational inference. 
+
+Some of these topics are new to me, but I am excited to at least have a look. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 06f52429d226811630d60c1c51883929798b9605 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 15 Sep 2024 23:17:41 +0900
Subject: [PATCH 423/638] Create 2024-09-15-day-258.md

---
 _posts/2024-09-15-day-258.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 _posts/2024-09-15-day-258.md

diff --git a/_posts/2024-09-15-day-258.md b/_posts/2024-09-15-day-258.md
new file mode 100644
index 000000000..dc6f39df5
--- /dev/null
+++ b/_posts/2024-09-15-day-258.md
@@ -0,0 +1,32 @@
+---
+layout: post
+title: (Day 258) Math exercises in ML
+categories: [theory,math]
+---
+
+# Hello :) Today is Day 258!
+A quick summary of today:
+* meeting to continue workijg on LingoMate (the hackathon project)
+* starting the Execises in ML booklet
+
+## Firstly, about [LingoMate](https://github.com/LingoMate-Seoul/LingoMate)
+
+Jieun, my partner during the 2-day Seoul Tech Impact Hackathon last weekend, organised myself and a 3rd person (Sangwook) who was also a participant in the hackathon who expressed interest in working with us to continue developing LingoMate. We met through google meets, introduced ourselves and our hopes for the project. I actually wanted to go to Seoul to meet in-person, as both of them live and work there, but because its the holidays right now, everything is booked so we settled on meeting online today. The tldr of the meeting is that we are excited to work together and conitnue the project, and also that adding speech interaction is good and an important improvement. [Here](https://github.com/LingoMate-Seoul) is an org on GitHub for the project.
+
+## Secondly, math exercises in ML
+
+Yesterday I mentioned I created a small booklet of a ~200 page called Pen & Paper: Exercises in ML that's all about different math used throughout different parts of ML. Today I started with chapter 1, and below are my notes. It is about proving different concepts. Not knowing the proof seems to be completely fine, but knowing, for example, the proof behind why the determinant of a diagonal matrix is the product of its eigenvalues just gives a bit more depth into understanding the math behind everything.
+
+![Linear algebra P1](https://github.com/user-attachments/assets/2565bf5d-cf67-47b3-b1a0-9f94e98ee3f6)
+![Linear algebra P2](https://github.com/user-attachments/assets/cb91c79b-3149-43dc-af3b-d4a217ab4c3b)
+![Linear algebra P3](https://github.com/user-attachments/assets/f20aa81d-1b05-47dd-b391-0c02a7b71cd3)
+![Linear algebra P4](https://github.com/user-attachments/assets/3929b6f0-7513-41c7-abfd-eb7747bf3cf0)
+![Linear algebra P5](https://github.com/user-attachments/assets/5c1d2eb5-21d0-4f0e-aae6-51969af0f60b)
+![Linear algebra P6](https://github.com/user-attachments/assets/972b1459-9507-4f2f-b119-e15884e8bf00)
+![Linear algebra P7](https://github.com/user-attachments/assets/795d38f3-9238-4da3-93e3-7b50b64efebc)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f1540e5c6b981540338aab1e0acfd6e60f0bdfb2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 16 Sep 2024 22:49:56 +0900
Subject: [PATCH 424/638] Create 2024-09-16-day-259.md

---
 _posts/2024-09-16-day-259.md | 995 +++++++++++++++++++++++++++++++++++
 1 file changed, 995 insertions(+)
 create mode 100644 _posts/2024-09-16-day-259.md

diff --git a/_posts/2024-09-16-day-259.md b/_posts/2024-09-16-day-259.md
new file mode 100644
index 000000000..9d7a6b83a
--- /dev/null
+++ b/_posts/2024-09-16-day-259.md
@@ -0,0 +1,995 @@
+---
+layout: post
+title: (Day 259) ML monitoring pipelines + Going deeper into neo4j
+categories: [mlops,applying-knowledge,theory]
+---
+
+# Hello :) Today is Day 259!
+A quick summary of today:
+* [Module 5: ML pipelines validation and testing](https://learn.evidentlyai.com/ml-observability-course/module-5-ml-pipelines-validation-and-testing)
+* starting Neo4j's free courses
+
+## 5.1. Introduction to data and ML pipeline testing
+
+### When to perform testing
+
+ML lifecycle involves various steps that require testing to ensure that our ML models function properly. Critical areas to test include the following steps of the ML lifecycle:
+
+* During the feature engineering stage: testing the input data quality as it affects the whole pipeline.
+
+* During model training (or retraining): model quality checks.
+
+* During model serving: validating incoming data and model outputs.
+
+* During performance monitoring: continuously testing the model quality to detect and resolve potential issues.
+
+### How to perform testing
+
+There are different types of checks you can use to test data and ML pipelines:
+
+**Individual tests** A test is a metric with a condition. You can perform a certain evaluation or measurement on top of a data batch and compare it against a threshold or expectation. You can formulate almost anything as a test: assertions on feature values, expectations about model quality on a specific segment, etc. Whatever you can measure, you can design as a test.
+
+Tests can be column-level (when metrics are calculated for a specific feature or column) or dataset-level (in this case, you calculate metrics for the whole dataset).
+
+**Test suites** Individual tests can be grouped into test suites. For each test in a test suite, you can define test criticality and set alerting conditions: for example, based on the number of failed critical tests.
+
+When you create a test, you must define the test conditions. There are two main strategies you can use for establishing test conditions:
+
+**Reference-based conditions** You can use a reference dataset to derive conditions automatically rather than set conditions manually for each individual test. This is a great option for certain types of checks, such as testing column types (which are easy to derive from a reference example) and for ad hoc testing such as when you import a new batch of data and can immediately visually explore the test results. However, be careful when designing alerting, as auto-generated test conditions are not perfect and may be prone to false alerts or missed issues.
+
+**Manually defined conditions** With this approach, you specify conditions for each test manually. This method does not require additional data and can be great for encoding specific conditions based on domain expertise.
+
+### Test automation
+
+![image](https://github.com/user-attachments/assets/5eaa0b50-5cc9-4031-931f-426c2390fa72)
+
+### Recording test results
+
+![image](https://github.com/user-attachments/assets/b918e75f-ba79-48d1-a3bc-beca0eabd431)
+
+### Example use case
+
+![image](https://github.com/user-attachments/assets/55a5709f-a393-4c53-8728-71237afe235a)
+
+![image](https://github.com/user-attachments/assets/93de6ba2-c690-49ae-8295-ad469f24e6ed)
+
+## 5.2. Train and evaluate an ML model (code practice)
+
+Using a bank marketing dataset from sklearn:
+
+#### Bank client data:
+* **Age** (numeric)
+* **Job :** type of job (categorical: 'admin.', 'blue-collar', 'entrepreneur', 'housemaid', 'management', 'retired', 'self-employed', 'services', 'student', 'technician', 'unemployed', 'unknown')
+* **Marital :** marital status (categorical: 'divorced', 'married', 'single', 'unknown' ; note: 'divorced' means divorced or widowed)
+* **Education** (categorical: 'basic.4y', 'basic.6y', 'basic.9y', 'high.school', 'illiterate', 'professional.course', 'university.degree', 'unknown')
+* **Default:** has credit in default? (categorical: 'no', 'yes', 'unknown')
+* **Balance:** average yearly balance, in euros (numeric)
+* **Housing:** has housing loan? (categorical: 'no', 'yes', 'unknown')
+* **Loan:** has personal loan? (categorical: 'no', 'yes', 'unknown')
+
+#### Related with the last contact of the current campaign:
+* **Contact:** contact communication type (categorical:
+'cellular','telephone')
+* **Day:** ast contact day of the month (numeric)
+* **Month:** last contact month of year (categorical: 'jan', 'feb', 'mar', …, 'nov', 'dec')
+* **Duration:** last contact duration, in seconds (numeric). Important
+note: this attribute highly affects the output target (e.g., if
+duration=0 then y='no'). Yet, the duration is not known before a call
+is performed. Also, after the end of the call y is obviously known.
+Thus, this input should only be included for benchmark purposes and
+should be discarded if the intention is to have a realistic
+predictive model.
+
+#### Other attributes:
+* **Campaign:** number of contacts performed during this campaign and for
+this client (numeric, includes last contact)
+* **Pdays:** number of days that passed by after the client was last
+contacted from a previous campaign (numeric; 999 means client was not
+previously contacted)
+* **Previous:** number of contacts performed before this campaign and for
+this client (numeric)
+* **Poutcome:** outcome of the previous marketing campaign (categorical:
+'failure','nonexistent','success')
+
+
+Doing some feature engineering:
+
+```python
+def feature_engineering(raw_data: pd.DataFrame) -> pd.DataFrame:
+    preprocessed_data = raw_data.copy(deep = True)
+
+    preprocessed_data.columns = ['age', 'job', 'marital', 'education', 'default', 'balance', 'housing', 'loan', 'contact', 
+                              'day', 'month', 'duration', 'campaign', 'pdays', 'previous', 'poutcome', 'class']
+
+    #client data preprocessing
+    preprocessed_data['has_default'] = preprocessed_data.default.apply(
+        lambda x : 0 if x == 'no' else 1 if x == 'yes' else -1
+    )
+
+    preprocessed_data['has_housing_loan'] = preprocessed_data.housing.apply(
+        lambda x : 0 if x == 'no' else 1 if x == 'yes' else -1
+    )
+
+    preprocessed_data['has_personal_loan'] = preprocessed_data.loan.apply(
+        lambda x : 0 if x == 'no' else 1 if x == 'yes' else -1
+    )
+
+    marital_dummies = pd.get_dummies(preprocessed_data.marital, prefix = 'marital')
+    preprocessed_data = pd.concat([preprocessed_data, marital_dummies], axis = 1)
+
+    job_dummies = pd.get_dummies(preprocessed_data.job, prefix = 'job')
+    preprocessed_data = pd.concat([preprocessed_data, job_dummies], axis = 1)
+
+    edu_dummies = pd.get_dummies(preprocessed_data.education, prefix = 'edu')
+    preprocessed_data = pd.concat([preprocessed_data, edu_dummies], axis = 1)
+
+    preprocessed_data.drop(columns = ['default', 'housing', 'loan', 'marital', 'job', 'education'], inplace=True)
+
+    # last contact data preprocessing
+    contact_dummies = pd.get_dummies(preprocessed_data.contact, prefix = 'contact_type')
+    preprocessed_data = pd.concat([preprocessed_data, contact_dummies], axis = 1)
+
+    month_dummies = pd.get_dummies(preprocessed_data.month, prefix = 'month')
+    preprocessed_data = pd.concat([preprocessed_data, month_dummies], axis = 1)   
+
+    preprocessed_data.drop(columns = ['contact', 'month'], inplace=True)
+    
+    # other attributes preprocessing
+    poutcome_dummies = pd.get_dummies(preprocessed_data.poutcome, prefix = 'prev_camp_outcome')
+    preprocessed_data = pd.concat([preprocessed_data, poutcome_dummies], axis = 1)
+    preprocessed_data.drop(columns = ['poutcome'], inplace=True)
+
+    #target preprocessing
+    preprocessed_data['target'] = preprocessed_data['class'].apply(lambda x : 0 if x == '1' else 1)
+    preprocessed_data.drop(columns = ['class'], inplace=True)
+    
+    return preprocessed_data
+```
+
+Then, splitting into train, reference, and prod simulation
+
+```python
+train_data = bank_marketing_data[:5000]
+
+reference_data = bank_marketing_data[5000:7000]
+
+prod_simulation_data = bank_marketing_data[7000:]
+```
+
+The model used is a simple RF classifier
+
+```python
+model = ensemble.RandomForestClassifier(random_state=42, n_estimators=100)
+```
+
+To create a simple report:
+
+```python
+model_quality_report = Report(metrics=[ClassificationPreset()])
+model_quality_report.run(reference_data=processed_train, current_data=processed_reference)
+model_quality_report.show(mode='inline')
+```
+
+The output report is:
+
+<img width="1408" alt="image" src="https://github.com/user-attachments/assets/9dd78617-2ad0-44ab-b464-99200dd697a4">
+<img width="1392" alt="image" src="https://github.com/user-attachments/assets/ee56b259-fb56-424b-b2f9-48da5e97c029">
+<img width="1389" alt="image" src="https://github.com/user-attachments/assets/2d7889d4-7a59-40ed-9370-5adb91221eca">
+<img width="1389" alt="image" src="https://github.com/user-attachments/assets/f27b5ee1-bffe-4b02-9a75-74f4888222eb">
+
+## 5.3. Test input data quality, stability and drift (code practice)
+
+### Raw data checks:
+
+```python
+data_stability_suite = TestSuite(tests=[DataStabilityTestPreset()])
+data_stability_suite.run(reference_data=reference_data, current_data=prod_simulation_data[:batch_size])
+data_stability_suite.show(mode='inline')
+```
+
+<img width="1386" alt="image" src="https://github.com/user-attachments/assets/858f3408-7a1f-421c-8ac3-393d5d593121">
+
+For any test we can see details:
+
+<img width="1371" alt="image" src="https://github.com/user-attachments/assets/acfc7fe5-b889-4bab-9da3-991d0e09245e">
+
+```python
+data_quality_suite = TestSuite(tests=[DataQualityTestPreset()])
+data_quality_suite.run(reference_data=reference_data, current_data=prod_simulation_data[:batch_size])
+data_quality_suite.show(mode='inline')
+```
+
+<img width="1391" alt="image" src="https://github.com/user-attachments/assets/c373f0e1-0efa-42af-9244-412654c7df0b">
+
+```python
+data_drift_suite = TestSuite(tests=[DataDriftTestPreset()])
+data_drift_suite.run(reference_data=reference_data, current_data=prod_simulation_data[:batch_size])
+data_drift_suite.show(mode='inline')
+```
+
+<img width="1378" alt="image" src="https://github.com/user-attachments/assets/96a807e0-cda7-4b57-9941-614ca71801ce">
+
+### Preprocessed data checks
+
+We preprocess the data using our feature_engineering function
+
+```python
+processed_reference_data = feature_engineering(reference_data)
+processed_prod_simulation_data = feature_engineering(prod_simulation_data)
+```
+
+Run the data drift preset report code
+
+```python
+data_drift_suite = TestSuite(tests=[DataDriftTestPreset()])
+data_drift_suite.run(reference_data=processed_reference_data, current_data=processed_prod_simulation_data[:batch_size])
+data_drift_suite.show(mode='inline')
+```
+
+<img width="1407" alt="image" src="https://github.com/user-attachments/assets/de5844d9-2d29-450b-8cc8-49e973fe068c">
+
+We can run some quick checks for warnings/alerts to check tests pass rate like below
+
+```python
+data_drift_suite.as_dict()['summary']['all_passed'] == True # False
+data_drift_suite.as_dict()['summary']['by_status']['SUCCESS'] > 40 # True
+```
+
+## 5.4. Test ML model outputs and quality (code practice)
+
+Load the trained model and add predictions for reference and the prod simulation data that was created earlier:
+
+```python
+processed_data_reference['prediction'] = model.predict(processed_data_reference.iloc[:, :-1])
+processed_data_prod_simulation['prediction'] = model.predict(processed_data_prod_simulation.iloc[:, :-1])
+```
+
+### Model output checks
+
+```python
+no_target_performance_suite = TestSuite(tests=[NoTargetPerformanceTestPreset()])
+no_target_performance_suite.run(reference_data=processed_data_reference, current_data=processed_data_prod_simulation[2*batch_size:3*batch_size])
+no_target_performance_suite.show(mode='inline')
+```
+
+<img width="1388" alt="image" src="https://github.com/user-attachments/assets/2922c20c-d70f-40fc-99c1-5e2b2be9fe91">
+
+
+### Model quality checks
+
+```python
+model_performance_suite = TestSuite(tests=[BinaryClassificationTestPreset()])
+model_performance_suite.run(reference_data=processed_data_reference, current_data=processed_data_prod_simulation[0*batch_size:1*batch_size])
+model_performance_suite.show(mode='inline')
+```
+
+<img width="1390" alt="image" src="https://github.com/user-attachments/assets/3e9c0724-8033-4d80-92ad-213ef0eef0a2">
+
+## 5.5. Design a custom test suite with Evidently (code practice)
+
+Here is an example custome suite
+
+```python
+custom_performance_suite = TestSuite(tests=[
+    #TestColumnsType(),
+    #TestShareOfDriftedColumns(ls=0.5),
+    TestShareOfMissingValues(eq=0),
+    TestPrecisionScore(gt=0.5),
+    TestRecallScore(gt=0.3),
+    TestAccuracyScore(gte=0.75),
+])
+
+custom_performance_suite.run(reference_data=processed_reference, current_data=processed_prod_simulation[:batch_size])
+custom_performance_suite.show(mode='inline')
+```
+
+<img width="1373" alt="image" src="https://github.com/user-attachments/assets/76394b63-a928-4d5b-a639-f2f6ba5f6b49">
+
+## Running and logging reports using Airflow and mlflow
+
+### Adding monitoring using Airlow
+
+```python
+import os
+import pandas as pd 
+
+from datetime import datetime, timedelta
+from sklearn import datasets
+
+from airflow import DAG
+from airflow.operators.python_operator import PythonOperator 
+
+from evidently.test_suite import TestSuite 
+from evidently.test_preset import DataDriftTestPreset
+
+default_args = {
+	"start_date" : datetime(2023, 1, 1),
+	"owner" : "emeli",
+	"retries" : 2,
+	"retry_delay" : timedelta(minutes = 5),
+}
+
+dir_path = "reports"
+file_path = "data_drift_suite.html"
+
+def load_data_execute(**context):
+	bank_marketing = datasets.fetch_openml(name='bank-marketing', as_frame='auto')
+	bank_marketing_data = bank_marketing.frame 
+	context["ti"].xcom_push(key="data_frame", value=bank_marketing_data)
+
+def drift_analysis_execute(**context):
+	data = context["ti"].xcom_pull(key="data_frame")
+
+	reference_data = data[5000:7000]
+	prod_simulation_data = data[7000:]
+	batch_size = 2000
+
+	data_drift_suite = TestSuite(tests=[DataDriftTestPreset()])
+	data_drift_suite.run(reference_data=reference_data, current_data=prod_simulation_data[:batch_size])
+
+	if not data_drift_suite.as_dict()['summary']['all_passed']:
+		return "create_report"
+
+def create_report_execute(**context):
+	data = context["ti"].xcom_pull(key="data_frame")
+
+	reference_data = data[5000:7000]
+	prod_simulation_data = data[7000:]
+	batch_size = 2000
+
+	data_drift_suite = TestSuite(tests=[DataDriftTestPreset()])
+	data_drift_suite.run(reference_data=reference_data, current_data=prod_simulation_data[:batch_size])
+
+	try:
+		os.mkdir(dir_path)
+	except OSError:
+		print("Creation of the directory {} failed".format(dir_path))
+	data_drift_suite.save_html(os.path.join(dir_path, file_path))
+
+
+with DAG(
+	dag_id = "evidently_conditional_drift",
+	schedule_interval = "@daily",
+	default_args = default_args,
+	catchup = False,
+	) as dag:
+
+	load_data = PythonOperator(
+		task_id = "load_data",
+		python_callable = load_data_execute,
+		provide_context = True
+		)
+
+	drift_analysis = PythonOperator(
+		task_id = "drift_analysis",
+		python_callable = drift_analysis_execute,
+		provide_context = True
+		)
+
+	create_report = PythonOperator(
+		task_id = "create_report",
+		python_callable = create_report_execute,
+		provide_context = True
+		)
+
+load_data >> drift_analysis >> [create_report]
+```
+
+<img width="1332" alt="image" src="https://github.com/user-attachments/assets/aa4b3d97-acea-4e47-bdd5-e1e5417db2b2">
+
+### Log artifacts using mlflow
+
+```python
+import os
+import pandas as pd 
+
+from datetime import datetime, timedelta
+from sklearn import datasets
+import mlflow
+
+from evidently.test_suite import TestSuite 
+from evidently.test_preset import DataDriftTestPreset
+
+bank_marketing = datasets.fetch_openml(name='bank-marketing', as_frame='auto')
+bank_marketing_data = bank_marketing.frame
+reference_data = bank_marketing_data[5000:7000]
+prod_simulation_data = bank_marketing_data[7000:]
+batch_size = 2000 
+
+#set experiment
+mlflow.set_tracking_uri("sqlite:///mlflow.db")
+mlflow.set_experiment("Drift Tests")
+
+for batch_id in range(10):
+	with mlflow.start_run() as run:
+		data_drift_suite = TestSuite(tests=[DataDriftTestPreset()])
+		data_drift_suite.run(reference_data=reference_data,
+			current_data=prod_simulation_data[batch_id * batch_size : (batch_id + 1) * batch_size])
+
+		data_drift_suite.save_html("data_drift_suite.html")
+
+		#Log parameters
+		mlflow.log_param("batch_id", f"batch_{batch_id}")
+		mlflow.log_param("success_tests", data_drift_suite.as_dict()['summary']['success_tests'])
+		mlflow.log_param("failed_tests", data_drift_suite.as_dict()['summary']['failed_tests'])
+
+		#Log report
+		mlflow.log_artifact("data_drift_suite.html")
+
+		print(run.info)
+```
+
+<img width="1343" alt="image" src="https://github.com/user-attachments/assets/0aeeacf0-ee0e-4212-a04a-b0a1bdab1773">
+
+---
+
+Below is [module 6](https://learn.evidentlyai.com/ml-observability-course/module-6-deploying-an-ml-monitoring-dashboard).
+
+## 6.1. How to deploy a live ML monitoring dashboard
+
+### Why build a monitoring dashboard?
+
+Tests and reports are great for running structured checks and exploring and debugging your data and ML models. However, when you run tests or build an ad-hoc report, you evaluate only a specific batch of data. This makes your monitoring “static”: you can get alerts but have no visibility into metric evolution and trends.
+
+That is where the ML monitoring dashboard comes into play, as it:
+
+* Tracks metrics over time,
+
+* Aids in trend analysis and provides insights,
+
+* Provides a shared UI to enhance visibility for stakeholders, e.g., data scientists, model users, product managers, business stakeholders, etc.
+
+![image](https://github.com/user-attachments/assets/c29e7aa1-76e4-4aa3-bd9d-4188fc63eba0)
+
+### ML monitoring architecture recap
+
+![image](https://github.com/user-attachments/assets/3dac8b3e-8cfa-46e4-8d11-3f6dc07d093b)
+
+## 6.2. ML model monitoring dashboard with Evidently. Batch architecture (code practice)
+
+
+```python
+import datetime
+
+from sklearn import datasets
+
+from evidently.report import Report
+from evidently.metrics import ColumnDriftMetric, DatasetDriftMetric
+
+from evidently.test_suite import TestSuite
+from evidently.test_preset import DataDriftTestPreset
+
+from evidently.ui.dashboards import CounterAgg
+from evidently.ui.dashboards import DashboardPanelCounter
+from evidently.ui.dashboards import DashboardPanelPlot
+from evidently.ui.dashboards import PanelValue
+from evidently.ui.dashboards import PlotType
+from evidently.ui.dashboards import ReportFilter
+from evidently.ui.dashboards import DashboardPanelTestSuite
+from evidently.ui.dashboards import TestFilter
+from evidently.ui.dashboards import TestSuitePanelType
+from evidently.renderers.html_widgets import WidgetSize
+
+from evidently.ui.workspace import Workspace
+from evidently.ui.workspace import WorkspaceBase
+
+bank_marketing = datasets.fetch_openml(name='bank-marketing', as_frame='auto')
+bank_marketing_data = bank_marketing.frame
+
+reference_data = bank_marketing_data[5000:7000]
+prod_simulation_data = bank_marketing_data[7000:]
+batch_size = 2000
+
+WORKSPACE = "bank_data"
+
+YOUR_PROJECT_NAME = "Bank Marketing Classification"
+YOUR_PROJECT_DESCRIPTION = "Test project using Bank Marketing dataset"
+
+
+def create_data_quality_report(i: int):
+    report = Report(
+        metrics=[
+            DatasetDriftMetric(),
+            ColumnDriftMetric(column_name="Class"),
+        ],
+        timestamp=datetime.datetime.now() + datetime.timedelta(days=i),
+    )
+
+    report.run(reference_data=reference_data, current_data=prod_simulation_data[i * batch_size : (i + 1) * batch_size])
+    return report
+
+def create_data_drift_test_suite(i: int):
+    suite = TestSuite(
+        tests=[
+            DataDriftTestPreset()
+        ],
+        timestamp=datetime.datetime.now() + datetime.timedelta(days=i),
+        tags = []
+    )
+
+    suite.run(reference_data=reference_data, current_data=prod_simulation_data[i * batch_size : (i + 1) * batch_size])
+    return suite
+
+def create_project(workspace: WorkspaceBase):
+    project = workspace.create_project(YOUR_PROJECT_NAME)
+    project.description = YOUR_PROJECT_DESCRIPTION
+    project.dashboard.add_panel(
+        DashboardPanelCounter(
+            filter=ReportFilter(metadata_values={}, tag_values=[]),
+            agg=CounterAgg.NONE,
+            title="Bank Marketing Dataset",
+        )
+    )
+    
+    project.dashboard.add_panel(
+        DashboardPanelPlot(
+            title="Target Drift",
+            filter=ReportFilter(metadata_values={}, tag_values=[]),
+            values=[
+                PanelValue(
+                    metric_id="ColumnDriftMetric",
+                    metric_args={"column_name.name": "Class"},
+                    field_path=ColumnDriftMetric.fields.drift_score,
+                    legend="target: Class",
+                ),
+            ],
+            plot_type=PlotType.LINE,
+            size=WidgetSize.HALF
+        )
+    )
+
+    project.dashboard.add_panel(
+        DashboardPanelPlot(
+            title="Dataset Drift",
+            filter=ReportFilter(metadata_values={}, tag_values=[]),
+            values=[
+                PanelValue(metric_id="DatasetDriftMetric", field_path="share_of_drifted_columns", legend="Drift Share"),
+            ],
+            plot_type=PlotType.BAR,
+            size=WidgetSize.HALF
+        )
+    )
+
+    project.dashboard.add_panel(
+        DashboardPanelTestSuite(
+            title="Data Drift tests",
+            filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+            size=WidgetSize.HALF
+        )
+    )
+
+    project.dashboard.add_panel(
+        DashboardPanelTestSuite(
+            title="Data Drift tests: detailed",
+            filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+            size=WidgetSize.HALF,
+            panel_type=TestSuitePanelType.DETAILED
+
+        )
+    )
+
+    project.save()
+    return project
+
+
+def create_demo_project(workspace: str):
+    ws = Workspace.create(workspace)
+    project = create_project(ws)
+
+    for i in range(0, 10):
+        report = create_data_quality_report(i=i)
+        ws.add_report(project.id, report)
+
+        suite = create_data_drift_test_suite(i=i)
+        ws.add_report(project.id, suite)
+
+
+if __name__ == "__main__":
+    create_demo_project(WORKSPACE)
+```
+
+First, execute the above script, and then we can run `evidently ui --workspace bank_data`
+
+<img width="667" alt="image" src="https://github.com/user-attachments/assets/d0e64ac2-b250-4021-b56e-7cfb0d8e2d84">
+
+And when we go to the browser:
+
+<img width="1508" alt="image" src="https://github.com/user-attachments/assets/4bf9d743-5131-476a-9141-02948e74f8b8">
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/e600edc2-bbce-4cc4-9c62-8cad239f04a5">
+
+We can view indiviaul reports and tests:
+
+<img width="488" alt="image" src="https://github.com/user-attachments/assets/fc7afad2-5112-4575-a0d1-1ab99d57232d">
+
+## 6.3. ML model monitoring dashboard with Evidently. Online architecture (code practice)
+
+```python
+import datetime
+import os.path
+import time
+import pandas as pd
+
+from requests.exceptions import RequestException
+from sklearn import datasets
+
+from evidently.collector.client import CollectorClient
+from evidently.collector.config import CollectorConfig, IntervalTrigger, ReportConfig
+
+from evidently.test_suite import TestSuite
+from evidently.test_preset import DataDriftTestPreset
+
+from evidently.ui.dashboards import DashboardPanelTestSuite 
+from evidently.ui.dashboards import ReportFilter
+from evidently.ui.dashboards import TestFilter
+from evidently.ui.dashboards import TestSuitePanelType
+from evidently.renderers.html_widgets import WidgetSize
+from evidently.ui.workspace import Workspace
+
+COLLECTOR_ID = "default"
+COLLECTOR_TEST_ID = "default_test"
+
+PROJECT_NAME = "Bank Marketing: online service "
+WORKSACE_PATH = "bank_data"
+
+client = CollectorClient("http://localhost:8001")
+
+bank_marketing = datasets.fetch_openml(name="bank-marketing", as_frame="auto")
+bank_marketing_data = bank_marketing.frame
+reference_data = bank_marketing_data[5000:5500]
+prod_simulation_data = bank_marketing_data[7000:]
+mini_batch_size = 50
+
+def setup_test_suite():
+	suite = TestSuite(tests=[DataDriftTestPreset()], tags=[])
+	suite.run(reference_data=reference_data, current_data=prod_simulation_data[:mini_batch_size])
+	return ReportConfig.from_test_suite(suite)
+
+def workspace_setup():
+	ws = Workspace.create(WORKSACE_PATH)
+	project = ws.create_project(PROJECT_NAME)
+	project.dashboard.add_panel(
+		DashboardPanelTestSuite(
+			title="Data Drift Tests",
+			filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+			size=WidgetSize.HALF
+		)
+	)
+	project.dashboard.add_panel(
+		DashboardPanelTestSuite(
+			title="Data Drift Tests",
+			filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+			size=WidgetSize.HALF,
+			panel_type=TestSuitePanelType.DETAILED
+		)
+	)
+	project.save()
+
+def setup_config():
+	ws = Workspace.create(WORKSACE_PATH)
+	project = ws.search_project(PROJECT_NAME)[0]
+
+	test_conf = CollectorConfig(trigger=IntervalTrigger(interval=5),
+		report_config=setup_test_suite(), project_id=str(project.id))
+
+	client.create_collector(COLLECTOR_TEST_ID, test_conf)
+	client.set_reference(COLLECTOR_TEST_ID, reference_data)
+
+def send_data():
+	print("Start sending data")
+	for i in range(50):
+		try:
+			data = prod_simulation_data[i * mini_batch_size : (i + 1) * mini_batch_size]
+			client.send_data(COLLECTOR_TEST_ID, data)
+			print("sent")
+		except RequestException as e:
+			print(f"collector service is not available: {e.__class__.__name__}")
+		time.sleep(1)
+
+def main():
+	if not os.path.exists(WORKSACE_PATH) or len(Workspace.create(WORKSACE_PATH).search_project(PROJECT_NAME)) == 0:
+		workspace_setup()
+
+	setup_config()
+	send_data()
+
+if __name__ == '__main__':
+	main()
+```
+
+This is the same as before, but now we update the evidently UI as new data is coming in. It is called 'near real-time' monitoring.
+
+---
+
+Overall, what a great course. Definitely going in my 'courses to recommend' list.
+
+## Neo4j courses
+
+Today I found neo4j offers free courses and tests. 
+
+* [neo4j certified professional](https://graphacademy.neo4j.com/certifications/neo4j-certification/) (its free and I get a free t-shirt if I succeed)
+* [neo4j graph data science certification](https://graphacademy.neo4j.com/courses/gds-certification/)
+
+## [Neo4j Fundamentals](https://graphacademy.neo4j.com/courses/neo4j-fundamentals/)
+
+<img width="285" alt="image" src="https://github.com/user-attachments/assets/70620086-bdbe-49c2-aed8-a8f4391f4b55">
+
+Graph thinking:
+
+- where graph theory originated (Russia)
+- what are nodes and edges
+- directed vs undirected graphs
+- shortest path algorithms
+- graph use cases (i.e. recommendations, network dependency analysis, supply chain management)
+
+Property graphs:
+
+<img width="813" alt="image" src="https://github.com/user-attachments/assets/73c84766-d6c3-414f-909b-93f4970f3db2">
+
+RDBMS vs Native Graph db
+
+<img width="537" alt="image" src="https://github.com/user-attachments/assets/64c2029d-24d7-476f-bff9-2e503a2de7fc">
+
+<img width="555" alt="image" src="https://github.com/user-attachments/assets/b32a369c-66a1-41ec-9f0b-a28c7b62f3ac">
+
+
+1. Neo4j offers faster queries due to index-free adjacency, as relationships are stored during writing, unlike relational databases where joins are computed during reading.
+
+2. It simplifies relationship modeling, making it more intuitive than using pivot tables for many-to-many relationships in relational databases.
+
+The last bit was using Cypher to do basic node and relationship querying on a movie dataset.
+
+<img width="1041" alt="image" src="https://github.com/user-attachments/assets/8c2ca07d-0275-45c7-be3f-737d75ab73b0">
+
+## [Cyper Fundamentals](https://graphacademy.neo4j.com/courses/cypher-fundamentals/)
+
+<img width="284" alt="image" src="https://github.com/user-attachments/assets/87b47e9f-11d5-4d42-86d8-a5addde9f886">
+<img width="269" alt="image" src="https://github.com/user-attachments/assets/5d49ce0f-ae00-4559-9149-f1ce00e29c36">
+
+
+Sample graph structure
+
+<img width="553" alt="image" src="https://github.com/user-attachments/assets/b8823d8b-1ce1-4e93-8326-5257488689b5">
+
+Retrieving exercise for Kevin Bacon's birht year:
+
+```cypher
+MATCH (p:Person)
+where p.name = "Kevin Bacon"
+RETURN p.born
+```
+
+-> 1958
+
+Another exercise: How many people directed the movie Cloud Atlas?
+
+```cypher
+MATCH (m:Movie {title: 'Cloud Atlas'})<-[:DIRECTED]-(p) 
+RETURN count(p)
+```
+
+-> 3
+
+Another exercise: How many actors in the movie The Matrix were born after 1960?
+
+```cypher
+MATCH (a:Person)-[:ACTED_IN]->(m:Movie)
+WHERE a.born > 1960 AND m.title = 'The Matrix'
+RETURN count(a)
+```
+
+-> 4
+
+Create a node exercise
+
+```cypher
+MERGE (p:Person {name: "Daniel Kaluuya"})
+```
+
+Added 1 label, created 1 node, set 1 property, completed after 98 ms. MERGE and CREATE can be used but MERGE first checks if a node with that property exists, and if it does, it does nothing (using MERGE is best practice)
+
+Another exercise:
+
+1. Find the Person node for Daniel Kaluuya.
+
+2. Create the Movie node, Get Out.
+
+3. Add the ACTED_IN relationship between Daniel Kaluuya and the movie, Get Out.
+
+```cypher
+MATCH (p:Person {name: "Daniel Kaluuya"})
+MERGE (m:Movie {title: "Get Out"})
+MERGE (p)-[:ACTED_IN]->(m)
+```
+
+-> Added 1 label, created 1 node, set 1 property, created 1 relationship, completed after 22 ms.
+
+Setting properties exercise: Write the Cypher code to find this Movie node and add the tagline and released properties for the node with the values tagline: Gripping, scary, witty and timely! and released: 2017
+
+```cypher
+MATCH (m:Movie {title: 'Get Out'})
+SET m.tagline = 'Gripping, scary, witty and timely!'
+SET m.released = 2017
+RETURN m.title, m.tagline, m.released
+```
+
+In neo4j we can create nodes and relationships on individual lines with MERGE, or in 1 line:
+
+```cypher
+MERGE (p:Person {name: 'Michael Caine'})-[:ACTED_IN]->(m:Movie {title: 'The Cider House Rules'})
+RETURN p, m
+```
+
+and this is what heppens under the hood:
+
+1. Neo4j will attempt to find a Person node with the name Michael Caine.
+
+2. If it does not exist, it creates the node.
+
+3. Then, it will attempt to expand the ACTED_IN relationships in the graph for this node.
+
+4. If there are any ACTED_IN relationships from this node, it looks for a Movie with the title 'The Cider House Rules'.
+
+5. If there is no node for the Movie, it creates the node.
+
+6. If there is no relationship between the two nodes, it then creates the ACTED_IN relationship between them.
+
+However this can result in an error if one of the items does not exist. I.e. if our graph has a node for Michael, but there is not for the movie. It tried to create the entire pattern, but it failed because the node Person with Michael Caine already exists. That is why it is best practice to use MERGE on individual lines when adding nodes/relationships
+
+```cypher
+// Find or create a person with this name
+MERGE (p:Person {name: 'Michael Caine'})
+
+// Find or create a movie with this title
+MERGE (m:Movie {title: 'The Cider House Rules'})
+
+// Find or create a relationship between the two nodes
+MERGE (p)-[:ACTED_IN]->(m)
+```
+
+To remove properties we can use REMOVE, to remove node/edge we use DELETE
+
+If a node has a relationship, the operation will result in an error. To avoid this, we can use DETACH DELETE which 1st deletes any relationship and then the node
+
+<img width="1043" alt="image" src="https://github.com/user-attachments/assets/49828498-3340-42af-b4bd-c89d34137d46">
+
+## [Graph Data Modelling Fundamentals](https://graphacademy.neo4j.com/courses/modeling-fundamentals/)
+
+Course structure:
+
+1. Intro
+2. Node modelling
+3. Relationships modelling
+4. Testing the model
+5. Refactoring the graph
+6. Eliminating duplicate data
+7. Using specific relationships
+8. Adding intermediate nodes
+
+
+
+Data modeling process
+
+Here are the steps to create a graph data model:
+
+* Understand the domain and define specific use cases (questions) for the application.
+
+* Develop the initial graph data model:
+
+  * Model the nodes (entities).
+
+  * Model the relationships between nodes.
+
+* Test the use cases against the initial data model.
+
+* Create the graph (instance model) with test data using Cypher.
+
+* Test the use cases, including performance against the graph.
+
+* Refactor (improve) the graph data model due to a change in the key use cases or for performance reasons.
+
+* Implement the refactoring on the graph and retest using Cypher.
+
+Graph data modeling is an iterative process. Your initial graph data model is a starting point, but as you learn more about the use cases or if the use cases change, the initial graph data model will need to change. In addition, you may find that especially when the graph scales, you will need to modify the graph (refactor) to achieve the best performance for your key use cases.
+
+Refactoring is very common in the development process. A Neo4j graph has an optional schema which is quite flexible, unlike the schema in an RDBMS. A Cypher developer can easily modify the graph to represent an improved data model.
+
+### Types of models
+
+When performing the graph data modeling process for an application, you will need at least two types of models:
+
+* Data model
+
+* Instance model
+
+Data model
+
+The data model describes the labels, relationships, and properties for the graph. It does not have specific data that will be created in the graph.
+
+Here is an example of a data model:
+
+<img width="463" alt="image" src="https://github.com/user-attachments/assets/ab1ba46c-528d-4287-bf6f-68d4a2826b6b">
+
+There is nothing that uniquely identifies a node with a given label. A graph data model, however is important because it defines the names that will be used for labels, relationship types, and properties when the graph is created and used by the application.
+
+Instance model
+
+An important part of the graph data modeling process is to test the model against the use cases. To do this, you need to have a set of sample data that you can use to see if the use cases can be answered with the model.
+
+Here is an example of an instance model:
+
+<img width="449" alt="image" src="https://github.com/user-attachments/assets/44b3edba-1196-403d-8b13-20d79e424487">
+
+In this instance model, we have created some instances of Person and Movie nodes, as well as their relationships. Having this type of instance model will help us to test our use cases.
+
+
+### Modelling nodes
+
+#### Defining labels
+
+Entities are the dominant nouns in your application use cases:
+
+* What ingredients are used in a recipe?
+
+* Who is married to this person?
+
+The entities of your use cases will be the labeled nodes in the graph data model.
+
+In the Movie domain, we use the nouns in our use cases to define the labels, for example:
+
+* What people acted in a movie?
+
+* What person directed a movie?
+
+* What movies did a person act in?
+
+Here are some of the labeled nodes that we will start with.
+
+<img width="444" alt="image" src="https://github.com/user-attachments/assets/692a5f3d-3cdd-4c36-8482-05b14c6686a9">
+
+#### Properties for nodes
+
+Depending on our use cases we need to decide on node properties
+
+| Use Case                                                        | Steps Required                                             |
+|------------------------------------------------------------------|------------------------------------------------------------|
+| 1: What people acted in a movie?                                 | Retrieve a movie by its title.                             |
+|                                                                  | Return the names of the actors.                            |
+| 2: What person directed a movie?                                 | Retrieve a movie by its title.                             |
+|                                                                  | Return the name of the director.                           |
+| 3: What movies did a person act in?                              | Retrieve a person by their name.                           |
+|                                                                  | Return the titles of the movies.                           |
+| 5: Who was the youngest person to act in a movie?                | Retrieve a movie by its title.                             |
+|                                                                  | Evaluate the ages of the actors.                           |
+|                                                                  | Return the name of the youngest actor.                     |
+| 7: What is the highest rated movie in a particular year?         | Retrieve all movies released in a particular year.         |
+|                                                                  | Evaluate the IMDb ratings.                                 |
+|                                                                  | Return the movie title.                                    |
+| 8: What drama movies did an actor act in?                        | Retrieve the actor by name.                                |
+|                                                                  | Evaluate the genres for the movies the actor acted in.      |
+|                                                                  | Return the movie titles.                                   |
+
+Here is the initial data model
+
+<img width="431" alt="image" src="https://github.com/user-attachments/assets/61db3d35-f992-471d-a762-491cb02e22a7">
+
+And an initial instance model
+
+<img width="458" alt="image" src="https://github.com/user-attachments/assets/83edc108-1bd9-4d44-a1b2-aabbcb172be9">
+
+
+### Modelling relationships
+
+Will continue tomorrow ^^
+
+
+## Pen and paper exercises in ML: Optimisation
+
+![Optimisation P1](https://github.com/user-attachments/assets/623ad2eb-70dc-4408-a757-72df5b9b4c7b)
+![Optimisation P2](https://github.com/user-attachments/assets/f34d04e0-e768-44e6-9483-b487a2d33fd6)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 5604ac1b968683503897264d1e22b0e5c616d217 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 17 Sep 2024 20:57:01 +0900
Subject: [PATCH 425/638] Create 2024-09-17-day-260.md

---
 _posts/2024-09-17-day-260.md | 259 +++++++++++++++++++++++++++++++++++
 1 file changed, 259 insertions(+)
 create mode 100644 _posts/2024-09-17-day-260.md

diff --git a/_posts/2024-09-17-day-260.md b/_posts/2024-09-17-day-260.md
new file mode 100644
index 000000000..a6a8dfe28
--- /dev/null
+++ b/_posts/2024-09-17-day-260.md
@@ -0,0 +1,259 @@
+---
+layout: post
+title: (Day 260) DeepLearning.AI Data Engineering Professional Certificate got realeased
+categories: [data-eng]
+---
+
+# Hello :) Today is Day 260!
+A quick summary of today:
+* [the DE course](https://www.coursera.org/professional-certificates/data-engineering) by the writer of the DE holy book - Fundamentals of DE is live
+
+I thought the course release date is the 19th, but I got an email saying I got access from today. I pre-registered so I got 14-day free trial and that means I have 14 days to complete the course
+
+Yesterday I was planning to continue with the neo4j courses, but given that this DE course is for free for a limited time, this takes priority now.
+
+Here is an overview of the course
+
+<img width="1226" alt="image" src="https://github.com/user-attachments/assets/38f2d92c-5df1-4967-8480-5a15d6cfede9">
+
+This program was designed by Joe Reis in partnership with DeepLearning.AI and AWS to cover the fundamentals of data engineering, both in terms of the underlying theory and frameworks for thinking like a data engineering, as well as practical skills for building data engineering solutions on the cloud. 
+
+## Today I started the 1st course: **Introduction to Data Engineering**
+
+----------------------------------------------------------------------------
+
+## Week 1
+
+<img width="824" alt="image" src="https://github.com/user-attachments/assets/69df0622-270b-42d2-9a8a-cfe7bf6ac4f0">
+
+As a DE we will focus on the big square in the middle, but we will also look into the other 2 sides.
+
+### A Brief History of Data Engineering
+
+Data is foundational to information, existing in various forms such as numbers, words, or even phenomena like photons or wind. While data has always existed, digital data—recorded and processed through computers—gained prominence in the 1960s with the advent of computerized databases, followed by relational databases in the 1970s and the development of SQL. The 1980s introduced data warehouses for analytical decision-making, while the 1990s saw the rise of data systems for business intelligence. The dotcom boom in the mid-1990s led to rapid growth in web applications, supported by backend systems like servers and databases.
+
+The 2000s saw the emergence of big data, driven by companies like Yahoo, Google, and Amazon. Big data refers to datasets characterized by velocity, variety, and volume, and its era began with innovations like Google’s MapReduce (2004) and Apache Hadoop (2006). Simultaneously, Amazon introduced cloud services like EC2 and S3, revolutionizing data storage and processing. AWS became a major player, followed by Google Cloud and Microsoft Azure.
+
+By the late 2000s, the transition to real-time data streaming and cloud-first tools simplified big data processing. Today, the term “big data” has lost relevance, as scalable data tools are accessible to all companies, and data engineers focus on building scalable, business-driven data systems. The role of the modern data engineer is to leverage innovations and tools developed over the past decades to create data solutions that drive business value.
+
+<img width="786" alt="image" src="https://github.com/user-attachments/assets/14d5d2b1-5559-4622-928e-ceaff626f08a">
+
+### The Data Engineer Among Other Stakeholders
+
+<img width="832" alt="image" src="https://github.com/user-attachments/assets/9e916c54-8f27-441f-a324-47e5a15ca81d">
+
+### System Requirements
+
+<img width="775" alt="image" src="https://github.com/user-attachments/assets/19301104-71b8-424b-aa1a-2b3d4c7e1161">
+
+<img width="753" alt="image" src="https://github.com/user-attachments/assets/87061e05-1f37-4d9b-aeb7-728378f8dc7e">
+
+The first step in any data engineering project is gathering system requirements, typically from downstream stakeholders who have business goals in mind. These goals often aren’t expressed as clear system requirements, so it's the data engineer's job to translate them. The process starts with conversations tailored to each stakeholder's technical background and role. In upcoming videos, experts Sol Rashidi and Jordan Morrow will provide advice on communicating with different stakeholders and effective requirements gathering. These videos are optional, and you can skip ahead to see a practical example involving an e-commerce company.
+
+### Translate Stakeholder Needs into Specific Requirements
+
+<img width="746" alt="image" src="https://github.com/user-attachments/assets/24f520a2-e9f0-47f2-9848-1c5f28502a53">
+
+Example conversation with a DS
+
+<img width="543" alt="image" src="https://github.com/user-attachments/assets/e4559273-bbe7-455d-a590-4a9719ac2ab4">
+
+<img width="684" alt="image" src="https://github.com/user-attachments/assets/d878e2ca-a32c-4e4f-b1b6-9412b335fd63">
+
+<img width="777" alt="image" src="https://github.com/user-attachments/assets/7311cf50-117a-48fb-b6c6-a6920a348c0d">
+
+<img width="818" alt="image" src="https://github.com/user-attachments/assets/2e8d40e3-6c42-410c-b4b6-d4a67ac18a4a">
+
+Key tactic: ask stakeholders what action they plan to take with the data (its not the same as asking what they need)
+
+<img width="809" alt="image" src="https://github.com/user-attachments/assets/cb0fe99b-d3bb-4993-b565-19ab28214f50">
+
+### Thinking like a DE
+
+<img width="843" alt="image" src="https://github.com/user-attachments/assets/8c99451b-034e-4c9a-bf16-c74e51947a16">
+
+<img width="737" alt="image" src="https://github.com/user-attachments/assets/df3350f9-5448-4fbe-b3c9-bb5b60283226">
+
+### DE on the Cloud (aws)
+
+The 2nd part of week 1 was an intro to the cloud and AWS' services.
+
+5 core services: compute, network, storage, databases and security (there are more but these are good to start with)
+
+<img width="835" alt="image" src="https://github.com/user-attachments/assets/3300b35b-395e-4851-8a49-64b9241cb6ac">
+
+<img width="823" alt="image" src="https://github.com/user-attachments/assets/21d916ed-6ed9-4401-8e1d-1800668ef224">
+
+<img width="771" alt="image" src="https://github.com/user-attachments/assets/811ba765-33d7-49c4-a96c-fcdb7504cc9a">
+
+Databases:
+
+<img width="776" alt="image" src="https://github.com/user-attachments/assets/1b82295f-a67d-40df-99ca-6844a0b3475e">
+
+<img width="825" alt="image" src="https://github.com/user-attachments/assets/84defbd9-5477-44e7-abe0-d80441447c3b">
+
+## Week 2
+
+This part was very similar to his (Joe's) book (unsurprisingly). And it's nice seeing a video of the material I read in the book. 
+
+<img width="834" alt="image" src="https://github.com/user-attachments/assets/bafefd0d-43dc-409c-8224-8d34834d546b">
+
+
+### Data generation in Source systems
+
+<img width="813" alt="image" src="https://github.com/user-attachments/assets/5bee449c-7698-4442-b84a-a8544fb34968">
+
+The first stage of the data engineering lifecycle involves generating data from various source systems. As a data engineer, you’ll work with data from diverse sources like internal databases, APIs, and IoT devices. These systems, often maintained by other teams or external vendors, are out of your control, but understanding how they work is vital for building reliable pipelines. Common sources include relational databases, NoSQL systems, files, APIs, and data-sharing platforms. 
+
+In reality, source systems are unpredictable—schemas may change, or systems may go down, causing downstream issues. Establishing good relationships with source system owners is crucial for navigating changes and ensuring smooth data workflows. Next, the process moves to data ingestion, which varies depending on the project.
+
+### Ingestion
+
+Ingestion is moving raw data from source systems into your data pipeline for further processing
+
+<img width="690" alt="image" src="https://github.com/user-attachments/assets/f9082040-247a-464b-add7-25b7ff0906e7">
+
+
+### Storage
+
+<img width="629" alt="image" src="https://github.com/user-attachments/assets/567cf7f0-f915-4023-bfc4-e27cbc39d9cb">
+
+As a data engineer, it's entirely possible that you will spend much of your time operating at or near the top of this hierarchy, meaning that you won't be required to think about the details of exactly how your data is moving between different storage components and systems. However, you will be most effective in your work. If you take the time to understand the inner workings capabilities and limitations of your entire storage solutions right down to the wrong ingredients. The truth is that many practicing data engineers today do not deeply understand the details of the storage systems they build. This leads to unfortunate consequences when it comes to things like performance and cost.
+
+### Transformation
+
+The transformation stage in the data engineering life cycle is where value is added by turning raw data into something useful for downstream users like business analysts and data scientists. Transformation involves three parts: queries, modeling, and transformation itself. Queries retrieve data from storage, often using SQL, and must be carefully written to avoid issues like performance problems or row explosion. Data modeling structures the data to reflect real-world relationships and make it easier for stakeholders to use, often requiring denormalization. Finally, transformation involves manipulating and enriching data at various stages, ensuring it's in the correct format for downstream use, whether for reporting, analytics, or machine learning. 
+
+### Serving data
+
+* analytics (BI, operational analytics, embedded analytics)
+* ML
+* reverse ETL
+
+### Undercurrents
+
+* security - use the principle of least privilage; adopt a defensive mindset
+* data management - the development, execution, and supervision of plans, programs, and practices that deliver, control, protect, and enhance the value of data and information assets throughout their life cycles. Data governance is first and foremost a data management function to ensure the quality, integrity, security, and usability of the data collected from an organization
+
+<img width="743" alt="image" src="https://github.com/user-attachments/assets/2c6888a9-61c1-4bce-a640-6f96b39cabc8">
+
+* data architecture - a roadmap or blueprint for our data systems; the design of systems to support the evolving data needs of an enterprise, achieved by flexible and reversible decisions reached through a careful evaluation of trade offs
+
+#### Principles of Good Data Architecture
+
+1. Choose common components wisely
+2. Plan for failure!
+3. Architect for scalability
+4. Architecture is leadership
+5. Always be architecting
+6. Build loosely coupled systems
+7. Make reversible decisions
+8. Prioritize security
+9. Embrace FinOps
+
+* dataOps - aims to improve the development process and quality of data products. DataOps is first and foremost a set of cultural habits and practices that you can adopt. These include things like prioritizing communication and collaboration with other business stakeholders, continuously learning from your successes and failures, and taking an approach of rapid iteration to work toward improvements to your systems and processes. These are also the cultural habits and practices of DevOps, and they're borrowed directly from the agile methodology
+
+<img width="664" alt="image" src="https://github.com/user-attachments/assets/0fca7528-7c89-43dc-9ff3-469bec8acec7">
+
+* orchestration
+
+<img width="835" alt="image" src="https://github.com/user-attachments/assets/bc93face-21ea-44f1-b314-bf01bad4b3eb">
+
+<img width="837" alt="image" src="https://github.com/user-attachments/assets/9b6a6b7b-4125-439a-a146-f854b27ed65d">
+
+### Practical examples on AWS
+
+Below are the AWS options for each part of the DE lifecycle
+
+<img width="842" alt="image" src="https://github.com/user-attachments/assets/6742a5e5-8d36-4d7d-a6f5-6fbe8eeca09d">
+
+<img width="842" alt="image" src="https://github.com/user-attachments/assets/d83ea176-eea9-452e-bc1e-68c89074ebeb">
+
+<img width="834" alt="image" src="https://github.com/user-attachments/assets/e3ad0b12-d13c-4ff4-bdd1-bb3f5e047b64">
+
+<img width="837" alt="image" src="https://github.com/user-attachments/assets/ef4aab60-dd66-4cfb-b8f0-ccfa2b7c97e8">
+
+<img width="837" alt="image" src="https://github.com/user-attachments/assets/96a9ab43-eafd-4053-ab92-e24f59f8d5b1">
+
+<img width="841" alt="image" src="https://github.com/user-attachments/assets/04083391-f7a0-40f9-afdd-f7947fbbd331">
+
+Below are the undercurrents on AWS
+
+<img width="832" alt="image" src="https://github.com/user-attachments/assets/d5cfbf63-6fc5-4fc8-ae7e-c6e5402e8e23">
+
+<img width="783" alt="image" src="https://github.com/user-attachments/assets/4926743c-d47e-49aa-8243-33d0ce968567">
+
+<img width="833" alt="image" src="https://github.com/user-attachments/assets/f07f57d7-cd95-4ca6-9c24-0eb58cdbee92">
+
+Orchestrator - airflow (but any of the other popular ones works)
+
+<img width="661" alt="image" src="https://github.com/user-attachments/assets/153b1daa-3a27-4e72-b2bd-ff84eca127ec">
+
+<img width="746" alt="image" src="https://github.com/user-attachments/assets/02b807b5-6771-4333-94de-021a1984d7fd">
+
+### Next is an assignment in a hosted AWS environment
+
+Below is a summary of the assignment as I do not want to give too much of the actual thing.
+
+<img width="474" alt="image" src="https://github.com/user-attachments/assets/1bc32c45-3287-4662-807b-030bc5e534a7">
+
+
+#### 1. Setup Review
+
+<img width="1491" alt="image" src="https://github.com/user-attachments/assets/11480a77-dc4c-4406-a8a9-d89cd83b7776">
+
+- setup Cloud9 (an AWS IDE)
+
+<img width="1509" alt="image" src="https://github.com/user-attachments/assets/fc458752-3061-455d-9364-676ef34719e4">
+
+
+#### 2. Exploring the Source System
+- Access the source system database (MySQL) using the AWS RDS service.
+- Retrieve the database’s endpoint and connection details from the AWS console.
+- Establish a MySQL connection using the endpoint, username, password, and port.
+- Explore the database structure using SQL commands (`show tables`, etc.).
+
+#### 3. Resource Creation
+- Use Terraform files to create and configure AWS resources (AWS Glue, S3).
+- Initialize the Terraform environment and run commands (`terraform init`, `terraform apply`) to deploy resources.
+- Review `.tf` files for understanding Glue, S3, and networking configurations.
+
+#### 4. Running the Glue Job
+- Start the AWS Glue job using a command provided in the lab.
+- Monitor the job progress in the AWS Glue console (ETL Jobs section).
+- After the Glue job completes, verify the transformed data in the S3 bucket.
+
+#### 5. Querying the Data
+- Use a Jupyter notebook to run analytical queries on the transformed data stored in S3.
+- Utilize AWS Wrangler and Amazon Athena to extract data from S3.
+- Perform queries, such as finding total sales by country and other insights.
+
+Tbh, this is amazing. The way this whole env is setup - really makes me feel like I am using the cloud and doing something real.
+
+The end result is creating an interactive dashboard in jupyter:
+
+<img width="869" alt="image" src="https://github.com/user-attachments/assets/4b9c4639-d6aa-4e30-a1f3-6c0e6c68aa47">
+
+My score after submitting the lab:
+
+<img width="441" alt="image" src="https://github.com/user-attachments/assets/067e6f91-4169-4e6a-a2fd-8b7bdfd4e209">
+
+## Week 3
+
+### Data Architecture
+
+<img width="782" alt="image" src="https://github.com/user-attachments/assets/4ede4b17-c1e7-4009-b534-fb96da044fde">
+
+Conway's law - any organization that designs a system will produce a design who structure is a copy of the organization's communication structure
+
+<img width="814" alt="image" src="https://github.com/user-attachments/assets/3fa0ee83-03cc-4258-b72b-eb61241fdd5f">
+
+To Be Continued ...
+
+---
+
+Tbh, I was expecting high quality from DeepLearning.AI and Joe Reis, and the course so far absolutely delivers on those expectations. 
+
+That is all for today!
+
+See you tomorrow :)

From a74b2024e13bc24454c853c4ba968993784830ff Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 19 Sep 2024 01:37:00 +0900
Subject: [PATCH 426/638] Create 2024-09-18-day-261.md

---
 _posts/2024-09-18-day-261.md | 184 +++++++++++++++++++++++++++++++++++
 1 file changed, 184 insertions(+)
 create mode 100644 _posts/2024-09-18-day-261.md

diff --git a/_posts/2024-09-18-day-261.md b/_posts/2024-09-18-day-261.md
new file mode 100644
index 000000000..a15471987
--- /dev/null
+++ b/_posts/2024-09-18-day-261.md
@@ -0,0 +1,184 @@
+---
+layout: post
+title: (Day 261) Day 2 of the DeepLearning.AI Data Engineering Professional Certificate course
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 261!
+A quick summary of today:
+* finished [course 1](https://www.coursera.org/learn/intro-to-data-engineering?specialization=data-engineering)
+* started [course 2](https://www.coursera.org/learn/source-systems-data-ingestion-and-pipelines?specialization=data-engineering)
+
+
+## Continuing with week 3
+
+### When your system fails
+
+To build resilient data systems, it's essential to not only meet stakeholder needs but also plan for potential failures. Planning for failure includes defining key system metrics like availability (uptime), reliability (ability to perform a function), and durability (resistance to data loss). For example, Amazon S3 offers high availability ranging from 99.5% to 99.99%, but 100% uptime is impossible due to unexpected events. Recovery Time Objective (RTO) defines the maximum time a system can be down, while Recovery Point Objective (RPO) defines the acceptable data loss during an outage.
+
+Alongside planning for failure, prioritizing security is crucial. The Zero Trust Security model, unlike the traditional hardened perimeter approach, ensures no users or applications are trusted by default. This reduces the risk of breaches.
+
+Additionally, unforeseen costs and missed revenue opportunities highlight the need for FinOps and scalability. For instance, mismanaging cloud resources can result in unexpected expenses or failure to scale can lead to system crashes. Thus, managing cloud costs dynamically while ensuring scalable architecture is essential to success.
+
+By planning for failure, prioritizing security, and embracing FinOps, data engineers can better serve their organizations, both when systems are performing well and when failures occur.
+
+### Batch architectures
+
+<img width="811" alt="image" src="https://github.com/user-attachments/assets/1e67d23e-0b73-4a24-a10e-24782c8158b1">
+
+<img width="819" alt="image" src="https://github.com/user-attachments/assets/d081abc8-ff5e-48d7-be31-ced491dad0c8">
+
+### Streaming architectures
+
+<img width="797" alt="image" src="https://github.com/user-attachments/assets/420fdd25-94e0-40ec-9e00-cc87b147da95">
+
+<img width="841" alt="image" src="https://github.com/user-attachments/assets/7c321ddb-cadf-4a7e-8010-65320a40c47e">
+
+<img width="831" alt="image" src="https://github.com/user-attachments/assets/57aa8f12-94b2-4b8a-b55b-e69f5470a898">
+
+<img width="811" alt="image" src="https://github.com/user-attachments/assets/0d968c8b-a338-4a60-bc26-09c556965cfa">
+
+### Choosing tech
+
+<img width="800" alt="image" src="https://github.com/user-attachments/assets/40bca9a5-aa40-471d-8a13-b1dd0e15faeb">
+
+### Location
+
+Nowadays companies are moving more from on-premise to the Cloud, so learning the Cloud is a good option. However, as many companies still use on-premise, its important to know how to transition. 
+
+### Monolith vs Modular system
+
+<img width="813" alt="image" src="https://github.com/user-attachments/assets/0bf6a1b0-68c4-4b9a-91c0-91d6c5ba2430">
+
+### Cost
+
+<img width="836" alt="image" src="https://github.com/user-attachments/assets/91db4084-18df-4ee2-8255-5d906765df63">
+
+### Build vs Buy
+
+<img width="688" alt="image" src="https://github.com/user-attachments/assets/b0b517e7-3277-4181-9c26-199af070286e">
+
+### The AWS well-architected framework
+
+It helps us evaluate and improve the solutions we build on AWS
+
+<img width="833" alt="image" src="https://github.com/user-attachments/assets/eea24eed-b1a7-479a-bd65-b4baca2b31e3">
+
+### Lab exercise
+
+I should not share detailed info about the lab, so here is just an overview:
+
+<img width="819" alt="image" src="https://github.com/user-attachments/assets/4acf36d3-7a5d-46e7-978c-79ef1ad3263b">
+
+<img width="821" alt="image" src="https://github.com/user-attachments/assets/f627c36f-a766-47c1-8d6f-e12674f28f18">
+
+- Three-Tier Architecture: I learned about the three-tier architecture of web applications in AWS, including the presentation, logic, and data tiers. Each plays a vital role in ensuring the web app functions efficiently and can scale.
+
+- Amazon EC2 Auto Scaling: I explored how EC2 Auto Scaling works to manage a group of EC2 instances, allowing the application to scale up or down based on demand. This helps ensure both performance and cost efficiency.
+
+- Application Load Balancer (ALB): I understood how the Application Load Balancer (ALB) distributes traffic across multiple EC2 instances, serving as the single point of contact for client requests.
+
+- VPC and Subnets: I gained insights into Virtual Private Cloud (VPC) and subnets, which allow me to isolate and control access to AWS resources. Subnets help manage public and private traffic within the VPC.
+
+- Monitoring with Amazon CloudWatch: Using CloudWatch, I learned to monitor CPU usage and network activity during stress tests on my web app. This helps track the performance and behavior of AWS resources in real time.
+
+- Enhancing Security with Security Groups: I applied security best practices by modifying security group rules to limit access to specific ports and IP addresses. This improved the web app's security, following the AWS Well-Architected Framework.
+
+- Reliability with Multi-AZ Deployments: By utilizing multiple availability zones (AZs) for EC2 instances, I ensured that my web app remains highly available and fault-tolerant, even in the event of an outage in one AZ.
+
+- Auto Scaling Policies: I created dynamic auto-scaling policies based on traffic demand. This ensures the web app scales automatically, maintaining performance while optimizing resource usage and cost.
+
+- Stress Testing with Apache Benchmark: I performed stress tests on my web app using Apache Benchmark to simulate high traffic and observe how the system reacts under load, making sure it scales efficiently.
+
+*For some reason one of the tasks keeps failing. The task is creating a Launch Template for EC2 instances which I do and then starting instances using that template. But when I submit for grading, the test that checks this particular tasks fails. I reported it and it seems coursera is already aware of it and working on a fix*. In the meantime I will start week 4.
+
+## Week 4
+
+<img width="819" alt="image" src="https://github.com/user-attachments/assets/52912c4c-179f-4c59-a13e-36db88ae97be">
+
+### Req. gathering
+
+<img width="840" alt="image" src="https://github.com/user-attachments/assets/a0b52c4b-4078-4022-8b3c-34a29aa0c1c2">
+
+<img width="752" alt="image" src="https://github.com/user-attachments/assets/879b21f3-2ad6-4e45-b936-d1e705cbe7e1">
+
+Examples
+
+<img width="826" alt="image" src="https://github.com/user-attachments/assets/256f3ec2-1e09-45a3-ba66-926a262c0afa">
+<img width="827" alt="image" src="https://github.com/user-attachments/assets/cf76905e-6526-484a-b377-95a62a75d139">
+
+### The iron triangle
+
+<img width="805" alt="image" src="https://github.com/user-attachments/assets/910eeef1-b03e-4aee-b143-71e51cb6407c">
+<img width="695" alt="image" src="https://github.com/user-attachments/assets/8d9625ae-a9d8-456f-9d72-a816fa121684">
+<img width="735" alt="image" src="https://github.com/user-attachments/assets/3bb9f057-04e2-41a7-b7d0-bbdac59db978">
+<img width="840" alt="image" src="https://github.com/user-attachments/assets/21f829aa-0706-4519-b1a7-a32d5f0a7702">
+
+### Lab overview
+
+<img width="794" alt="image" src="https://github.com/user-attachments/assets/b4b843de-7267-46f1-9278-8e6d9373b3c2">
+
+In more detail:
+
+<img width="510" alt="image" src="https://github.com/user-attachments/assets/132a9286-e616-4bd5-8c83-b6bdd50e5872">
+<img width="692" alt="image" src="https://github.com/user-attachments/assets/542383c8-4239-41fc-977a-903b716ae11e">
+<img width="844" alt="image" src="https://github.com/user-attachments/assets/e79a104c-9f33-4045-a9c9-57a2e9481d08">
+
+End of course 1:
+
+<img width="822" alt="image" src="https://github.com/user-attachments/assets/d640b7d5-98a6-4c38-8fcf-772493eb2ab6">
+
+I just need to wait for the backteam to figure the issue with that 1 assignment and I will officially finish course 1. For now, I am moving to course 2.
+
+Edit after a few hours of writing the above line. I re-did it and I got it. Turns out I will be able to get the certificate after my free trial.
+
+<img width="537" alt="image" src="https://github.com/user-attachments/assets/5ac4712d-2d6c-48cb-a403-8f1f3c208271">
+
+
+## Course 2: Source Systems, Data Ingestion, and Pipelines
+
+----
+
+## Week 1
+
+### Types of source systems
+
+<img width="845" alt="image" src="https://github.com/user-attachments/assets/589168ea-8cfd-4300-8692-a55840668dca">
+<img width="807" alt="image" src="https://github.com/user-attachments/assets/40f0d29c-e8a7-495c-9bb9-db2b533530e9">
+
+### ACID principles
+
+<img width="768" alt="image" src="https://github.com/user-attachments/assets/8df27cc9-30fd-4f48-b37e-abf1202855f0">
+
+- intro to sql
+- SQL dbs
+- noSQL dbs
+- object storage
+- logs
+
+The 1st graded item is using AWS' DynamoDB - all done in a jupyter notebook. Doing CRUD operations
+
+The 2nd graded item is Interacting with amazon S3 object storage
+
+### Streaming systems
+
+<img width="722" alt="image" src="https://github.com/user-attachments/assets/c8ae6bf5-bbdb-46cb-9582-40b5c0f57131">
+<img width="727" alt="image" src="https://github.com/user-attachments/assets/0e778305-a213-478f-bda8-537c586eb3df">
+
+### IAM basics
+
+<img width="842" alt="image" src="https://github.com/user-attachments/assets/d6f4541a-50af-4b3c-9943-24e95417eb2e">
+
+### Networking basics on AWS
+
+<img width="837" alt="image" src="https://github.com/user-attachments/assets/eecb1080-c0fe-468a-a2fa-c4c89aaf58cb">
+
+Tomorrow I continue with the Week 1 final assignment and quiz. 
+
+---
+
+I am covering lots of material. I am glad I am not a beginner thanks to the DE zoomcamp, the side projects I did and also reading some of the popular DE books (i.e. Fundamentals of DE and Streaming DBs). So far, this course is great. It really focuses on the way of thinking as a data engineer, and also gives example conversation b/e a DE and DS/CTO/Marketing manager that gives an idea of what kind of (stakeholder management) meetings a DE might have.
+
+That is all for today!
+
+See you tomorrow :)

From fdcad87da506a49b4504a039622e40ed0bd97c5e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 20 Sep 2024 00:06:17 +0900
Subject: [PATCH 427/638] Create 2024-09-19-day-262.md

---
 _posts/2024-09-19-day-262.md | 327 +++++++++++++++++++++++++++++++++++
 1 file changed, 327 insertions(+)
 create mode 100644 _posts/2024-09-19-day-262.md

diff --git a/_posts/2024-09-19-day-262.md b/_posts/2024-09-19-day-262.md
new file mode 100644
index 000000000..63d660246
--- /dev/null
+++ b/_posts/2024-09-19-day-262.md
@@ -0,0 +1,327 @@
+---
+layout: post
+title: (Day 262) Day 3 of the DeepLearning.AI Data Engineering Professional Certificate course
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 262!
+A quick summary of today:
+* finished course 2 of the specialisation [Source Systems, Data Ingestion, and Pipelines](https://www.coursera.org/learn/source-systems-data-ingestion-and-pipelines?specialization=data-engineering)
+
+
+## Finishing Course 2 Week 1
+
+From yesterday, the last thing from week 1 was the lab and quiz. Below is a simple overview of the issues tackled in the lab as I should not share detailed info.
+
+- Fixing Database Connectivity Issues
+
+So I had to look at the nwtworking setup like:
+
+![image](https://github.com/user-attachments/assets/0affcd73-578d-4741-8183-4a66ff596747)
+
+The issue was that I was trying to connect to the DB through c9, but the two were launched on different VPCs
+
+- Fixing Permission Issues (making sure cloud9 and the db are in the same security group and also can communicate with each other)
+
+![image](https://github.com/user-attachments/assets/e1100898-0be5-4e3e-a944-1eb431a31be2)
+
+We covered:
+
+* how to connect to data sources
+* basics of networking
+* importance of IAM in ensuring security in source systems
+
+![image](https://github.com/user-attachments/assets/b62591a8-9984-494f-ade8-e6bdbfbb7c3d)
+
+## Week 2: Data Ingestion
+
+### Summary: Batch vs. Stream Processing in Data Engineering
+
+Throughout the course, the focus has been on understanding batch and stream processing, particularly in data engineering contexts. Almost all data can be seen as continuous streams of events—such as stock price changes or user interactions—that generate digital data when recorded.
+
+#### Data Ingestion: Stream vs. Batch
+- Stream ingestion processes events individually in real-time as they occur.
+- Batch ingestion imposes boundaries, grouping data by time, size, or number of records, processing it as a single unit.
+
+Both methods exist on a **continuum**, with micro-batch processing blurring the lines between batch and streaming. The choice between batch and streaming depends on the **source system** and the **business use case**.
+
+#### Tools and Technologies
+- Databases: Connectors like JDBC or ODBC facilitate regular or event-based ingestion.
+- Serverless Tools: Services like AWS Glue ETL automate ingestion.
+- APIs: Custom connections may be needed, though API client libraries simplify the process.
+- Files: Sometimes ingestion is manual, using protocols like SFTP or SCP.
+- IoT/Sensor Data: Message queues or streaming systems are often required.
+
+#### Practical Application
+Hands-on use cases include batch ingestion from APIs and streaming ingestion from Kinesis, helping learners become familiar with various ingestion patterns and challenges.
+
+### Batch
+
+![image](https://github.com/user-attachments/assets/bcf4bc6d-f63d-4903-8626-993a330e458a)
+
+Advantages of ELT
+
+* faster to implement
+* makes data available more quicly to end users
+* transformations can still be done efficiently
+
+Disadvantages of ELT
+
+*  if we don't plan on how we will use the data, our pipe can end up being just EL, and have a 'data swamp' of unorganised, unmanegable and essentially useless data
+
+| **Aspect**                  | **ETL**                                                                                                                                     | **ELT**                                                                                                                                                      |
+|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **History**                 | - In the 80’s and 90’s, data warehouse cost was very expensive (millions of dollars), so engineers wanted to be very intentional about the data they were about to load into the data warehouse. <br> - Data volume was still manageable.   | - Cloud data warehouse reduced the cost of storing and processing data significantly (from millions of dollars to hundreds/thousands of dollars only). <br> - Data volume exploded. |
+| **Processing (transformation)** | - Data is transformed into a predetermined format before it is loaded into a data repository. So, data engineers have to carefully model the data and transform it into its final form. <br> - Transformations rely on the processing power of the processing tool that is used to ingest data (unrelated to the target destination). | - Raw data is loaded into the target destination. Then it is transformed just before analysis (can be used with well-defined data requests). <br> - Transformation relies on the processing power of the data repository, such as the data warehouse. |
+| **Maintenance time**        | If the transformation is found to be inadequate, data needs to be re-loaded.                                                                 | The original data is intact and already loaded and can be used when necessary for additional transformation. Less time required for data maintenance.           |
+| **Load Time & transformation time** | Load time: It typically takes longer as it uses a staging area and system. <br> Transformation time: It depends on the data size, the transformation complexity and the tool that is used to perform the transformation. | Load time: There is no transformation involved, the data is directly loaded into the destination system. <br> Transformation time: It is typically faster because it relies on the processing power and parallelization of modern data warehouses (generally considered more efficient). |
+| **Flexibility (data types)** | ETLs are typically designed to handle structured data.                                                                                       | ELT can handle all types of data: structured, unstructured, semi-structured. Once the data is loaded into the target system, you can transform it.               |
+| **Cost**                    | It depends on what ETL/ELT tool is used and what target system the data is loaded. (And of course, it depends on the data volume).               | It depends on what ETL/ELT tool is used and what target system the data is loaded. (And of course, it depends on the data volume).                              |
+| **Scalability**              | Nowadays, most of the cloud tools are scalable. However, the challenge here is that if you have lots of data sources and lots of targets, you would need to put in lots of effort to manage the code and handle data from multiple sources. | ELT uses the scalable processing power of the data warehouse to enable transformation on a large scale.                                                        |
+| **Data quality/security**   | It ensures data quality by cleaning it first. Transformations can also include masking personal information.                                    | The data needs to be transferred first to the target system before transformations that enhance data quality or security are applied. <br> *There’s a sub-pattern called ELT where small does not refer to business modeling but to transformation with limited scope (mask sensitive data, deduplicate rows).* |
+
+### Batch lab
+
+* extract data from the spotify API
+* explore pagination
+* send an API request that requires authorisation
+
+### Streaming ingestion
+
+![image](https://github.com/user-attachments/assets/8e5c55fc-5255-4375-a21d-5a7a954df7e9)
+
+![image](https://github.com/user-attachments/assets/4de553e9-5184-4ef3-bc45-5dae29d8b54d)
+
+In Amazon's Kinesis Data Stream, the wording goes: stream (instead of topic) and shard (instead of partition)
+
+![image](https://github.com/user-attachments/assets/002385b8-1627-4926-894f-64ab31897c25)
+
+### Summary: Change Data Capture (CDC)
+
+#### What is CDC?
+Change Data Capture (CDC) is a method used to track and extract changes (insert, update, delete) in a database, ensuring data synchronization across systems. It allows data updates to be captured incrementally, rather than reloading entire datasets.
+
+##### Strategies for Data Updates:
+1. Full Load (Snapshot): Reloads the entire dataset, replacing old data with the new. It guarantees consistency but is resource-intensive, making it inefficient for high-volume or frequently updated data.
+2. Incremental Load: Loads only the data that has changed since the last update, using mechanisms like `last_updated_at` columns. This approach is faster for large datasets and is commonly known as CDC.
+
+#### Use Cases for CDC:
+- Data Synchronization: Continuously syncs data across systems (e.g., from a transactional database to a data warehouse).
+- Historical Change Tracking: Captures every change for audit or regulatory purposes, preserving historical data.
+- Microservices Integration: Tracks changes in a source database to trigger updates in dependent services, such as shipment or customer service systems.
+
+#### CDC Approaches:
+1. Push: The source system pushes data updates to the target system in real-time. However, if the target is unavailable, data may be lost.
+2. Pull: The target system polls the source database for changes at regular intervals. While simpler, it introduces a time lag.
+
+#### CDC Implementation Patterns:
+1. Batch-oriented/Query-based CDC (Pull-based): Relies on querying the source database for changes, using columns like `updated_at`. It can be computationally expensive due to table scans.
+2. Continuous/Log-based CDC (Pull-based): Utilizes the database's log to capture changes in real-time with minimal overhead. Tools like Debezium can extract log events and send them to streaming platforms like Apache Kafka.
+3. Trigger-based CDC (Push-based): Uses database triggers to notify CDC systems of changes. However, excessive use of triggers can degrade database performance.
+
+### Choosing an ingestion tool can depend on the data
+
+* data type and structure
+* data volume
+* latency requirements
+* data quality
+* changes in schema
+
+Also the tool's reliability and durability
+
+**Advice:** Evaluate the tradeoffs between the cost of losing data vs building an appropriate level of redundancy
+
+### Streaming lab overview
+
+![image](https://github.com/user-attachments/assets/0a476df4-5244-4e2e-baa0-c6476988c8b1)
+
+## Week 3 - DataOps
+
+A reminder, DataOps is a set of practices and cultural habits centred around building robust data systems and delivering high quality data products
+
+### Automation with IaC
+
+![image](https://github.com/user-attachments/assets/33772e3d-60f4-498a-976c-d6276351bde6)
+![image](https://github.com/user-attachments/assets/9e4234dd-6ab1-440f-bb24-0951ce971dba)
+
+- creating an EC2 instance
+- defining variables and outputs
+- defining data sources and modules
+
+### Terraform lab
+
+![image](https://github.com/user-attachments/assets/ad9765d8-be9b-479d-9361-6adc4f4eeb07)
+
+the lab is about setting up and configuring all these terraform files
+
+![image](https://github.com/user-attachments/assets/df33dac2-7301-4a1a-833a-10f4f52d7304)
+
+### Observability
+
+Data issues are inevitable and they could occur at any stage of your data pipeline. The earlier you are able to detect them, the less damage to the organization it will cause. To detect data issues, you need to first choose metrics that assess the data quality, similarly to how software teams monitor metrics that assess the health of their software's infrastructure.
+
+In her book (Data Quality Fundamentals), Barr Moses suggests to start with the following questions:
+
+* Is the data up-to-date?
+
+* Is the data complete?
+
+* Are fields within expected ranges?
+
+* Is the null rate higher or lower than it should be?
+
+* Has the schema changed?
+
+She formulated these questions into 5 pillars for data observability, which aim to fully describe the state of the data. 
+
+#### Barr Moses 5 Pillars
+
+1. Distribution/ Internal Quality: The quality pillar refers to the internal characteristics of the data, and checks metrics such as the percentage of NULL elements, percentage of unique elements, summary statistics and if your data is within the expected range. It helps you ensure that your data is trusted based on your data expectation.
+
+2. Freshness: Data freshness refers to how “fresh” or “up-to-date” the data is within the final asset (table, BI report), i.e., when the data was last updated, and how frequently it is updated. Stale data results in wasted time and money.
+
+3. Volume: Data volume refers to checking the amount of data ingested and looking for unexpected spikes or drops. Sudden drops in data volume can indicate issues like lost data or system outages, and sudden increases may indicate unexpected surges in usage.
+
+4. Lineage: According to Barr, “When data breaks, the first question is always “where?” Data lineage helps you trace the data journey from its source to its destination, visualizing how data was transformed and where it was stored. This way, you can identify the source of errors or anomalies.  
+
+5. Schema: Data schema refers to monitoring changes in data structure or types. This pillar helps avoid the failure of the data pipeline.
+
+![image](https://github.com/user-attachments/assets/45a689d4-401d-4e35-a88b-61af4128bc82)
+
+Metrics can vary on the project
+
+![image](https://github.com/user-attachments/assets/2b8e0464-121c-48a4-8679-10337cb0351f)
+
+When it comes to data accuracy and completeness, we can assume that these are important for every project.
+
+![image](https://github.com/user-attachments/assets/1a38c26b-a8f5-470e-8094-154baa58a5ef)
+
+### Great Expectations workflow
+
+![image](https://github.com/user-attachments/assets/f37b6427-31e3-4153-a4ce-8537c7d256f2)
+
+When working with Great Expectations, a typical workflow involves the following core components:
+
+#### 1. Data Context
+- Purpose: The entry point for the Great Expectations API.
+- Function: Allows you to create objects to connect to data sources, create expectations, and validate data.
+- Role: Manages objects and metadata for your project.
+
+#### 2. Data Sources
+- Purpose: Specify the location of the data you wish to test.
+- Sources: Can be a SQL database, local file system, S3 bucket, or a pandas DataFrame.
+
+#### 3. Data Assets & Batch Requests
+- Data Asset: A collection of records in a data source (e.g., a table, file, or query asset).
+- Batches: Data can be partitioned into smaller batches (e.g., by month or store IDs).
+- Batch Request: Primary way to retrieve data for validation from a data asset.
+
+#### 4. Expectations
+- Purpose: Statements used to verify if data meets certain conditions (e.g., column values are not null, or values are unique).
+- Expectation Suite: A collection of multiple expectations applied to a data asset.
+
+#### 5. Validator
+- Purpose: Validates data against a batch request and its corresponding expectation suite.
+- Manual/Automated Validation: Can be performed directly with the validator or streamlined using a checkpoint.
+
+#### 6. Checkpoints
+- Purpose: Automates the validation process by providing batch requests and expectation suites to a validator.
+- Role: Simplifies generating validation results.
+
+#### 7. Stores
+- Expectation Store: Stores expectation suites.
+- Validation Store: Stores objects generated during data validation.
+- Checkpoint Store: Stores checkpoint configurations.
+- Data Docs Store: Stores reports on expectations, checkpoints, and validation results.
+
+#### Typical Workflow Steps:
+1. Instantiate a Data Context: Set up the context for your Great Expectations project.
+2. Declare a Data Source: Specify from where to get the data for validation.
+3. Create a Batch Request: Retrieve data batches from the data asset.
+4. Define Expectations: Specify conditions your data must meet.
+5. Create an Expectation Suite: Group multiple expectations for a data asset.
+6. Validate Data: Use a validator or checkpoint to run the validation.
+7. Store and Access Metadata: Access the results and configurations through various stores.
+
+GX (Great Expectations) seems like a good validating tool. I got the experience using it in one of the labs. (seems to have some similarities with Evidently, but this is mainly focused on data)
+
+### AWS CloudWatch for monitoring
+
+![image](https://github.com/user-attachments/assets/4d9eeadf-752f-4c21-a796-48b755d03ebf)
+
+![image](https://github.com/user-attachments/assets/569a7d9d-044b-40f2-ab75-d84b1e8c24cb)
+
+## Week 4: Orchestration
+
+Before orchestrators, there was Cron
+
+![image](https://github.com/user-attachments/assets/943cb839-535f-4a53-bf2c-773837a9d007)
+
+Cron is good:
+
+* to schedule simple and repetitive tasks (ex. regular data downloads)
+* when we are in the prototyping phase (ex. testing aspects of our data pipes)
+
+![image](https://github.com/user-attachments/assets/4908c969-0be8-4061-b9d7-46439c7856cd)
+
+### Airflow
+
+- the final part of week 4 and this course 2 is about airflow, basics
+- the lab is about using airflow as well
+
+![image](https://github.com/user-attachments/assets/846a7825-68c3-4254-846e-3e45d71aeb50)
+
+#### 1. Web Server
+- Purpose: Hosts the Airflow User Interface (UI).
+- Function: Provides a graphical interface where users can monitor, trigger, and troubleshoot DAGs and their tasks.
+- Role: Displays the status of DAGs and tasks by extracting information from the metadata database.
+
+#### 2. DAG Directory
+- Purpose: Stores the Python scripts that define your DAGs.
+- Function: The folder where DAG files are placed, and automatically connected to the web server.
+- Role: Any DAG added to the directory is visualized in the Airflow UI.
+
+#### 3. Scheduler
+- Purpose: Constantly monitors DAGs and their tasks.
+- Function: Triggers tasks based on a schedule or when dependencies are met. Pushes tasks to a queue.
+- Role: Determines when a task should run and transitions tasks from scheduled to queued.
+
+#### 4. Workers
+- Purpose: Executes tasks from the queue.
+- Function: Executes tasks assigned by the scheduler and reports the task's success or failure.
+- Role: Handles the actual execution of tasks in a distributed fashion.
+
+#### 5. Metadata Database
+- Purpose: Stores metadata about DAGs and tasks.
+- Function: Keeps records of the status and state of DAGs and tasks.
+- Role: The web server retrieves information from this database to display in the UI.
+
+#### Workflow Example:
+1. DAGs are stored in the DAG Directory.
+2. The Scheduler monitors DAGs and their tasks, and based on schedules or task dependencies, triggers tasks.
+3. The triggered tasks are placed in a queue and executed by Workers.
+4. Workers update the status of tasks (e.g., running, success, failed) in the Metadata Database.
+5. The Web Server retrieves these task statuses from the metadata database and displays them in the UI for monitoring and management.
+
+[Airflow best practices docs](https://airflow.apache.org/docs/apache-airflow/stable/best-practices.html#communication)
+
+<img width="1432" alt="image" src="https://github.com/user-attachments/assets/65bd126b-b2ef-4334-ad61-0fb1cd944f07">
+
+
+### Orchestration options on AWS
+
+<img width="937" alt="image" src="https://github.com/user-attachments/assets/4dfa0eb2-6b68-4b82-9f40-05b5047dbfca">
+
+## Course 2 - completed
+
+<img width="1299" alt="image" src="https://github.com/user-attachments/assets/0d182b38-9515-455d-8738-5ea9d0e46d0c">
+
+Ofc cannot get the certificate now as I am in a free trial period.
+
+---
+
+Tomorrow I continue with course 3
+
+That is all for today!
+
+See you tomorrow :)

From 7cbcb91fa2eb38113237292e95cfc81c42236fb2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 20 Sep 2024 23:38:08 +0900
Subject: [PATCH 428/638] Create 2024-09-20-day-263.md

---
 _posts/2024-09-20-day-263.md | 126 +++++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100644 _posts/2024-09-20-day-263.md

diff --git a/_posts/2024-09-20-day-263.md b/_posts/2024-09-20-day-263.md
new file mode 100644
index 000000000..e861fbb3a
--- /dev/null
+++ b/_posts/2024-09-20-day-263.md
@@ -0,0 +1,126 @@
+---
+layout: post
+title: (Day 263) Day 4 of the DeepLearning.AI Data Engineering Professional Certificate course
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 263!
+A quick summary of today:
+* started course 3 - [data storage and queries](https://www.coursera.org/learn/data-storage-and-queries/home/welcome)
+
+## Structure provided by the course
+
+This course consists of 3 weeks of content and covers these main learning objectives:
+
+* Explain how data is physically stored on disk and in memory
+
+* Compare how data is stored and queried in object, block, and file storage systems
+
+* Explain how data is stored in row-oriented vs column-oriented databases
+
+* Explain how graph and vector databases store and retrieve data
+
+* Explain the key architectural features of data warehouses, data lakes, and data lakehouses
+
+* Implement a data lake using AWS Glue
+
+* Implement a data lakehouse with a medallion-like architecture using Lake Formation and Iceberg
+
+* Explain the stages of the life of a query
+
+* Implement advanced SQL queries
+
+* Explain the role of of an index and its impact on query performance
+
+* Summarize approaches for processing aggregate and join queries
+
+* Compare the execution times of aggregate queries between row and columnar storage
+
+* List some strategies for enhancing query performance
+
+* Aggregate and join streaming data
+
+
+## Week 1
+
+<img width="910" alt="image" src="https://github.com/user-attachments/assets/02aba514-d0b2-4a62-a2d0-722ad592f116">
+
+### Raw storage
+
+<img width="913" alt="image" src="https://github.com/user-attachments/assets/fc307ff0-9b5b-4911-83df-8353c6704bf0">
+
+Data storage systems rely on various "raw ingredients," including magnetic disk drives (HDDs), solid-state drives (SSDs), RAM, and CPU cache, each with unique performance, durability, and cost factors.
+
+#### 1. Magnetic Disk Drives (HDDs)
+- Mechanism: Use rotating platters with magnetic film to store data. Similar to record players, a read/write head locates the right track and encodes/reads binary data.
+- Performance: Mechanical operations limit speed, with latency around 4ms and a few hundred IOPS. Data transfer rate is around 300MB/s.
+- Use Case: Suitable for cost-efficient storage when data access is infrequent and speed is less critical.
+- Cost: HDDs are 2-3 times cheaper than SSDs.
+
+#### 2. Solid-State Drives (SSDs)
+- Mechanism: Stores data as electrical charges in flash memory cells, allowing faster, purely electronic operations.
+- Performance: SSDs handle tens of thousands of IOPS with a data fetch latency of ~0.1ms. They can transfer data up to 10 times faster than HDDs.
+- Use Case: Ideal for systems requiring fast random access and OLTP systems but costlier for analytical workloads.
+- Cost: More expensive than HDDs, especially for large data volumes.
+
+#### 3. Random Access Memory (RAM)
+- Mechanism: Volatile memory used to transfer data from persistent storage for processing.
+- Performance: Very fast, with transfer speeds of ~100GB/s and latency of 0.1 microseconds, performing millions of IOPS.
+- Use Case: Used for caching, data processing, and indexing, but expensive and unsuitable for long-term storage.
+- Cost: 30-50 times more expensive than SSDs. Volatile, meaning data loss if power is lost.
+
+#### 4. CPU Cache
+- Mechanism: Located directly on the CPU, offering ultrafast data retrieval.
+- Performance: Latency of 1 nanosecond and transfer speeds of 1TB/s.
+- Use Case: Best for storing frequently accessed data during processing.
+
+<img width="887" alt="image" src="https://github.com/user-attachments/assets/9173d2c9-9716-4979-98b5-04c278337287">
+
+### Cloud storage options
+
+<img width="817" alt="image" src="https://github.com/user-attachments/assets/4b1c79da-4c07-4fb7-b824-83901f29c404">
+
+As a data engineer, you're faced with various cloud storage options. These systems operate at a higher abstraction level and include block storage, file storage, and object storage. Each has unique performance and scalability trade-offs.
+
+#### File Storage
+File storage systems organize files into directory trees, much like your computer's folder structure. Metadata such as names, owners, modification dates, and permissions help manage these files. Managed services like Amazon Elastic File System (EFS) provide centralized access to shared files over a network, abstracting much of the complexity. However, file storage is generally slower due to the need to manage file hierarchies.
+
+#### Block Storage
+Block storage splits data into small, fixed-size blocks stored on disks like SSDs or magnetic drives. It offers high performance and low latency, often used in transactional databases and virtual machines (e.g., EC2 with Amazon Elastic Block Store, EBS). Block storage provides flexible, distributed architecture but is limited in scalability compared to object storage.
+
+#### Object Storage
+Object storage scales beyond block storage, often into petabytes. It decouples storage from compute, allowing you to handle massive data sets cost-effectively. Data is stored as immutable objects in a flat structure (e.g., AWS S3), and changes require rewriting the entire object. While object storage excels in scalability and parallel processing, it's not suited for transactional workloads.
+
+#### Summary
+- **File Storage**: Simple data sharing, low performance needs.
+- **Block Storage**: High performance, low-latency workloads.
+- **Object Storage**: Massive datasets, read-heavy analytical queries.
+
+### Storage tiers
+
+<img width="857" alt="image" src="https://github.com/user-attachments/assets/2852e68f-749c-41b2-919c-ad93fa89b1f6">
+
+### How DBs store data
+
+A Database Management System (DBMS) facilitates interaction with databases and consists of a transport system, query processor, execution engine, and storage engine. The storage engine is responsible for physically storing data on disk, handling tasks like serialization, arrangement, and indexing. Modern storage engines are optimized for SSDs and can manage complex data structures.
+
+For large-scale data retrieval, using indexes can significantly improve query performance. For example, querying a relational database for rows based on country can be sped up by creating an index, allowing for binary search instead of scanning the entire dataset. This reduces time complexity from O(N) to O(log N).
+
+In-memory databases, such as Memcached and Redis, use RAM for ultra-fast data retrieval. However, they are volatile and should not be used for persistent storage. They are ideal for caching and applications like real-time bidding or gaming leaderboards. Redis, in particular, supports more complex data types and has persistence mechanisms.
+
+### Row vs Column storage
+
+* row-oriented storage is suitable for transactional workloads that require read and write to be performed with low latency
+* column storage for analytical processing (suitable for analytical workloads where you want to apply aggregating operations on columns)
+
+Parquet and ORC (optimized row columnar) are  file formats that combine both approaches by following a hybrid approach that tries to get the best from both worlds. The hybrid approach relies on partitioning rows into groups where each row group is stored in a column-wise format.
+
+### Lab
+
+The lab for this week was about neo4j and writing Cypher queries.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 4b224516699ed90ff73cc47279d035694b705105 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 21 Sep 2024 23:59:45 +0900
Subject: [PATCH 429/638] Create 2024-09-21-day-264.md

---
 _posts/2024-09-21-day-264.md | 112 +++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 _posts/2024-09-21-day-264.md

diff --git a/_posts/2024-09-21-day-264.md b/_posts/2024-09-21-day-264.md
new file mode 100644
index 000000000..8d026e83c
--- /dev/null
+++ b/_posts/2024-09-21-day-264.md
@@ -0,0 +1,112 @@
+---
+layout: post
+title: (Day 264) Day 5 of the DeepLearning.AI Data Engineering Professional Certificate course
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 264!
+A quick summary of today:
+* continue with week 2 and 3 of [course 3](https://www.coursera.org/learn/data-storage-and-queries/home/welcome)
+
+## Week 2
+
+### Data Warehouses
+
+A data warehouse is a subject oriented, integrated, non volatile and time variant collection of data in support of management's decisions
+
+<img width="917" alt="image" src="https://github.com/user-attachments/assets/8a2d99c5-8448-4557-9fba-570c323903f1">
+
+<img width="930" alt="image" src="https://github.com/user-attachments/assets/80c8986b-3d80-4072-9988-fde2a24d20a1">
+
+<img width="936" alt="image" src="https://github.com/user-attachments/assets/4762ec0e-0ef3-4518-aecf-ae31572698c0">
+
+Data warehouse implementation
+
+<img width="922" alt="image" src="https://github.com/user-attachments/assets/7f81cac5-3f24-4932-9303-609793998de6">
+
+<img width="905" alt="image" src="https://github.com/user-attachments/assets/8becae00-9028-4bec-97db-6e4e3693349c">
+
+### Data lakes
+
+<img width="845" alt="image" src="https://github.com/user-attachments/assets/8eba0d9e-9b95-4d3b-99fd-ea3d2dd2987c">
+
+Shortcomings of data lakes 1.0
+
+* data lakes became data swamps: no proper data management, no data cataloging, no discovery tools, no guarantee on the data integrity and quality
+* 1.0 was write-only: data manipulation operations were hard to implement, difficult to comply with regulations (like GDPR)
+* no schema management and data modelling (joins were a huge headache)
+
+Next-gen data lakes
+
+In response to the limitations of the original data lake 1.0, engineers have developed strategies to enhance data management and retrieval. This video discusses three main approaches: **data zones**, **partitioning**, and **data catalogs**.
+
+#### Data Zones
+Data can be organized into different zones within a data lake, often following a three-zone model:
+1. **Landing/Raw Zone**: Stores raw data ingested from source systems, providing a permanent record.
+2. **Cleaned/Transformed Zone**: Contains data that has been cleaned, validated, and standardized, with any PII removed.
+3. **Curated/Enriched Zone**: Holds data modeled with business logic, ready for consumption and stored in efficient open formats like Parquet, Avro, or ORC.
+
+While the number and naming of zones can vary, they allow for appropriate data governance and ensure data quality for users.
+
+#### Data Partitioning
+To improve query performance, data in the cleaned or curated zones can be partitioned based on criteria such as time, date, or location. This technique enables query engines to scan only relevant partitions, resulting in faster performance.
+
+#### Data Catalogs
+A data catalog serves as a centralized collection of metadata about datasets, allowing users to search for information based on data owner, source, partitioning details, and column definitions. It maintains schemas and records changes over time, fostering a common understanding of data structure across the organization.
+
+#### The Data Lake House Architecture
+Despite advancements, organizations often relied on multiple storage systems to meet various needs—leveraging the low-cost storage of data lakes and the superior performance of data warehouses. This led to the costly ETL process for moving data between systems, which introduced risks of data quality issues.
+
+<img width="924" alt="image" src="https://github.com/user-attachments/assets/89595686-d719-4b31-b67b-ca62ed5e4570">
+
+To address these challenges, the **Data Lake House** architecture was developed, merging the benefits of data warehouses and data lakes into a unified solution.
+
+### Data lakehouse
+
+<img width="770" alt="image" src="https://github.com/user-attachments/assets/db2ac502-aafa-43af-91d6-8e91eae6780b">
+
+<img width="918" alt="image" src="https://github.com/user-attachments/assets/51a99bac-5f79-4c62-8fc0-bfabdb104a81">
+
+<img width="883" alt="image" src="https://github.com/user-attachments/assets/1aabc990-5bb6-480b-98f7-8d7e37ebb849">
+
+<img width="939" alt="image" src="https://github.com/user-attachments/assets/547f6f4e-3162-465a-9ce0-f84e8b51d0d0">
+
+I have seen complaints online and memes about AWS' *many* services. And the above pic from the course definitely confirms to this meme 😆
+
+### Summary
+
+<img width="915" alt="image" src="https://github.com/user-attachments/assets/f044d94c-ce94-4f76-9fbf-5e1217c95433">
+
+## Week 3
+
+### Batch queries
+
+There was some SQL practice like:
+
+<img width="940" alt="image" src="https://github.com/user-attachments/assets/1e6a4dc8-d54d-45f2-83b7-665bd05c5742">
+
+And also a lab for SQL which was *very* long and had to create CTE on top of CTE on top of CTE for different cases
+
+<img width="910" alt="image" src="https://github.com/user-attachments/assets/008fa45d-07c2-4a44-a4a5-eae77e58a3e3">
+
+Sometimes we can reduce query time by creating an index
+
+<img width="935" alt="image" src="https://github.com/user-attachments/assets/a1293e5e-b31e-4473-ae3c-bde2248ecff8">
+
+### Streaming queries
+
+<img width="933" alt="image" src="https://github.com/user-attachments/assets/6e4b1656-9fb8-41d9-a703-6723364dc163">
+
+<img width="935" alt="image" src="https://github.com/user-attachments/assets/f91f70cc-d70a-462d-a4ab-54c632b59c7e">
+
+This part introduced AWS' managed Apache Flink services, and the lab was about using and deploying them.
+
+<img width="892" alt="image" src="https://github.com/user-attachments/assets/aca22311-f7bd-48d2-9aaa-aefd78a4dfcb">
+
+<img width="933" alt="image" src="https://github.com/user-attachments/assets/613a94b4-c789-4b9e-bdf5-81b591582ffa">
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f4a8f7944a06098c6b3b4bccdcc22d5b06b775dc Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 22 Sep 2024 23:36:22 +0900
Subject: [PATCH 430/638] Create 2024-09-22-day-265.md

---
 _posts/2024-09-22-day-265.md | 87 ++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 _posts/2024-09-22-day-265.md

diff --git a/_posts/2024-09-22-day-265.md b/_posts/2024-09-22-day-265.md
new file mode 100644
index 000000000..63a5095b1
--- /dev/null
+++ b/_posts/2024-09-22-day-265.md
@@ -0,0 +1,87 @@
+---
+layout: post
+title: (Day 265) Day 6 of the DeepLearning.AI Data Engineering Professional Certificate course
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 265!
+A quick summary of today:
+* started [course 4 - data modelling, transformations and serving](https://www.coursera.org/learn/data-modeling-transformation-serving/home/welcome)
+
+## Started Week 1 - Intro to data modelling for analytics
+
+<img width="913" alt="image" src="https://github.com/user-attachments/assets/c1361b80-7741-4c3f-b71d-a03ae66a6074">
+
+| Good Data Models                                                                 | Poor Data Models                                                                |
+| -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- |
+| :heavy_check_mark: Reflect the business goals and logic while incorporating business rules | :x: Don’t reflect how the business operates                                      |
+| :heavy_check_mark: Ensure compliance with operational standards and legal requirements    | :x: Create more problems than they solve                                         |
+| :heavy_check_mark: Outline the relationships between business processes                  | :x: Provide stakeholders with inaccurate information and create confusion         |
+| :heavy_check_mark: Serves as a powerful communication tool, creating a shared language     |                                                                                  |
+
+<img width="846" alt="image" src="https://github.com/user-attachments/assets/93682108-805a-4207-9793-6685857e3493">
+
+### Data model types
+
+1. Conceptual
+   - describes business entities, relationships, & attributes
+   - includes business logic and rules
+
+2. Logical
+   - details about the implementation of the conceptual model
+
+3. Physical
+   - details about the implementation of the logical model in a specific DBMS
+
+### Normalisation
+
+<img width="886" alt="image" src="https://github.com/user-attachments/assets/06c031f3-93b6-43e0-b0f3-a0cf0f36754a">
+
+Normalization is achieved through several stages called **normal forms**, each building on the previous one:
+
+1. First Normal Form (1NF):
+   - ensures all data in a table is atomic (no nested structures)
+   - each column must have unique values and each table must have a unique primary key
+   
+   *Example:* Splitting a column with nested data (like order items) into individual columns like item SKU, price, and name, while ensuring rows are unique
+
+2. Second Normal Form (2NF):
+   - meets 1NF requirements
+   - eliminates partial dependencies, meaning no column should depend on only part of a composite primary key
+
+   *Example:* Splitting a sales order table into two tables—one for orders and one for order items—removes partial dependencies
+
+3. Third Normal Form (3NF):
+   - meets 2NF requirements.
+   - removes **transitive dependencies**, where non-key columns depend on other non-key columns
+
+   *Example:* Splitting the orders table into separate customer and item tables removes transitive dependencies (ie customer data depending on order ID)
+
+Each normal form progressively reduces data redundancy and improves data integrity. However depending on the use case, denormalization (combining tables) may sometimes be preferable to avoid complex joins and improve performance.
+
+The lab for this part was about data normalisation only 
+
+### Dimensional Modeling - Star Schema
+
+<img width="928" alt="image" src="https://github.com/user-attachments/assets/6ff1d61d-555d-4d6b-a19a-43fd9b7269a2">
+
+<img width="843" alt="image" src="https://github.com/user-attachments/assets/7ca142e0-06bd-4e12-ba63-a0a9b979730a">
+
+**How exactly does the star schema help with analytical queries? **
+
+We can start with a fact table by applying aggregate queries to find, for example, the sum, average, or maximum value of a particular fact measure in the fact table. Then we use the dimension tables to filter or group the facts.
+
+<img width="941" alt="image" src="https://github.com/user-attachments/assets/28f7271a-9374-4061-a46e-c7a613c769ca">
+
+* star schema organizes the data in a way that's easier for business users to understand and navigate
+* also results in simpler queries with fewer joints which speeds up the query performance
+
+---
+
+Today I had to travel around a bit so I mainly studied on the bus rides so I covered just part of week 1 from course 4.
+
+Also this is day 265 meaning there are less than 100 days left for this blog (wow).
+
+That is all for today!
+
+See you tomorrow :)

From 5e16065d02d9367a2567e201c9bb2835ab991d55 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 24 Sep 2024 00:33:39 +0900
Subject: [PATCH 431/638] Create 2024-09-23-day-266.md

---
 _posts/2024-09-23-day-266.md | 59 ++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 _posts/2024-09-23-day-266.md

diff --git a/_posts/2024-09-23-day-266.md b/_posts/2024-09-23-day-266.md
new file mode 100644
index 000000000..44ae6cb20
--- /dev/null
+++ b/_posts/2024-09-23-day-266.md
@@ -0,0 +1,59 @@
+---
+layout: post
+title: (Day 265) T5 model PEFT
+categories: [nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 265!
+A quick summary of today:
+* fine-tuned a t5 model on the movie review dataset
+
+## [Here](https://huggingface.co/divakaivan/t5-large-finetuned-reviewer-kr) is a link to the fine-tuned t5 model
+
+Just a reminder, this project is a similar project to what I will do with my professor when the actual data comes in. I cannot share details for the actual project, so to pre-develop a model and just learn how to fine-tune a model, I created a movie dataset where the input is 50 user reviews, and target is a summary of the review in terms of 5 categories (i.e. character development, story, etc)
+
+Going back to topic, fine-tuning took some time as I had to use the free GPU from google colab. My professor requested me to try to fine-tune a t5 model and then compare its performance to the [llama3 one I fine-tuned](https://huggingface.co/divakaivan/llama3-finetuned-reviewer-kr). 
+
+At the start I was having a hard time because t5 models have 512 token input context and that is significantly lower than the input we want to give it. But then I found a model that for some reasone could take the full input ([this KR t5-large](https://huggingface.co/kfkas/t5-large-korean-P2G)). Looking at the tokenizer-config.json the max_token_input is some random huge number (compared to 512 for the official t5 models), so I kind of doubt the model can understand all the text I am giving it. I fine-tuned, using LoRA, a few models and a sample output is below. It is in Korean, but the tldr is that it repeats categories (you can see Pacing is output 3 times). There can be many reasons for this, including hyperparams, LoRA config, training issues, and also model input context. I guess I just need to continue trying different configs and see if I can get something good. 
+
+```
+영화에 대한 50개의 리뷰를 분석한 결과, 다음과 같이 감정 표현에 대한 평가가 나뉘어집니다. 
+
+**감정 (Emotional)**: 4/10
+- 많은 리뷰에서 영화를 긍정적으로 보았다고 언급된 감정 요소들은 부정적으로 평가되고 있습니다. 이 감정은 영화 자체의 무게감을 감소시키고 있습니다.
+
+**캐릭터 (Characters)**: 3/10
+- 일부는 캐릭터가 답답하다는 비판이 있습니다. 몇몇 리뷰는 캐릭터가 연기가 부족하다고 평가하며, 특히 몇몇은 연기가 형편없다고 비판하기도 했습니다.
+
+**줄거리 (Plot)**: 3/10
+- 여러 리뷰에서 줄거리에 대한 불만이 있습니다. 흥미롭고 재미있는 이야기라는 평가와 부정적인 의견이 공존합니다.
+
+**비주얼 (Visuals)**: 4/10
+- 많은 리뷰에서 비주얼과 관련해서 긍정적인 언급이 있었으며, 특히 카메오가 인상적이었다는 언급이 많았습니다. 시각적인 요소에 대한 호평이 많으며, 몇몇 리뷰에서 화면의 색감이 만족스럽다는 의견도 있었습니다.
+
+**템포 (Pacing)**: 4/10
+- 템포는 지루함을 느꼈습니다. 템포가 빠르다는 의견이 있었지만, 전반적으로 느리고 지루하다는 평가가 많았습니다.
+
+**페이싱 (Pacing)**: 3/10
+- 템포에 대한 긍정적인 의견이 많아, 전반적으로 지루하다는 의견이 많이 보였습니다.
+
+**템포 (Pacing)**: 5/10
+- 페이싱에 대한 비판이 많았습니다. 지루함과 몰입을 방해한다는 의견이 많이 나타났습니다. 일부 리뷰에서 시간이 부족하다는 지적이 있었으며, 지루함과 지루해하는 의견이 존재했습니다.
+
+**비주얼 (Visuals)**: 5/10
+- 많은 리뷰에서 비주얼이 긍정적인 평가를 받았습니다. 일부는 비주얼에 대한 긍정적인 의견도 있었지만, 전체적인 평가는 전반적으로 인상적이지 못했습니다.
+
+종합적으로, 이 영화는 감정적인 요소나 캐릭터, 줄거리가 많이 부족했다는 지적이 많이 있습니다. 그러나 일부 리뷰에서는 재미가 있거나 감동적인 부분이 존재했다고 언급하며, 다양한 부정적 요소들이 존재하고 있습니다.
+
+종합적으로, 이 영화는 비판을 받은 요소가 많으며, 특히 스토리와 템포 부분에서 아쉬움이 있는 영화로 평가받았습니다.
+```
+
+---
+
+I did not do the DE course today, as I wanted to fully concetrate on fine-tuning a t5 model. However, tomorrow I will continue with it. 
+
+As for fine-tuning the model, as it takes a lot of time using the free colab GPU, I might request to use my professor's powerful GPU in the next few days.
+
+That is all for today!
+
+See you tomorrow :)

From 073ade95fbcbd000c46b84af108801b5d41d30b1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 24 Sep 2024 10:14:37 +0900
Subject: [PATCH 432/638] fix day of blog from 265 -> 266

---
 _posts/2024-09-23-day-266.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-09-23-day-266.md b/_posts/2024-09-23-day-266.md
index 44ae6cb20..df49ef4a9 100644
--- a/_posts/2024-09-23-day-266.md
+++ b/_posts/2024-09-23-day-266.md
@@ -1,10 +1,10 @@
 ---
 layout: post
-title: (Day 265) T5 model PEFT
+title: (Day 266) T5 model PEFT
 categories: [nlp,applying-knowledge]
 ---
 
-# Hello :) Today is Day 265!
+# Hello :) Today is Day 266!
 A quick summary of today:
 * fine-tuned a t5 model on the movie review dataset
 

From 1c7262bcbcfda1afd63de2e8a4abbc6f0c60da90 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 24 Sep 2024 23:54:52 +0900
Subject: [PATCH 433/638] Create 2024-09-24-day-267.md

---
 _posts/2024-09-24-day-267.md | 120 +++++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 _posts/2024-09-24-day-267.md

diff --git a/_posts/2024-09-24-day-267.md b/_posts/2024-09-24-day-267.md
new file mode 100644
index 000000000..4134fa5de
--- /dev/null
+++ b/_posts/2024-09-24-day-267.md
@@ -0,0 +1,120 @@
+---
+layout: post
+title: (Day 267) Continuing with the DE course by Joe Reis
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 267!
+A quick summary of today:
+* covered week 1-3 of the final course in the DE specialisation
+
+## Course 4 week 1 continued
+
+### Data Modelling Techniques
+
+![image](https://github.com/user-attachments/assets/08d5d022-3a87-44be-90fb-d99e4b875000)
+
+![image](https://github.com/user-attachments/assets/40fa3a50-36b3-4dd8-9240-4aaba62ecde4)
+
+![image](https://github.com/user-attachments/assets/666fd45b-a889-4a48-a7b3-9570da88f25e)
+
+![image](https://github.com/user-attachments/assets/5bb675d6-4958-4646-b4bd-739a659ff50e)
+
+### From normalised model to Star schema
+
+### Transforming Normalized Data into a Star Schema
+
+In many cases data engineers need to convert normalized data stored in a relational database into a star schema for easier querying, often in preparation for department-specific data marts. This process can follow Kimball's four steps for designing a star schema:
+
+1. Understand Business Needs: begin by identifying which business processes to model in the fact tables and define the grain, for example the level of detail for each row in the fact table. In the example, the grain is set to the individual product items in sales transactions
+  
+2. Identify Facts and Dimensions: based on requirements gathered from a data analyst, dimensions such as stores, items, and dates are identified. For example, the stores dimension includes store details like store_id, name, city, and zip code
+
+3. Create Dimension Tables: surrogate keys are used for dimension tables to uniquely identify each row. Hash functions, such as MD5, can generate surrogate keys from natural keys like store_id and SKU
+
+4. Build the Fact Table: the fact table contains metrics (i.e. quantity sold, item price) and foreign keys to link to dimension tables. For example, the fact table stores store_key, item_key, and date_key alongside transaction details
+
+Once the star schema is built, tools like DBT or AWS Glue can be used to automate transformations. DBT focuses on transforming data within a single warehouse, while AWS Glue offers more flexibility, connecting multiple sources and destinations
+
+The lab for this week is using dbt to model normalised data into a star schema.
+
+There was a short talk between Joe and dbt's co-founder Drew Banin, and Drew said that dbt does not take data from the warehouse, do checks, transform, and put it back in, it just instructs the data warehouse on what to do. So the data never leaves the warehouse. This is something I did not know about dbt and another reason it is so awesome.
+
+### Data Vault (another data modelling technique)
+
+Dan Linstedt introduced it as a different approach to modelling data in a warehouse.
+
+![image](https://github.com/user-attachments/assets/8c921cd4-17a4-4b29-8033-32df36a33867)
+
+![image](https://github.com/user-attachments/assets/c900500a-cb44-4d83-ab11-7eea642b3ad6)
+
+Sample data vault modelled data
+
+![image](https://github.com/user-attachments/assets/47167549-4b43-46ee-b044-a01893df04ef)
+
+### One Big Table
+
+The above approaches were developed in the past when data warehouses were expensive, on premises and heavily resource constrained with tightly coupled compute and storage. While batch data modeling has traditionally been associated with one of these strict approaches more relaxed approaches such as as one big table(obt) are becoming more common.
+
+![image](https://github.com/user-attachments/assets/bf94a096-d16d-4951-b82a-05a2bbac0d3b)
+
+![image](https://github.com/user-attachments/assets/273f93b5-b3c2-4322-8df9-ac7abcfcdfc9)
+
+Joe Reis suggests using a wide table when we have a lot of data that needs more flexibility than a traditional data modeling approach might provide. When it comes to modelling our data, there's no one size fits all solution, so we have to make sure to understand the trade offs between the possible approaches when it comes to flexibility, data integrity, and ease of use by downstream stakeholders to choose the best approach for your use case
+
+### Lab
+
+I will just share the successful dbt run after creating 5 dim tables and the fact table models:
+
+![image](https://github.com/user-attachments/assets/ec296e0c-0be6-40da-8b5e-f036c07d0e64)
+
+## Week 2: Modelling and processing data for ML
+
+![image](https://github.com/user-attachments/assets/87010355-255f-4172-b1b5-3de65cbbd47f)
+
+As a DE, I might not be responsible for the models in the ML pipelines, but for setting up and maintaining the pipelies to serve data that supports these phases.
+
+This week was about data(table/imgage/text) preprocessing and getting it ready for a ML model. 
+
+## Week 3: Batch and Streaming transformations using Spark
+
+<img width="856" alt="image" src="https://github.com/user-attachments/assets/dad6d627-88ba-4872-adff-e6e48eebab92">
+
+<img width="904" alt="image" src="https://github.com/user-attachments/assets/4f43f273-81de-4559-8d62-66358400d017">
+
+**Hadoop**
+
+* integrates compute and persistent storage via the HDFS
+
+**Apache Spark**
+
+Computing engine designed for processing large datasets
+
+* retains intermediate results in memory
+* limits disk I/O interactions, enabling faster computation
+
+The graded assignment this week was to use EMR - AWS' managed hadoop cluster environment to do some data transformations with PySpark.
+
+### Technical considerations
+
+<img width="934" alt="image" src="https://github.com/user-attachments/assets/043483ae-750b-478e-a717-8bbfb07b9587">
+
+<img width="933" alt="image" src="https://github.com/user-attachments/assets/3202fbf5-6671-4d42-b510-5bbe118e2e76">
+
+### 2nd lab for this week for streaming practice
+
+<img width="895" alt="image" src="https://github.com/user-attachments/assets/c4d1393f-1c87-4bee-b3a0-5a52031e1410">
+
+* Debezium: Distributed platform for CDC, it connects to the source system and captures the changes based on the configuration one setup
+* Apache Zeppelin: Web-based notebook environment, it allows us to run Flink in a notebook alongside other possible interpreters
+* Apache Flink: Distributed processing engine for data streams, you will use it to connect to the Kafka topics and update the target database accordingly
+
+There were some new cool looking technologies in this lab but I should not share more. 
+
+---
+
+There is only week 4 left. This is the final week of the whole DE specialisation. However, from the deeplearning.ai online blog I saw that there are problems with the final assignments and they will get fixed (hopefully) by Friday. So until then I guess I will continue with some other study items on my list.
+
+That is all for today!
+
+See you tomorrow :)

From 6a83ebaaa8b081e88e64b19c4790e7e26c511558 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 25 Sep 2024 22:28:37 +0900
Subject: [PATCH 434/638] Create 2024-09-25-day-268.md

---
 _posts/2024-09-25-day-268.md | 191 +++++++++++++++++++++++++++++++++++
 1 file changed, 191 insertions(+)
 create mode 100644 _posts/2024-09-25-day-268.md

diff --git a/_posts/2024-09-25-day-268.md b/_posts/2024-09-25-day-268.md
new file mode 100644
index 000000000..4305f4998
--- /dev/null
+++ b/_posts/2024-09-25-day-268.md
@@ -0,0 +1,191 @@
+---
+layout: post
+title: (Day 268) Going back to some basics, math, and neo4j
+categories: [theory,math,statistics]
+---
+
+# Hello :) Today is Day 268!
+A quick summary of today:
+* continued with Graph Data Modelling Fundamentals
+* decided to check UC Berkely's free Intro to ML course
+* started casually watching a lecture on the math behind neural nets
+
+
+Today I did a bit of a few things. 
+
+## [Continue with Graph Data Modelling Fundamentals](https://graphacademy.neo4j.com/courses/modeling-fundamentals/3-defining-relationships/4-c-creating-relationships/)
+
+### Exercises to create new relationships
+
+```cypher
+MATCH (sandy:User {name: 'Sandy Jones'})
+MATCH (clinton:User {name: 'Clinton Spencer'})
+MATCH (apollo:Movie {title: 'Apollo 13'})
+MATCH (sleep:Movie {title: 'Sleepless in Seattle'})
+MATCH (hoffa:Movie {title: 'Hoffa'})
+MERGE (sandy)-[:RATED {rating:5}]->(apollo)
+MERGE (sandy)-[:RATED {rating:4}]->(sleep)
+MERGE (clinton)-[:RATED {rating:3}]->(apollo)
+MERGE (clinton)-[:RATED {rating:3}]->(sleep)
+MERGE (clinton)-[:RATED {rating:3}]->(hoffa)
+```
+
+Here, I created new 'user rated movie' relationships
+
+### Testing with Instance Model
+
+![image](https://github.com/user-attachments/assets/c1463031-031a-4eca-a974-b659dd831410)
+
+I did some basic queries like who acted where, who directed what, etc - with such queries we can do basic tests on our models
+
+### Refactoring our graph
+
+* The graph as modeled does not answer all of the use cases
+
+* A new use case has come up that you must account for in your data model
+
+* The Cypher for the use cases does not perform optimally, especially when the graph scales
+
+**Steps for refactoring**
+
+To refactor a graph data model and a graph we must:
+
+* Design the new data model
+
+* Write Cypher code to transform the existing graph to implement the new data model
+
+* Retest all use cases, possibly with updated Cypher code
+
+We can use the keyword `PROFILE` to get a query plan just like in SQL - `EXPLAIN`.
+
+![image](https://github.com/user-attachments/assets/66e9f974-d8ca-46e9-8372-79529b08d177)
+
+After refactoring, it is important to retest all queries that were affected and make sure they return the expected results.
+
+![image](https://github.com/user-attachments/assets/218ae664-04f3-44f6-a14e-6bc305dd9ca1)
+
+## Berkley University's Intro to ML is available for free ?!
+
+That is another top uni in the world that publishes its ML course for free and it is mindblowing. It is amazing. And I decided to start doing it. It is the 2024 Fall version so it is ongoing *this* semester. 
+
+**The lecture videos are available, lecture slides are available, homeworks are available, homework solutions are available, the suggested book to read is available too. Everything!**
+
+### [Here is the link](https://eecs189.org/)
+
+I decided to check it out. 
+
+[CS109](https://www.youtube.com/watch?v=2MuDZIAzBMY&list=PLoROMvodv4rOpr_A7B9SriE_iZmkanvUg&ab_channel=StanfordOnline) Probability for Software Engineers is a legendary course for me in terms of a deep, going back to bare basics intro that sets you up to learn ML. So I guess this can count as a next step after Stanford's CS109 and a competitor for Stanford's CS229 - Intro to ML taught by Andrew Ng.
+
+## Lecture 1: Intro
+
+![image](https://github.com/user-attachments/assets/fee72df9-c6c2-40c3-8bbe-59b04af4e2b5)
+
+Classification
+
+- linear classifier (logistic regression)
+- knn
+
+Neural networks - composing simple (logistic) regression functions over and over again
+
+![image](https://github.com/user-attachments/assets/faded91d-8e27-4e28-a348-f3a4bed90710)
+
+![image](https://github.com/user-attachments/assets/b8763541-a422-435f-ace6-b367aff624f6)
+
+This is just the intro so it is not directly jumping into neural nets. The next lectures cover MLE, Guassians, linear regression, classification, grad. descent - all crucial basics to know before the neural net. 
+
+### Reading 1.1 to 1.2.4 in [Bishop & Bishop](https://www.bishopbook.com/)
+
+The impact of DL
+
+* medical diagnosis
+* protein structure understanding and discovery
+* image synthesis - generative AI
+* LLMs
+* linear models - predict the target for some input x; y = (x, w)
+* error functions - a measure for the misfit between the function y, for any given value of w, and x. A popular choise is:
+
+![image](https://github.com/user-attachments/assets/5b89884d-29e4-4e50-98bf-6a24f85ab89a)
+
+
+## Lecture 2: MLE
+
+![image](https://github.com/user-attachments/assets/d5a5c43d-1c55-430b-afb9-6ab0d86e73f5)
+
+![image](https://github.com/user-attachments/assets/3ab1031a-c96d-48e3-8e31-289eb9097199)
+
+![image](https://github.com/user-attachments/assets/3af65659-edfd-48b8-adbc-ffcf6784609c)
+
+![image](https://github.com/user-attachments/assets/cde8eae6-997a-449d-8db8-017b9b030ca6)
+
+Properties of MLE
+
+* consistency: as we get more and mroe data (drawn from one distribution in our family), then we converge to estimating the true value of theta for D
+* statistically efficient: making good use of the data ('least variance' param estimates)
+* the value of p(D|theta) is invariant to re-parameterisation
+* MLE can still yield a parameter estimate even when the data were not generated from that family
+
+![image](https://github.com/user-attachments/assets/32e0740b-6e71-4f02-ad89-0c9cec67cbc5)
+
+![image](https://github.com/user-attachments/assets/f8fd5ec4-569d-4224-a8c7-907750cd4a37)
+
+![image](https://github.com/user-attachments/assets/f4f0b954-779d-4e45-97df-bd85185dc006)
+
+![image](https://github.com/user-attachments/assets/9d5bf7e6-1392-448d-9158-b345b0d1b088)
+
+### Readings from Bishop & Bishop
+
+* 2-2.1.2 (rules of probability: sum, product)
+
+There are two kinds of uncertainty:
+
+1. systematic: because we only get to see datasets of finite size
+2. intrinsic/stochastic (noise): noise arises because weare able to observe only partial information about the world, so to reduce this we need to gather different kinds of data
+
+Sum rule: p(X) = ∑p(X, Y)
+
+Product rule: p(X, Y) = p(Y|X)p(X)
+
+* 2.1.6 (independent RVs)
+
+If the joint distribution of two variables factorises into the product of the marginals - p(X,Y) = p(X)p(Y) then X and Y are said to be independent
+
+* 2.2-2.2.1 (probability densities in continuous spaces)
+
+![image](https://github.com/user-attachments/assets/8028431d-a42d-4589-9ac0-6d6ab2a01283)
+
+If x and y are two real, variables, then the sum and product rules take the form:
+
+![image](https://github.com/user-attachments/assets/bb2b0772-e3a6-43e2-a9a6-92b4184d19c9)
+
+* 2.3-2.3.2 (univariate Gaussian, likelihood)
+* 3-3.1.3 (Bernoulli, binomial, multinomial, MLE)
+
+These parts are pretty much the same as in the lecture pictures. But still nicely and explained in detail.
+
+
+## [The Complete Mathematics of Neural Networks and Deep Learning](https://youtu.be/Ixl3nykKG9M)
+
+I think this 16 year old student is faily famous for this lecture video on the math behind neural nets. So I decided to relax and just watch it. It is great and with examples which is crucial. 
+
+It is material that I know and am comfortable with. Thanks to the almighty Andrej Karpathy and his forward/backprop ninja video. But I was curious how this student teaches the math so I started watching and it seems great. 
+
+Today I just covered:
+
+**PART I - Introduction**
+
+1.1 Prerequisites
+1.2 Agenda 
+1.3 Notation 
+1.4 Big Picture
+1.5 Matrix Calculus Review
+    1.5.1 Gradients
+    1.5.2 Jacobians
+    1.5.3 New Way of Seeing the Scalar Chain Rule
+    1.5.4 Jacobian Chain Rule
+
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 54e0349a1158f35baaf5d99bdebc6145ced3dbbf Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 27 Sep 2024 01:06:48 +0900
Subject: [PATCH 435/638] Create 2024-09-26-day-269.md

---
 _posts/2024-09-26-day-269.md | 47 ++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 _posts/2024-09-26-day-269.md

diff --git a/_posts/2024-09-26-day-269.md b/_posts/2024-09-26-day-269.md
new file mode 100644
index 000000000..919865169
--- /dev/null
+++ b/_posts/2024-09-26-day-269.md
@@ -0,0 +1,47 @@
+---
+layout: post
+title: (Day 269) The math behind neural nets + trying the capstone project from DL.AI's DE specialisation
+categories: [theory,math,data-eng]
+---
+
+# Hello :) Today is Day 269!
+A quick summary of today:
+* [did my 1st stream](https://youtu.be/dvqmZHNLtp4)
+* attempted the capstone project (part 1) from the DE specialisation
+
+## Firstly, about my first stream
+
+There is content that I want to watch/rewatch and not take notes. But just pasting a link here (in the blog) to the thing I watched seems too little. If I do it on stream, it can count as a bit more content (and as proof that I watched/covered that content). But also if any other learners join my stream we can have a chat about whatever I am doing. Today I spent some time setting up my stream and camera
+
+![image](https://github.com/user-attachments/assets/5ae4057d-e3fb-46dc-b124-686fa592ae5a)
+
+And I want to stream on youtube. I could stream on twitch, but I do not associate twitch with studying, whereas on youtube I have seen many people do study with me livestreams. Unfortunately, I had to request access and it takes 24hrs to get it. Today I requested it and tomorrow I will get it, and today I streamed on twitch and uploaded the vod to youtube. For now, I am planning to do 1-2 hour streams Monday to Thursday from 6pm KST. 
+
+Today I finished [The Complete Mathematics of Neural Networks and Deep Learning](https://youtu.be/Ixl3nykKG9M). I watched most of it earlier in the day, and on stream I watched the last 1 hour.
+
+Here are some resources shared which seem interesting: 
+
+* [The Matrix Calculus You Need For Deep Learning](https://arxiv.org/pdf/1802.01528)
+* [How the backpropagation algorithm works](http://neuralnetworksanddeeplearning.com/chap2.html)
+
+## DE specialisation Capstone project part 1
+
+I am at course 4 on Joe Reis' DE specialisation on Coursera. The only things left for me to complete the course are the last 2 capstone projects. I have been waiting for an error to be fixed by the Coursera engineers and today I saw that it got fixed. The error was related to access when running terraform apply so that I can run glue jobs for extracting and transforming data. That part was fixed and my grade is now:
+
+<img width="442" alt="image" src="https://github.com/user-attachments/assets/2c319d65-8169-4e33-ac06-87cbdb56e2fc">
+
+For the parts I have 0, I wrote a [post](https://community.deeplearning.ai/t/capstone-project-part-1-grading-optional-chapter/697934/11) as I genuinly don't understand where the error could come from as I have followed the instructions carefully. Other people are stuck at the same place as me, and it seems the problem might come from the auto-grader (but we will see what the engineers say).
+
+## New books from the library
+
+Today I decided to check the uni library for some books, and it turns out that two gems were available to borrow
+
+![image](https://github.com/user-attachments/assets/38fba151-d2a5-4e09-9abb-0cc19df20389)
+
+I can't wait to read them. I really prefer having physical books than reading them online.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From b81adac15f6a15e2aa45ea48d87b0fbb9e01373c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 27 Sep 2024 22:41:10 +0900
Subject: [PATCH 436/638] Create 2024-09-27-day-270.md

---
 _posts/2024-09-27-day-270.md | 152 +++++++++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)
 create mode 100644 _posts/2024-09-27-day-270.md

diff --git a/_posts/2024-09-27-day-270.md b/_posts/2024-09-27-day-270.md
new file mode 100644
index 000000000..467a38967
--- /dev/null
+++ b/_posts/2024-09-27-day-270.md
@@ -0,0 +1,152 @@
+---
+layout: post
+title: (Day 270) DE course by Joe Reis - completed
+categories: [theory,math,statistics,applying-knowledge,data-eng]
+---
+
+# Hello :) Today is Day 270!
+A quick summary of today:
+* completed [DeepLearning.AI Data Engineering Professional Certificate](https://www.coursera.org/professional-certificates/data-engineering) but will review it on stream next week
+* read [The matrix calculus you need for deep learning ](https://arxiv.org/abs/1802.01528)
+* started reading [DL by Bishop & Bishop](https://www.bishopbook.com/)
+
+## Firstly, the DE course by Joe Reis 
+
+<img width="1384" alt="image" src="https://github.com/user-attachments/assets/b56a6e4c-7ffa-4e04-83f1-4e4051a8c9ab">
+
+Today on DeepLearning.AI's community blog I found that the errors associated with the last capstone projects were fixed so I went in and completed them. 
+
+<img width="890" alt="image" src="https://github.com/user-attachments/assets/c5ae0cb0-e005-414c-9077-8292f9318fd3">
+
+🥳🥳🥳
+
+This course was amazing. Very in-depth on the way of thinking as a DE but also provides a good deal of practice through AWS' many different tools. 
+
+At the end Joe shared his opinion on where the DE field is going and how the role of a DE might look:
+
+<img width="901" alt="image" src="https://github.com/user-attachments/assets/d9e1128a-5a77-484a-9c0f-689ce6d1f22a">
+
+Next week, on stream, I might review what I learned in depth so that I can write a better review of the course.
+
+## [The matrix calculus you need for deep learning ](https://arxiv.org/abs/1802.01528)
+
+I got this paper from yesterday's lecture and it is 100% awesome! I printed it and made a small booklet, and read it.
+
+![image](https://github.com/user-attachments/assets/6939962b-1075-4f69-9094-55cd1915d3a3)
+
+Definitely in my 'to-recommend' sources on math for DL. 
+
+## DL by Bishop & Bishop
+
+Today when I went to the library I read the above paper, and after B&B's book on DL. I did not record myself reading the paper, but after I finished it I realised I better turn on a timelapse video on my phone.
+
+[Here is the video on my youtube channel](https://www.youtube.com/shorts/WSD_SAZRIek)
+
+Today I covered:
+
+### 2 Probabilities
+
+#### 2.1 The Rules of Probability
+- 2.1.1 A medical screening example
+- 2.1.2 The sum and product rules
+- 2.1.3 Bayes’ theorem
+- 2.1.4 Medical screening revisited
+- 2.1.5 Prior and posterior probabilities
+- 2.1.6 Independent variables
+
+#### 2.2 Probability Densities
+- 2.2.1 Example distributions
+- 2.2.2 Expectations and covariances
+
+#### 2.3 The Gaussian Distribution
+- 2.3.1 Mean and variance
+- 2.3.2 Likelihood function
+- 2.3.3 Bias of maximum likelihood
+- 2.3.4 Linear regression
+
+#### 2.4 Transformation of Densities
+- 2.4.1 Multivariate distributions
+
+#### 2.5 Information Theory
+- 2.5.1 Entropy
+- 2.5.2 Physics perspective
+- 2.5.3 Differential entropy
+- 2.5.4 Maximum entropy
+- 2.5.5 Kullback–Leibler divergence
+- 2.5.6 Conditional entropy
+- 2.5.7 Mutual information
+
+#### 2.6 Bayesian Probabilities
+- 2.6.1 Model parameters
+- 2.6.2 Regularization
+- 2.6.3 Bayesian machine learning
+
+### 3 Standard Distributions
+
+#### 3.1 Discrete Variables
+- 3.1.1 Bernoulli distribution
+- 3.1.2 Binomial distribution
+- 3.1.3 Multinomial distribution
+
+#### 3.2 The Multivariate Gaussian
+- 3.2.1 Geometry of the Gaussian
+- 3.2.2 Moments
+- 3.2.3 Limitations
+- 3.2.4 Conditional distribution
+- 3.2.5 Marginal distribution
+- 3.2.6 Bayes’ theorem
+- 3.2.7 Maximum likelihood
+- 3.2.8 Sequential estimation
+- 3.2.9 Mixtures of Gaussian
+
+#### 3.3 Periodic Variables
+- 3.3.1 Von Mises distribution
+
+#### 3.4 The Exponential Family
+- 3.4.1 Sufficient statistics
+
+#### 3.5 Nonparametric Methods
+- 3.5.1 Histograms
+- 3.5.2 Kernel densities
+- 3.5.3 Nearest-neighbours
+
+### 4 Single-layer Networks: Regression
+
+#### 4.1 Linear Regression
+- 4.1.1 Basis functions
+- 4.1.2 Likelihood function
+- 4.1.3 Maximum likelihood
+- 4.1.4 Geometry of least squares
+- 4.1.5 Sequential learning
+- 4.1.6 Regularized least squares
+- 4.1.7 Multiple outputs
+
+#### 4.2 Decision theory
+
+#### 4.3 The Bias–Variance Trade-off
+
+One thing I was thinking when reading the KL-convergence part was why are we introducing this at the start of the book when the next thing we will talk about is regression in a single neuron. KL-convergence is not really applicable to regression. From a beginner's point of view, reading about KL-convergence so early might be intimidating, and a better time to include it might be when it's actually used, like before variational autoencoders. Introducing it in a more relevant context could help learners better understand its significance and application. But as it is part of Information Theory, it is also understandable to include it in that part of the book. (And I have seen this pattern in other books as well but it's fine, just commenting)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From a5f4d55dcafaa7a31fea43aac883a5de12b6d3ed Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 28 Sep 2024 22:48:27 +0900
Subject: [PATCH 437/638] Create 2024-09-28-day-271.md

---
 _posts/2024-09-28-day-271.md | 186 +++++++++++++++++++++++++++++++++++
 1 file changed, 186 insertions(+)
 create mode 100644 _posts/2024-09-28-day-271.md

diff --git a/_posts/2024-09-28-day-271.md b/_posts/2024-09-28-day-271.md
new file mode 100644
index 000000000..fc9cbd11e
--- /dev/null
+++ b/_posts/2024-09-28-day-271.md
@@ -0,0 +1,186 @@
+---
+layout: post
+title: (Day 271) DE - insight and advice from industry experts
+categories: [theory,gnn,data-eng]
+---
+
+# Hello :) Today is Day 271!
+A quick summary of today:
+* conversations with industry experts from the DE course by Joe Reis
+* starting [Graph Algorithms for Data Science](https://learning.oreilly.com/library/view/graph-algorithms-for/9781617299469/) by Tomaz Bratanic
+
+At the end of the DE specialisation course, there are 3 videos (total ~1hr) from industry experts that Joe Reis is interviewing that I did not realise it was there.
+
+## Conversation with [Zach Wilson](https://www.dataexpert.io/about)
+
+Zach Wilson, a seasoned data engineer turned entrepreneur, shared his thoughts on the evolving landscape of data engineering, the challenges faced by newcomers, and how the role is expected to change in the coming years. Zach, who has worked with tech giants like Facebook, Netflix, and Airbnb, offered valuable insights from his decade-long career.
+
+### From Engineer to Entrepreneur
+
+Zach’s career began in big tech, where he developed his skills as a data engineer. After nearly a decade working on cutting-edge projects, he transitioned from engineering roles to entrepreneurship. He now runs a bootcamp called DataExpert.io, aimed at helping aspiring data engineers break into the field. His career shift highlights the importance of continuous learning and adapting to new challenges in the tech industry.
+
+### Breaking into Data Engineering
+
+One of the key points Zach emphasized is the difficulty of landing a junior data engineering role. Unlike software engineering or data analysis, entry-level positions in data engineering are relatively rare. This is primarily because companies often prefer experienced professionals to manage critical data pipelines. Zach recommends starting in adjacent roles, such as software engineering or data analysis, to build relevant skills. He shared his own experience of starting in a software engineering role focused on automation, which later helped him transition into data engineering.
+
+### The Skill Set of a Data Engineer
+
+Zach highlighted the importance of mastering several key skills to succeed as a data engineer: proficiency in Python, Bash, SQL, and cloud technologies like BigQuery, Airflow, and Spark. He also stressed the importance of understanding Linux commands and being able to integrate various services effectively. However, beyond technical skills, Zach believes that knowing when not to build a pipeline is equally important—a sign of a mature and experienced data engineer.
+
+### Knowing When You’ve Made It
+
+When asked how one knows they’ve become a good data engineer, Zach provided an interesting perspective. It's not just about building pipelines efficiently; it’s also about understanding the bigger picture. A good data engineer thinks critically about why a pipeline is needed and its impact on the business. Zach recounted his own shift in mindset at Airbnb, where he learned to say no to unnecessary tasks, allowing him to focus on more valuable work and avoid burnout.
+
+### Career Advice for Aspiring Data Engineers
+
+Zach offered valuable advice for those just starting out. He advised against the common belief that “your work will speak for itself.” Instead, he stressed the importance of marketing yourself and seeking feedback to improve. He also encouraged learning in-demand technologies like Apache Spark early in one’s career, as this can open up more opportunities and make a data engineer more competitive in the job market.
+
+### Keeping Skills Sharp and Growing in Your Career
+
+To stay relevant in data engineering, Zach suggested continuously pushing the boundaries of the three V’s—volume, velocity, and variety. He shared his experience at Netflix, where he took on projects outside his job responsibilities to work on larger, more complex pipelines. This kind of initiative not only enhances one’s skills but also positions a data engineer as a valuable asset within a company.
+
+### The Future of Data Engineering
+
+Looking ahead, Zach sees data engineering evolving in two significant ways. First, he believes that the roles of data engineers and backend engineers will increasingly merge, particularly in environments where service owners are responsible for their data. This hybrid role, which he refers to as "Software Engineer, Data," is likely to become more common. Second, Zach predicts a rise in the importance of unstructured data, driven by advancements in AI and machine learning. However, he cautions that this will require new techniques to ensure data quality, especially in the face of challenges like LLM hallucination.
+
+## Conversation with Carly Taylor
+
+Carly Taylor, a machine learning engineer at Activision working on Call of Duty, shared her remarkable journey into data science, her experiences in the field, and valuable career advice for those aspiring to enter the tech world. Carly’s story is one of determination, continuous learning, and adapting to new challenges—a journey that took her from a background in chemistry to the cutting-edge world of machine learning.
+
+### From Chemistry to Data Science
+
+Carly's entry into data science is a testament to the diverse paths one can take into this field. With a graduate degree in chemistry, Carly initially found it challenging to secure a job in her domain. However, as many of her peers began moving into the emerging field of data science, particularly in FinTech, she saw an opportunity to apply her quantitative skills in a new way. Intrigued by the potential and the exciting challenges data science offered, Carly made the leap, beginning her career as a marketing analyst despite having no prior knowledge of marketing.
+
+This unconventional start highlights a key point Carly emphasized: sometimes, breaking into data science means leveraging the skills you have, even if you're not a domain expert. Her technical background helped her land the role, and she learned the necessary marketing knowledge on the job. If she could do it over again, Carly noted she might have taken some business classes to ease the transition, but ultimately, her story illustrates the power of on-the-job learning and adaptability.
+
+### Life as a Machine Learning Engineer at Activision
+
+Today, Carly works on the machine learning team at Activision, focusing on security for Call of Duty. Her role involves a mix of tasks, but much of her time, like many in similar positions, is spent on exploratory data analysis (EDA). Carly estimates that about 70% of her team's time is dedicated to EDA, with the remaining 30% split between modeling and production tasks. The work is hands-on, data-driven, and requires constant adaptation to new challenges, especially in a fast-paced environment like gaming.
+
+Interestingly, Carly explained that at her company, the lines between different roles—like data engineering, software engineering, and machine learning—are often blurred. This fluidity allows her team to be flexible, with engineers frequently stepping into roles that might not be strictly defined by their job titles. This versatility is a crucial skill in modern tech environments, where the ability to wear multiple hats can significantly enhance a team's productivity and innovation.
+
+### Understanding the Difference: Data Engineering vs. Machine Learning Engineering
+
+Carly offered her perspective on the distinction between data engineering and machine learning engineering, a topic often debated in tech circles. She explained that while data engineers focus on ensuring data integrity, availability, and pipeline reliability, machine learning engineers are tasked with understanding the data in a business context and building predictive models. The two roles complement each other, with data engineers ensuring that the data is clean and usable, while machine learning engineers apply that data to solve complex problems.
+
+### Climbing the Ladder: From Individual Contributor to Leader
+
+Carly’s transition from individual contributor to team leader was not a planned move, but rather a response to a gap in leadership that needed filling. Her advice to those considering a move into management is to be open to stepping up when the situation calls for it, rather than feeling pressured to follow a strict career plan. Leadership, in her experience, often comes from recognizing a need and having the courage to fill it.
+
+### Advice to Her Younger Self and to Learners
+
+Reflecting on her career, Carly shared the advice she would give her younger self: give yourself grace. Early in her career, she felt immense pressure to keep up with every new development in data science, leading to burnout. Over time, she realized the importance of focusing on a specialty, mastering it, and letting go of the need to know everything about everything. Carly introduced the concept of a T-shaped skill set—a broad understanding of many areas with deep expertise in one. This, she believes, is key to success in data science.
+
+### Standing Out in a Competitive Job Market
+
+In today’s competitive job market, Carly emphasized the importance of going the extra mile, especially when applying for jobs. One emerging trend she discussed is the use of video submissions during the application process. While many candidates might resist this, Carly sees it as an opportunity to stand out. By putting in a bit more effort—whether it’s dressing well, speaking naturally, or ensuring good lighting—you can differentiate yourself from the majority who might approach these tasks with minimal effort.
+
+## Conversation with Ben Rogojan
+
+In a recent interview, Ben Rogojan, widely known as the Seattle Data Guy, sat down to discuss his journey and insights into the world of data engineering. Ben, who has spent nearly a decade in the data industry, shared how he initially started with a strong focus on data science, influenced by the popularity of the field around 2012 to 2015. He adjusted his college courses to align more with bioinformatics and statistics, preparing for a career in data science. However, over time, Ben discovered a preference for the software and programming aspects of data work, which led him to accidentally stumble upon data engineering. His first official role as a data engineer came at a healthcare analytics startup, where he honed his skills in SQL, Python, automation, and data warehousing. This role set the foundation for his career, eventually leading him to work at Facebook before transitioning into consulting within the data engineering and data infrastructure space.
+
+### Early Career and Healthcare Analytics
+
+During the interview, Ben discussed the nature of his work at the healthcare analytics company, where he was responsible for managing and standardizing large datasets from various insurance providers. The data, primarily healthcare claims, required meticulous standardization due to the differing formats in which it was received, even though the content was generally similar. Ben explained how this experience taught him about the various ways data can be formatted and the challenges associated with managing such complex information. His work at the company also involved building products focused on fraud detection and monitoring opioid distribution, all while working with tools like PowerShell and SQL Server, which he described as unglamorous but essential for the job.
+
+### Transition to Facebook and Mature Data Infrastructure
+
+Ben’s move to Facebook marked a significant shift in his career, where he encountered a more mature data infrastructure. He noted that while Facebook’s data systems were advanced and often seen as unchallenging by some, the real challenge lay in identifying valuable problems to solve and effectively communicating within the organization. Ben emphasized the importance of not just being technically proficient but also understanding and addressing the business's needs. He shared a valuable lesson from his time at Facebook: instead of merely completing tasks, data engineers should focus on identifying recurring issues and finding ways to eliminate them, thereby adding more value to the business.
+
+### Advice for Aspiring Data Engineers
+
+When asked about advice for aspiring data engineers, Ben recommended starting with a solid foundation in technical skills such as SQL, Python, and data warehousing. However, he also stressed the importance of understanding the business context of these technologies. Ben encouraged engineers to ask critical questions about the purpose of the systems they are building and the real impact on the business. He acknowledged the challenge of learning about the business side of things, especially early in one's career, but emphasized that it becomes increasingly important as engineers advance in their roles.
+
+### Future of Data Engineering and Industry Insights
+
+Looking to the future, Ben expressed hope that the industry would continue to focus on the fundamentals of data management and quality, especially as interest in AI continues to grow. He pointed out the challenges companies face in managing their diverse data infrastructures and the importance of implementing reliable and usable datasets over the long term. Ben remains optimistic that with a renewed focus on these basics, the industry can better leverage AI and other advanced technologies.
+
+---
+
+Actually, I listened to these talks in the morning, but after that I saw that for one of my courses - Research Ethics and Methodology, I had a lot of assignments to do because I joined a bit late and I wanted to cover the recorded lectures and mini-assignments. So lunch-evening I was watching lectures and doing the mini-assignments related to some research writing and methodology basics. One of the mini-assignments was 'Why did I choose the title for my thesis?'. And my title at the moment is 'Using Video Generation Models over Graph Neural Networks for Taxi OD Demand Matrix Prediction', and I had to write something like: *This title highlights key topics like video generation models, graph neural networks, and taxi OD demand matrices, making it easier for the paper to be found in search engines and giving readers a clear idea of its content right away. A well-crafted title can also capture a reader's interest, helping them decide whether the paper is worth their time. Additionally, this title reflects an innovative approach to tackling real-world traffic prediction problems by comparing video generation techniques with advanced graph neural networks.*
+
+## [Graph Algorithms for Data Science](https://learning.oreilly.com/library/view/graph-algorithms-for/9781617299469/) by Tomaz Bratanic
+
+This is the 1st thing in the book
+
+<img width="589" alt="image" src="https://github.com/user-attachments/assets/fa1681ed-269a-4df7-a02a-dcbfd82cd2d1">
+
+So there is a Graph DS role now ~ cool
+
+This book has been on my to-read list for a bit so I'm excited to read it in the coming days (maybe even on stream)
+
+## Chapter 1 Graphs and network science: An introduction
+
+![image](https://github.com/user-attachments/assets/445402c1-fa04-48d6-819d-f247129b724d)
+
+If you have ever done any analysis, you have probably used a table representation of data, like an Excel spreadsheet or SQL database. Additionally, if you are dealing with large numbers of documents, you might have used parquet format or JSON-like objects to represent the data.
+
+Sometimes, the relationships are explicitly specified between data points, like a person and their relationships with friends; and in others, the relationships may be indirect or implicit, like when there is a correlation between data points that could be calculated. Whether relationships are explicit or implicit, they contain the additional context of data points that can significantly improve the analysis output. The image below shows a small instance of a graph, which consists of five nodes connected by five relationships. Nodes or vertices can be used to represent various real-world entities, such as persons, companies, or cities; however, you can also use nodes to depict various concepts, ideas, and words.
+
+![image](https://github.com/user-attachments/assets/d30ffd12-e33e-4adb-9e9d-8cdee7f10caa)
+
+### Understanding data through relationships
+
+Graph visualizations are valuable for gaining insights but can lose effectiveness with large datasets. When faced with vast amounts of data, graph algorithms can be utilized to derive insights or identify significant parts of the graph for further exploration through visualization. For example, in a large social network like Facebook, visualizing the entire structure is challenging, but graph algorithms can reveal key user groups or important nodes that influence information flow.
+
+Graph algorithms, such as community detection, help identify groups of closely connected nodes, similar to friend groups in social networks. These communities can be used for content recommendations, marketing strategies, or understanding the graph's structure. Centrality algorithms identify the most important nodes, like social media influencers, who control information flow between different communities.
+
+![image](https://github.com/user-attachments/assets/a7eac9e0-c35a-4f8c-b4f2-44704a7ae4b0)
+
+These insights can enhance machine learning models by incorporating information about node positions and roles, improving predictive accuracy. Graph-based machine learning leverages these connections, using features that describe a node's network role, aiding in tasks like node classification, regression, and link prediction.
+
+![image](https://github.com/user-attachments/assets/6725b596-a073-4990-851a-6ea8553dd6c5)
+
+### How to spot a graph-shaped problem
+
+#### Self-referencing relationships
+
+The first scenario deals with self-referencing relationships between entities of the same type. In relational databases, a self-referencing relationship occurs between data points within the same table. The self-referencing relationships can be modeled as a single node and relationship type in a graph.
+
+![image](https://github.com/user-attachments/assets/e302407c-a756-47c1-8158-2a1e0e3a8960)
+
+Graph algorithms are domain agnostic, meaning they can be applied across various fields regardless of what the nodes represent—be it people, services, or other entities. This versatility allows the same algorithms to be used in diverse scenarios, such as:
+
+- **Bank transaction networks** for fraud detection.
+- **Service dependency graphs** to assess vulnerability spread and defense against cyberattacks.
+- **Supply chain optimization**
+- **Social media analysis**
+- **Managing complex telecommunication networks**
+- **Cybersecurity**
+
+These applications highlight the broad utility of graph algorithms in optimizing and securing complex systems.
+
+#### Pathfinding networks
+
+Another fairly common graph scenario is discovering paths or routes between entities or locations - like finding the most optimal route when travelling.
+
+Traditional databases can struggle with queries that require traversing an unknown number of relationships between nodes, leading to complex and computationally expensive operations. By treating your data as a graph, we can more efficiently manage these challenges.
+
+Finding optimal routes using graph algorithms is beneficial in several scenarios, including:
+
+- **Logistics and routing**
+- **Infrastructure management**
+- **Identifying optimal paths for making new contacts**
+- **Payment routing**
+
+Using a graph-based approach can simplify and optimize these processes.
+
+#### Bipartite graphs
+
+Another compelling use case for graphs is examining indirect or hidden relationships. The graphs in the previous two scenarios had only one node type. On the other hand, a bipartite graph contains two node types and a single relationship type.
+
+![image](https://github.com/user-attachments/assets/60180b09-4f01-436c-83b2-5d65f11e5f86)
+
+#### Complex networks
+
+The last example is a complex network with many connections between various entities; a biomedical graph is one such scenario
+
+![image](https://github.com/user-attachments/assets/6c5c8721-edbe-4a03-8389-386c862e5e12)
+
+Explaining all the medical terminology behind medical entities and their relationships could be the subject of a whole book. However, we can observe that biomedical concepts are highly interconnected and therefore an excellent fit for a graph model. Biomedical graphs are popular for representing existing knowledge and making predictions about new treatments, side effects, and more.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From faf0070b83f847aeedff91ba29cb574cd32e750a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 29 Sep 2024 23:07:27 +0900
Subject: [PATCH 438/638] Create 2024-09-29-day-272.md

---
 _posts/2024-09-29-day-272.md | 211 +++++++++++++++++++++++++++++++++++
 1 file changed, 211 insertions(+)
 create mode 100644 _posts/2024-09-29-day-272.md

diff --git a/_posts/2024-09-29-day-272.md b/_posts/2024-09-29-day-272.md
new file mode 100644
index 000000000..75a1c3150
--- /dev/null
+++ b/_posts/2024-09-29-day-272.md
@@ -0,0 +1,211 @@
+---
+layout: post
+title: (Day 272) A bit more of Graph Algorithms for Data Science
+categories: [theory,gnn]
+---
+
+# Hello :) Today is Day 272!
+A quick summary of today:
+
+## Ch. 2 Representing network structure: Designing your first graph model
+
+### 2.1 Graph terminology
+
+Directed vs. undirected graph ((A) represents a directed graph; (B) represents an undirected graph)
+
+![image](https://github.com/user-attachments/assets/7e26e185-38b5-45d1-ad82-d582b3e459b7)
+
+Weighted vs. unweighted graphs ((A) represents an unweighted graph; (B) represents a weighted graph)
+
+![image](https://github.com/user-attachments/assets/3028fd91-86b3-43b7-a835-be608683b035)
+
+Bipartite vs. monopartite graphs ((A) represents a monopartite graph; (B) represents a bipartite graph)
+
+![image](https://github.com/user-attachments/assets/baba8325-6007-4161-8fe7-9098d57ff526)
+
+Multigraph vs. simple graph ((A) represents a simple graph; (B) represents a multigraph)
+
+![image](https://github.com/user-attachments/assets/10de26a5-3acf-48c1-8c03-24a6d033a60e)
+
+A complete graph (a graph in which each node is connected to all the other nodes)
+
+![image](https://github.com/user-attachments/assets/9292e5e7-f888-43cc-ae6b-216a928b7906)
+
+### 2.2 Network representations
+
+This section introduced Cypher's syntax for text-based network representations, used to describe nodes and relationships. Nodes are enclosed in parentheses with labels (e.g., Person) and properties (e.g., {name: "Thomas"})
+
+Example Cypher representation of a work relationship:
+
+```cypher
+(:Person {name:"Tomaz"})-[:WRITES_FOR {since: 2020}]->(:Organization {name:"Manning Publications"})
+```
+
+**Labeled-Property Graph Model**
+
+The labeled-property graph (LPG) model consists of nodes with labels and properties, and directed relationships. For example
+
+```cypher
+(:Person {name:"Tomaz"})<-[:WRITES_FOR {since:2020}]-(:Organization {name:"Manning Publications"})
+```
+
+In bidirectional relationships like friendship, direction is less important:
+
+```cypher
+(:Person {name:"Thomas"})-[:FRIEND {since: 2016}]->(:Person {name:"Elaine"})
+```
+
+### 2.3 Designing your first labeled-property graph model
+
+Now, imagine a client asks to perform a network analysis of Twitter. They’ll provide all the necessary data, and my job is to represent it as a graph and extract insights through network analysis. I’ll use the LPG model to represent Twitter. A good approach is to work backward, starting with the questions I want to answer. In this case, no specific questions were provided, so it’s up to me as the network scientist to uncover as many insights as possible. To begin, I can define the key elements of the Twitter domain, such as:
+
+- Users can follow other users (follower network).
+- Users can publish tweets (user-tweet network).
+- Users can retweet posts from others (retweet network).
+
+**Follower network**
+
+On Twitter, you have the option to follow other users. By following users, you are subscribing to their activity and indicating that you would like to see their tweets on your feed. The follower-network specification is as follows: A user can follow other users.
+
+There is no correct or perfect way to design a graph model, but some models can be useful for a specific scenario. When designing a graph model, try to answer the following questions:
+
+* How many different types of nodes are present?
+
+* What kind of properties do these nodes have?
+
+* Which property would you use to store the unique identifier of nodes?
+
+* What type of relationships are present?
+
+* Is a single relationship type enough to accurately describe your domain?
+
+* Does the relationship direction hold any semantic value?
+
+* How do properties qualify or quantify relationships?
+
+In the follower-network specification, both the subject and the object of the sentence are users and can be represented as nodes. Relationships can be used to represent the verb of the specification sentence. Here, it makes sense to represent a follow interaction as a relationship between two users. For instance:
+
+```cypher
+(:User{id:"Vanessa", registeredAt:"2019-03-05"})-[:FOLLOWS{since:"2020-01-01"}]->(:User{id:"Thomas", registeredAt:"2011-03-05"})
+```
+
+![image](https://github.com/user-attachments/assets/1a9a8c43-197f-48e8-ae16-f9588753e5f6)
+
+**User-tweet network**
+
+Tweets are the primary way to share content on Twitter. The simplest description of the user-tweet network is as follows: A user can publish a tweet.
+
+For instance:
+
+```cypher
+(:User)-[:PUBLISH]->(:Tweet)
+```
+
+![image](https://github.com/user-attachments/assets/1d5275b8-ea01-4772-bc53-52ca92fa8113)
+
+We can add  PUBLISHED_BY relationship too
+
+![image](https://github.com/user-attachments/assets/0780fc1e-4a79-4e8b-9760-9b284f000e71)
+
+And also LIKE
+
+![image](https://github.com/user-attachments/assets/27e68b38-905a-41eb-981e-bd84db69c18b)
+
+**Retweet network**
+
+The only remaining task specification is the definition of a graph model for retweets. When users strongly react to a tweet, they might want to share it with their followers to amplify its reach. In this case, they have the option of retweeting the original tweet. Optionally, users can like the retweet, and those likes do not count toward the original tweet. The task specification is defined as follows: A user can retweet posts from other users.
+
+```cypher
+(:User)-[:PUBLISH]->(:Tweet)<-[:RETWEETS]-(:User)
+```
+
+![image](https://github.com/user-attachments/assets/3684cbea-54ff-4443-9c18-4830471fc5a9)
+
+**Representing graph schema**
+
+The beauty of the graph approach to data modeling is that we can connect new information to an existing graph. We can now combine all the graph model decisions so far into a single graph model.
+
+![image](https://github.com/user-attachments/assets/8f38ee6a-912e-4138-8c2e-a0d87cd69aa6)
+
+The above is a labeled-property graph representing a Twitter network, where users can follow one another as well as publish and retweet posts
+
+### 2.4 Extracting knowledge from text
+
+**Links**
+
+The first information we can extract from tweet content is any links included in the tweet
+
+We could either store the URL as a tweet node property or store it as a separate node and add a relationship between the link and the tweet
+
+![image](https://github.com/user-attachments/assets/0154b114-f1e4-4a3d-8eb8-f987f6dcd89b)
+
+Considerations:
+
+1. Standardization: If values like links in tweets are not standardized, storing them as separate nodes can reduce traversal efficiency. Standardized values allow faster queries and should be represented as single nodes in the graph
+
+2. Node Representation: A real-world entity should only be represented by one node. Storing non-standardized data as multiple nodes (e.g., the same website as different nodes) can lead to invalid results
+
+3. Super Nodes: Unspecific information, like gender, is better stored as node properties to avoid "super nodes"—nodes connected to a large portion of the graph—which can negatively impact query performance
+
+**Hashtags**
+
+People use the hashtag symbol (#) before a relevant keyword or phrase in their tweet to categorize them. A tweet can contain many hashtags; it makes sense to store them as separate nodes and connect tweets to them
+
+![image](https://github.com/user-attachments/assets/d70a587b-7920-4463-a627-4079535f061d)
+
+An important thing to remember is to avoid generic relationship types. For example `HAS`. `HAS` can relate to many relationships so it is best to be more descriptive. 
+
+**Mentions**
+
+A user can mention other users in their tweet by using the mention symbol (@). A mention can be understood as an invitation to comment or a callout, while at other times, it can be used to notify users to look at specific content. We already have users defined in the graph schema, so it only makes sense to connect tweets to mentioned users
+
+![image](https://github.com/user-attachments/assets/e5b3a618-951e-4dff-b3a1-b0d5050d8a82)
+
+Like the hashtag network, the mention network is also a classic bipartite network.
+
+**Final Twitter social network schema**
+
+![image](https://github.com/user-attachments/assets/1a7baa10-5023-48b4-8103-a0a4da23d47c)
+![image](https://github.com/user-attachments/assets/2f9d45f2-0deb-4ccb-9b80-b1fb0da8c2eb)
+
+
+Designing a graph model schema is an iterative process where additional data enriches the graph over time. A self-describing graph model is recommended to avoid the need for a separate schema manual. The schema may evolve based on query performance needs.
+
+For example, in a Twitter network model, nodes include users, tweets, hashtags, and links, with six types of relationships. Inferred relationships should be left out until they can be instantiated.
+
+A key theme in network analysis is converting indirect relationships to direct ones, such as turning retweet links into direct amplification relationships between users. Monopartite projections are often used for applying graph algorithms to networks.
+
+## Ch. 3 Your First Steps With Cypher Query Language
+
+As the title suggests this was an introduction to Cypher - the language used to interact with Neo4j. I just read this part, as I have already used Neo4j, but also covered Neo4j's official [Intro to Cypher course](https://graphacademy.neo4j.com/courses/cypher-fundamentals/) 
+
+Here is a summary of the chapter:
+
+- Cypher syntax uses parentheses, `()`, to encapsulate a node.
+- Cypher syntax uses square brackets, `[]`, to encapsulate a relationship.
+- A relationship cannot exist on its own but needs to be described with adjacent nodes.
+- In Neo4j, all relationships are stored as directed, although you can ignore the direction at query time.
+- It is not advisable to ever create any nodes without node labels.
+- The CREATE clause is used to create data.
+- The MATCH clause is used to identify existing patterns in the database.
+- The WHERE clause can be used in combination with MATCH or WITH clauses to specify various filters.
+- If a MATCH clause doesn’t find the specified graph pattern, the whole Cypher statement returns no results.
+- You can use OPTIONAL MATCH when you are not certain whether a graph pattern exists in the database, but you still want to return other variables in the query output.
+- The WITH clause can be used to filter, aggregate, select, paginate, or limit intermediate rows in the Cypher statement.
+- The SET clause is used to add node properties and labels.
+- The REMOVE clause is used to remove node properties and labels.
+- The DELETE clause is used to delete nodes and relationships.
+- If you want to delete a node with existing relationship patterns, you need to use the DETACH DELETE clause.
+- The MERGE clause is a combination of MATCH and CREATE clauses that ensures the specified graph pattern exists in the database.
+- The MERGE clause is frequently used for data import, as it allows for idempotent queries and automatic deduplication.
+- In Neo4j, you can define unique constraints that ensure unique values for the specified node property of a particular node label.
+- It is recommended to import nodes and relationships separately into the database to improve performance and data quality.
+- You can easily evaluate the graph schema of existing data with the `db.schema.visualization()` procedure.
+
+---
+
+Tomorrow I will continue with this book. And also will be my 1st stream on youtube 🥳 from 6PM KST
+
+That is all for today!
+
+See you tomorrow :)

From df13ccfbbe253c3ad0f032d0e98d535113d4d3bf Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 30 Sep 2024 10:14:11 +0900
Subject: [PATCH 439/638] add quick summary of the data

---
 _posts/2024-09-29-day-272.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_posts/2024-09-29-day-272.md b/_posts/2024-09-29-day-272.md
index 75a1c3150..3b8b1228c 100644
--- a/_posts/2024-09-29-day-272.md
+++ b/_posts/2024-09-29-day-272.md
@@ -6,6 +6,7 @@ categories: [theory,gnn]
 
 # Hello :) Today is Day 272!
 A quick summary of today:
+* read a bit more of [Graph Algorithms for Data Science](https://learning.oreilly.com/library/view/graph-algorithms-for/9781617299469/)
 
 ## Ch. 2 Representing network structure: Designing your first graph model
 

From 77e9b48e695f61ed97bf0809b43c35685d3932d1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 30 Sep 2024 11:38:24 +0900
Subject: [PATCH 440/638] save 273

---
 _posts/2024-09-30-day-273.md | 219 +++++++++++++++++++++++++++++++++++
 1 file changed, 219 insertions(+)
 create mode 100644 _posts/2024-09-30-day-273.md

diff --git a/_posts/2024-09-30-day-273.md b/_posts/2024-09-30-day-273.md
new file mode 100644
index 000000000..47f333d58
--- /dev/null
+++ b/_posts/2024-09-30-day-273.md
@@ -0,0 +1,219 @@
+---
+layout: post
+title: (Day 273)
+categories: [theory,gnn]
+---
+
+# Hello :) Today is Day 273!
+A quick summary of today:
+
+## Ch. 4 Exploratory graph analysis
+
+### 4.1 Exploring the Twitter network
+
+When going into a neo4j database, by default there is some information about the graph
+
+![image](https://github.com/user-attachments/assets/14277dd1-5b13-4f65-b0f6-f1ea049ce142)
+
+We can visualisa the graph using
+
+```cypher
+MATCH p=()-[r :FOLLOWS]->()
+RETURN p LIMIT 25;
+```
+
+![image](https://github.com/user-attachments/assets/6dbb0af0-ac55-4c72-ada8-b8a9988b75ff)
+
+### 4.2 Aggregating data with Cypher query language
+
+Example:
+
+Calculating the ratio of non-null values of `registeredAt` node property
+
+```cypher
+MATCH (u:User)
+WITH count(*) AS numberOfRows,
+     count(u.registeredAt) AS usersWithRegisteredAtDate
+RETURN toFloat(usersWithRegisteredAtDate) / numberOfRows * 100 AS result
+```
+
+**NOTE** In Neo4j, when we divide an integer value by another integer value, the result will also be the integer data type. If we want the result to be of the float type we need to cast either of the variables to float using the toFloat() function
+
+Counting the number of nodes by labels
+
+```cypher
+MATCH (n)
+RETURN labels(n) AS labels,
+       count(n) AS count;
+```
+
+Retrieving the earliest and last created date values of tweets
+
+```cypher
+MATCH (n:Tweet)
+RETURN min(n.createdAt) AS earliestDate, max(n.createdAt) as lastDate
+```
+
+Extracting datetime attributes
+
+```cypher
+MATCH (t:Tweet)
+WITH t LIMIT 1
+RETURN t.createdAt.year AS year,
+       t.createdAt.month AS month,
+       t.createdAt.day AS day,
+       t.createdAt.epochSeconds AS epochSeconds;
+```
+
+### 4.3 Filtering graph patterns
+
+Counting the number of users who were mentioned in a tweet and discounting the retweet mention pattern with an existential subquery
+
+```cypher
+MATCH (u:User)<-[:MENTIONS]-(tweet:Tweet)
+WHERE NOT EXISTS {
+  (original)<-[:PUBLISH]-(u)<-[:MENTIONS]-(tweet)-[:RETWEETS]->(original)
+}
+RETURN count(distinct u) AS countOfUsers
+```
+
+### 4.4 Counting subqueries
+
+Retrieving the top five most mentioned users
+
+```cypher
+MATCH (u:User)<-[:MENTIONS]-(:Tweet)
+WITH u, count(*) AS mentions
+ORDER BY mentions DESC LIMIT 5
+RETURN u.username AS user, mentions
+```
+
+There is nothing wrong with the statement above. However, you will frequently be performing multiple aggregations in a single query. When performing multiple aggregations in a query, you must be very mindful of the query cardinality (number of intermediate rows in the query). A simple yet very effective syntax to not increase the cardinality when counting the number of relationships a node has is to use the count {} operator and describe the desired graph pattern you want to count, as shown in the following listing.
+
+Convenient way of retrieving the top five most mentioned users by not increasing main query cardinality
+
+```cypher
+MATCH (u:User)
+WITH u, count { (u)<-[:MENTIONS]-() } AS mentions
+ORDER BY mentions DESC LIMIT 5
+RETURN u.username AS user, mentions
+```
+
+### 4.5 Multiple aggregations in sequence
+
+when performing multiple aggregation in sequence we must be careful with the intermediate cardinality
+
+Say we have 2 MATCH clauses in a row
+
+```cypher
+MATCH (u:User)
+MATCH (t:Tweet)
+RETURN count(*) AS numberOfRows,
+       count(u) AS countOfUsers,
+       count(t) AS countOfTweets
+```
+
+The cardinality problem in Cypher queries arises when we have multiple MATCH or OPTIONAL MATCH clauses executed in sequence without reducing the intermediate results. Each MATCH or OPTIONAL MATCH clause produces a number of rows, and if we don’t control this, subsequent clauses will be executed for every row produced by the previous one. This can lead to an exponential growth in the number of rows, causing inefficient queries and incorrect results.
+
+To prevent this, we should reduce the cardinality after each MATCH or OPTIONAL MATCH clause before moving on to the next one. The key techniques are:
+
+* Using Aggregating Functions: after each MATCH clause, we can use an aggregation function like count() to reduce the number of rows produced
+
+```cypher
+MATCH (u:User)
+WITH count(u) AS countOfUsers
+MATCH (t:Tweet)
+RETURN count(*) AS numberOfRows, countOfUsers, count(t) AS countOfTweets
+```
+
+This reduces the number of rows between the MATCH clauses, allowing subsequent clauses to be executed only once.
+
+* Using the WITH Clause: we can reduce intermediate results by using WITH to pass along only the necessary data; by aggregating after each MATCH we ensure that subsequent MATCH clauses operate on a single or a reduced set of rows, avoiding exponential row growth
+
+Example:
+
+```cypher
+MATCH (u:User)
+WHERE u.username = "IainLJBrown" 
+OPTIONAL MATCH (u)-[:PUBLISH]->(rt)<-[:RETWEETS]-()
+WITH u, count(rt) AS numberOfRetweets 
+OPTIONAL MATCH (u)<-[:MENTIONS]-(t)
+WHERE NOT (t)-[:RETWEETS]->()
+WITH u, numberOfRetweets, count(t) AS mentionsInOriginalTweets 
+OPTIONAL MATCH (u)<-[:MENTIONS]-(ort)
+WHERE (ort)-[:RETWEETS]->() AND NOT (ort)-[:RETWEETS]->()<-[:PUBLISH]-(u)
+WITH u, numberOfRetweets, mentionsInOriginalTweets, count(ort) AS mentionsInRetweets
+RETURN u.username AS user, numberOfRetweets, mentionsInOriginalTweets, mentionsInRetweets
+```
+
+In the above query, after each OPTIONAL MATCH clause, the cardinality is reduced using the count() function. This ensures the query doesn’t balloon out of control in terms of the number of rows, and each clause is executed only once for the necessary entities.
+
+## Ch. 5 Introduction to social network analysis
+
+An essential aspect of characterizing any network is to look at the node degree distribution. In simple terms, the node degree is the number of links each node has. In a random network, the degree distribution will follow the Gaussian distribution
+
+![image](https://github.com/user-attachments/assets/f6cac7de-1302-4d63-a722-92a77f8e074f)
+
+The vast majority of nodes have roughly the same number of links. There won’t be many hugely popular nodes, but there won’t be many isolated nodes either.
+
+
+However, the Gaussian distribution is most often used for independent observations. On the other hand, a graph consists of highly interconnected observations that are not independent. It turns out that almost no real-world network follows the random network degree distribution
+
+When PageRank was developed by Google, they discovered that **the web** degree distribution follows a different pattern
+
+![image](https://github.com/user-attachments/assets/c67dd23c-8e05-4c08-be71-a87b051d5fac)
+
+
+### 5.1 Follower network
+
+This chapter will introduce some community detection and centrality graph algorithms. The **community detection algorithms** will be used to characterize the network and also find tightly connected groups of users. In the context of networks, a community refers to a densely connected group of nodes, whose members have comparatively fewer connections to other communities in the network
+
+![image](https://github.com/user-attachments/assets/41762e52-3c55-44ab-b283-bdc109a0e67f)
+
+The metrics used to identify important or influential nodes in the graph are called centrality measures, and the algorithms that calculate them are called **centrality algorithms**. For example, the most basic metric to determine a node’s importance is degree centrality, which simply counts the number of relationships a node has. The higher the count of relationships is, the more influential the node is in the network.
+
+**Node degree distribution**
+
+One of the fundamental characteristics of a network is the node degree distribution. With a directed network, you can split the degree distribution into in-degree and out-degree distribution. The node in-degree counts the number of incoming relationships, and the out-degree counts the number of outgoing connections per node.
+
+Evaluating the node out-degree distribution with the apoc.agg.statistics function
+
+```cypher
+MATCH (u:User)
+WITH u, count{ (u)-[:FOLLOWS]->() } AS outDegree
+RETURN apoc.agg.statistics(outDegree)
+```
+
+| Metric      | Value               |
+|-------------|---------------------|
+| Total       | 3,594               |
+| Min         | 0                   |
+| MinNonZero  | 1.0                 |
+| Max         | 143                 |
+| Mean        | 6.924874791318865   |
+| 0.5 (Median)| 2                   |
+| 0.99        | 57                  |
+| 0.75        | 8                   |
+| 0.9         | 21                  |
+| 0.95        | 32                  |
+| Stdev       | 11.94885358058576   |
+
+Node out-degree distribution users' follows
+
+![image](https://github.com/user-attachments/assets/cbb5db8e-f2d0-4e91-846c-b23789fcf76f)
+
+### 5.2 Introduction to the Neo4j Graph Data Science library
+
+
+
+
+
+
+
+
+
+
+
+
+
+

From 86b8cf7af4038f07217667b59b8b3ed7352c02f1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 30 Sep 2024 15:28:27 +0900
Subject: [PATCH 441/638] save 273

---
 _posts/2024-09-30-day-273.md | 421 +++++++++++++++++++++++++++++++++++
 1 file changed, 421 insertions(+)

diff --git a/_posts/2024-09-30-day-273.md b/_posts/2024-09-30-day-273.md
index 47f333d58..4818b3b94 100644
--- a/_posts/2024-09-30-day-273.md
+++ b/_posts/2024-09-30-day-273.md
@@ -204,6 +204,427 @@ Node out-degree distribution users' follows
 
 ### 5.2 Introduction to the Neo4j Graph Data Science library
 
+The Neo4j Graph Data Science library is a plugin for Neo4j that features more than 50 graph algorithms, ranging from community detection and centrality to node-embedding algorithms and link prediction pipelines—and [more](https://neo4j.com/docs/graph-data-science/current/). 
+
+Graph algorithms in the GDS library are executed on a projected in-memory graph structure separate from the graph stored in the database. To execute graph algorithms with the GDS library, you must first project an in-memory graph. The projected graph is stored entirely in-memory, using an optimized data structure for scalable and parallel graph algorithm execution. You can create a projected in-memory graph using either native projection or Cypher projection
+
+![image](https://github.com/user-attachments/assets/2b4df98b-3d44-4b22-8c31-3169612afcab)
+
+Native projection is a bit more limited in selecting or filtering a specific subgraph you want to project, as you can only filter based on node labels and relationship types. However, it is the recommended way of projecting a graph as it is highly performant due to reading data directly from Neo4j storage.
+
+The second available option for creating an in-memory graph is the Cypher projection. With it, you get all the flexibility of Cypher query language to select or filter any specific subgraph you might want to project. Of course, Cypher projection has a drawback, as it is slower than native projection and generally recommended only for the experimental or explorational phase of a project.
+
+As the in-memory graph projection can be costly when dealing with large graphs, the GDS library also features a graph catalog. The graph catalog comes in handy when you want to execute multiple graph algorithms on the same projected graph. Instead of having to create an in-memory graph for each algorithm execution separately, you can create an in-memory graph once and then execute multiple graph algorithms on it. The projected graph can then be accessed via its name when executing graph algorithms, so the term named graph stuck with projected graphs stored in a graph catalog. Once the in-memory graph is created, you can execute graph algorithms on top of it.
+
+Each algorithm has four modes of execution, depending on the use case:
+
+* `stream` — Returns results as a stream of records and does not store results
+
+* `stats` — Returns a summary statistics of the result and does not store results
+
+* `mutate` — Writes the results back to the projected in-memory graph. This mode can only be used in combination with a named graph stored in a graph catalog. It is very useful when you want to use an output of one graph algorithm as an input to another
+
+* `write` — Writes the results back to the Neo4j database graph
+
+#### Graph catalog and native projection
+
+Native projection syntax to create a named graph in the graph catalog
+
+```cypher
+CALL gds.graph.project(
+    graphName,
+    nodeProjection,
+    relationshipProjection,
+    optional configuration
+)
+```
+
+GDS procedures are executed using the CALL clause in combination with the procedure name. The procedure to store a named graph in the graph catalog with native projection is called gds.graph.project(). It contains three mandatory and one optional parameter. The first parameter is used to name the graph, under which it will be accessed when executing graph algorithms. The second parameter, called nodeProjection, defines the subset of nodes you want to project. Similarly, the relationshipProjection parameter specifies which relationships should be considered when creating an in-memory graph. An important thing to note is that a relationship will be skipped during projection if both adjacent nodes are not described in the nodeProjection parameter. In GDS terms, the starting node of the relationship is called the source node and the end node is called the target node.
+
+The below statement uses native projection to store an in-memory graph. The first parameter specifies its name, which will be used to access it when executing graph algorithms. The second parameter defines which nodes you want to include in the projection. When you only want to project a single type of node, you can define the desired node label as a string. Similarly, when you only want to project one type of relationship in the third parameter, you can specify the type as a string.
+
+```cypher
+CALL gds.graph.project('follower-network', 'User', 'FOLLOWS') 
+```
+
+### 5.3 Network characterization
+
+#### Weakly connected component (WCC) algorithm
+
+A WCC is a set of nodes within the graph, where a path exists between all nodes in the set if the direction of relationships is ignored. A WCC can be considered an “island” that cannot be reached from other graph components. While the algorithm identifies connected sets of nodes, its output can help you evaluate how disconnected the overall graph is.
+
+Understanding the connectivity and how connected the overall graph is sets the stage for more detailed investigations, such as community detection or centrality analysis. The WCC algorithm is probably a graph algorithm that should be executed as the first step of any graph analysis to evaluate graph connectivity.
+
+The WCC algorithm is executed using the `write` mode. As mentioned, the write mode stores the results back to the Neo4j database but also provides summary statistics of the algorithm result.
+
+_Graph algorithm procedure syntax_
+
+```cypher
+CALL gds.<algorithm>.<mode>(namedGraph, {optional configuration})
+```
+
+When using the write mode of an algorithm, you need to provide the mandatory writeProperty parameter, which specifies the name of the node property the algorithm results will be stored to. The procedure to execute the WCC algorithm is gds.wcc.
+
+_Executing the WCC algorithm on the follower network and storing the results as a followerWcc node property_
+
+```cypher
+CALL gds.wcc.write('follower-network', {writeProperty:'followerWcc'})
+YIELD componentCount, componentDistribution
+```
+
+Here are summary statistics from the above:
+
+![image](https://github.com/user-attachments/assets/8351d3d7-cae2-491a-bd6d-1b198dbf6b86)
+
+There are 547 disconnected components in the followers network, and the largest contains 2,997 members. Most real-world networks have a single connected component containing most of the nodes in the network and a couple of disconnected peripheral components. As the dataset you are analyzing is only a portion of a larger network, having a higher count of components is not unusual, due to many missing users and relationships that would otherwise connect various parts of the network if they were included. 
+
+The `p90` result, or the 90th percentile of the component size, has a value of 1, which indicates that 90% of the components have only a single member. When a component contains only a single member, this means the node has no relationships.
+
+#### Strongly connected components (SCC) algorithm
+
+A strongly connected component (SCC) is a subgraph of a directed graph in which a path exists between all its nodes. The only difference between the WCC and SCC algorithms is that the SCC algorithm considers relationship directions. Therefore, the SCC algorithm is only applicable to a directed graph. If the relationship direction is ignored, then you are dealing with WCCs.
+
+The SCC algorithm is useful when directed paths and reachability play an important role. For example, imagine a road network where the nodes represent intersections and relationships represent road connections. For example, many large city centers have a lot of one-way road connections. Using the SCC algorithm, you could evaluate the consequences of closing one or several road connections and how it would affect the reachability of places within the city.
+
+![image](https://github.com/user-attachments/assets/072bc182-2273-4040-b0ef-ce359f067579)
+
+_Executing the SCC algorithm on the follower network and storing the results as a followerScc node property_
+
+```cypher
+CALL gds.scc.write('follower-network', {writeProperty:'followerScc'})
+YIELD componentCount, componentDistribution
+```
+
+Summary statistics:
+
+![image](https://github.com/user-attachments/assets/3aa9549c-5e36-4946-9be2-fae8027072a9)
+
+As expected, the count of SCCs is higher than the count of WCCs. There are 2,704 SCCs, and the largest one contains 796 members.
+
+#### Local clustering coefficient (LCC)
+
+The local clustering coefficient (LCC) is a metric that quantifies how connected or close the neighbors of a particular node are. The LCC measures the average probability that two neighbors of a node are connected. Therefore, the value of the LCC ranges from 0 to 1. The LCC value of 0 indicates that the neighboring nodes have no connections between each other. On the other hand, the LCC value of 1 indicates that the network of neighbors forms a complete graph, where all the neighbors are connected.
+
+![image](https://github.com/user-attachments/assets/c01f9f25-2183-412a-bb33-d185b2ba5562)
+
+The LCC algorithm provides a metric to evaluate how strongly the neighbors of a node are connected. You can calculate the LCC value of a single node by dividing the number of existing links between neighbor nodes with the number of possible links between neighbor nodes
+
+_Calculating the LCC on the directed followers network_
+
+```cypher
+MATCH (u:User)
+OPTIONAL MATCH (u)-[:FOLLOWS]-(n)
+WITH u,count(distinct n) AS neighbors_count 
+OPTIONAL MATCH (u)-[:FOLLOWS]-()-[r:FOLLOWS]-()-[:FOLLOWS]-(u)
+WITH u, neighbors_count, count(distinct r) AS existing_links 
+WITH u,
+     CASE WHEN neighbors_count < 2 THEN 0 ELSE
+       toFloat(existing_links) / (neighbors_count * (neighbors_count - 1))
+       END AS lcc
+SET u.lcc = lcc
+```
+
+1. Matches all User nodes
+2. Counts the number of their distinct neighbors
+3. Counts the number of distinct links between neighbors
+4. Calculates the LCC value
+5. Stores the LCC value under the lcc node property
+
+### 5.4 Identifying central nodes
+
+#### PageRank
+
+PageRank was designed by Larry Page and Sergey Brin (1999) and helped make Google search what it is today. PageRank measures the transitive or directional influence of nodes. For example, the node degree quantifies the influence or importance of a node by considering only its direct neighbors. In contrast, PageRank also considers the indirect relationships with other nodes in the graph spanning over multiple hops. To put it into our Twitter subgraph context, if, for example, Elon Musk or Andrew Ng follows you, you gain more influence than if I follow you. PageRank evaluates the number of followers a particular node has as well as how influential those followers are.
+
+_Executing PageRank on the followers network_
+
+```cypher
+CALL gds.pageRank.write('follower-network',
+  {writeProperty:'followerPageRank'})
+```
+
+_Examining the top five followers for the highest-ranking users_
+
+```cypher
+MATCH (u:User)<-[:FOLLOWS]-(f)
+WHERE u.username IN
+  ["elonmusk", "NASA", "wmktech", "Twitter", "Wajdialkayal1"]
+WITH u,f
+ORDER BY f.followerPageRank DESC 
+RETURN u.username AS user,
+       round(u.followerPageRank, 2) AS pagerankScore,
+       collect(f.username)[..5] AS topFiveFollowers
+ORDER BY pagerankScore DESC
+```
+
+#### Personalized PageRank algorithm
+
+The Neo4j GDS library also supports the personalized PageRank variation. In the PageRank definition, a web surfer can get bored and randomly jump to other nodes. With the personalized PageRank algorithm, you can define which nodes the web surfer should jump to when they get bored. It can be said that by defining the sourceNodes to which the surfer is biased to jump, you are effectively inspecting the influence of nodes by looking through a particular node’s or multiple nodes’ point of view.
+
+_Running the personalized PageRank algorithm from the point of view of users who registered in 2016_
+
+```cypher
+MATCH (u:User)
+WHERE u.registeredAt.year = 2016
+WITH collect(u) AS sourceNodes 
+CALL gds.pageRank.stream('follower-network', {sourceNodes: sourceNodes})
+YIELD nodeId, score 
+RETURN gds.util.asNode(nodeId).username AS user, score
+ORDER BY score DESC
+LIMIT 5;
+```
+
+#### Dropping the named graph
+
+After completing the planned graph algorithms execution sequence, it is recommended to drop the projected graph from memory using `CALL gds.graph.drop('your_network_name')`
+
+## Ch. 6 Projecting monopartite networks
+
+Overview:
+
+* Translating an indirect graph pattern into a direct relationship
+* Using Cypher projection to project an in-memory graph
+* Presenting self-loops
+* Introducing weighted variations of degree and PageRank centrality algorithms
+
+Most graph algorithms are designed to be executed on a monopartite network, where only a single node and relationship type are present. However, the Twitter social network schema contains multiple node types and relationships. Instead of adjusting graph algorithms to support multipartite networks (multiple node and relationship types), the general approach is to first project a monopartite network (single node and relationship type).
+
+### 6.1 Translating an indirect multihop path into a direct relationship
+
+With Neo4j Graph Data Science (GDS) we could take two different approaches to accomplish this task
+
+![image](https://github.com/user-attachments/assets/819e8558-a84e-45ae-9c9c-47c50f4a5d87)
+
+To create a monopartite retweet network using native projection, you must first materialize it in your Neo4j database. However, native projection does not allow for custom network transformations during graph loading. In contrast, Cypher projection enables you to load a virtual graph into memory. A virtual graph is one that is not stored in the database but is constructed only at the time of projection. This capability allows for custom transformations without the need to store them in the database, making it easier to explore and analyze different graph projections while keeping your graph database organized. Cypher projection utilizes the full expressiveness of the Cypher query language, allowing you to select, filter, and transform the graph before projection. 
+
+#### Cypher projection
+
+Cypher projection is a more flexible and expressive approach to projecting an in-memory graph. As you might deduce from the feature’s name, you can use Cypher statements to define the nodes and relationships you want to load in the in-memory graph. The Cypher projection function is called `gds.graph.project` and has three mandatory and two optional parameters.
+
+```cypher
+MATCH (sourceNode)-[relationship]->(targetNode)
+RETURN gds.graph.project(
+   'graph',
+   sourceNode,
+   targetNode,
+   {dataConfig},
+   {configuration} 
+) YIELD
+    graphName,
+    nodeCount,
+    relationshipCount
+```
+
+1. Projected graph name
+2. Source node of a relationship
+3. Target node of a relationship
+4. Optional property and type configuration map
+5. Optional parameter map to define undirected relationships
+
+_You can think of Cypher projection as using Cypher statements to describe relationships of a projected graph, where each connection is defined with its source and target nodes. First, you must use the Cypher syntax to match the source and target nodes of the relationships you want to project. As mentioned, you can match existing relationships in the database or define virtual connections not materialized in the database. Once you have specified the source and target nodes of the desired relationships, you can use the gds.graph.project function in a WITH or RETURN clause to project a graph instead of having to use the CALL clause. The first parameter of the gds.graph .project function is used to define the name of the projected in-memory graph. On the other hand, the second and third parameters describe the source and target nodes of the projected relationship. The fourth parameter is optional and is used to specify node and relationship properties and their labels or types, if needed. By defining node labels and relationship types, you can efficiently filter them at algorithm runtime._
+
+### 6.2 Retweet network characterization
+
+**Degree centrality**
+
+_Evaluating the out-degree distribution of the inferred retweet amplification network_
+
+```cypher
+CALL gds.degree.stats('amplify')
+YIELD centralityDistribution
+```
+
+Top 5 users by out-degree:
+
+![image](https://github.com/user-attachments/assets/fc6e22a1-a7f4-4dc9-a47a-20576ae87de9)
+
+### 6.3 Identifying the most influential content creators
+
+We can achieve that by following the below steps:
+
+1. Ignore self-loops during projection (like retweeting your own tweets)
+2. Execute the weighted variant of the PageRank algorithm
+
+![image](https://github.com/user-attachments/assets/4e9b77bf-581d-4ae4-8f1a-2fb1d2ebd928)
+
+_Executing the PageRank algorithm on the retweet amplification network with no self-loops_
+
+```cypher
+CALL gds.pageRank.stream('amplify-noselfloops',
+  {relationshipWeightProperty:'weight'})
+YIELD nodeId, score
+RETURN gds.util.asNode(nodeId).username AS user, score
+ORDER BY score DESC
+LIMIT 5
+```
+
+Result:
+
+![image](https://github.com/user-attachments/assets/8c38c5e7-7799-4424-9f4c-b047d713f58a)
+
+## Ch. 7 Inferring co-occurrence networks based on bipartite networks
+
+Imagine we're working in a marketing analytics role for a company focused on natural language processing (NLP) and knowledge graphs. Our boss suggests improving the hashtag strategy for the content published on Twitter. We've been tasked with identifying relevant hashtags to better target the company’s ideal customer. Since the Twitter dataset we've been using contains tweets related to NLP and knowledge graphs, we can analyze the hashtags in the dataset to determine which ones the company should focus on.
+
+Our initial thought might be to use the PageRank algorithm to identify the most important hashtags. However, we need to remember that graph algorithms like PageRank typically expect monopartite networks as input, or at least networks where influence can flow through all connections. In a bipartite network, one type of node has only incoming relationships, while the other type has only outgoing connections. If we were to run PageRank on this bipartite network of tweets and hashtags, which nodes would come out on top?
+
+Since tweets don’t have any incoming relationships, their PageRank score will equal the chance of a web surfer randomly landing on them. With the default value of the damping factor of 0.85, the PageRank score for nodes with no incoming connections is 0.15. On the other hand, hashtags have only incoming relationships. The influence flows from tweets to hashtags but does not flow further, as there are no outgoing connections from hashtags. In practice, the PageRank rank of hashtags would be equal to their count of incoming relationships (in-degree). However, the actual values of PageRank and in-degree would be different due to distinct score calculation techniques.
+
+The goal is to determine the most important and relevant hashtags in the dataset. If the definition of hashtag importance is as simple as their frequency, then using the in-degree metric of the Tag nodes would suffice.
+
+But some hashtags in the dataset may not be relevant to the marketing objective, and focusing only on the most frequently mentioned ones could cause important hashtags to be overlooked. Additionally, combining certain hashtags could extend content reach to a broader audience. The goal is to identify both key hashtags and those that can be used together to increase virality. A useful technique for this is analyzing co-occurrence networks, which show how often pairs of hashtags appear together in tweets. This method helps identify communities of hashtags that are frequently used in combination.
+
+The below pic shows the co-occurrence network of keywords in medical articles. Co-occurrence networks are constructed by connecting pairs of entities in the text using a set of criteria defining co-occurrences. The co-occurrence definition can vary from scenario to scenario. In this example, co-occurrence is defined as two keywords occurring in the same article. The more times a pair of keywords are present in the same article, the stronger the connection is between the two.
+
+![image](https://github.com/user-attachments/assets/33de941b-dfa3-47e7-809e-a44e4a6802bf)
+
+_Transforming a bipartite network into a co-occurance network_
+
+![image](https://github.com/user-attachments/assets/ac0d5ec1-5cbc-4e75-aeec-295efe0f2630)
+
+```cypher
+MATCH (s:Tag)<-[:HAS_TAG]-(:Tweet)-[:HAS_TAG]->(t:Tag)
+CREATE (s)-[:CO_OCCURRENCE]->(t);
+```
+
+To find communities or clusters of hashtags that form a topic, we can utilize community detection algorithms like the label propagation algorithm (LPA). LPA is an algorithm to evaluate the community structure of a network. Most of the literature on the internet introduces the LPA as a semisupervised algorithm in which you can input initial communities for some nodes in the network. However, here the unsupervised variant of the LPA is used, as no initial communities are presented. The unsupervised variant of the LPA works as follows. First, it assigns a unique community label to each node. Then, it iterates over the network and updates each node label to the one most of its neighbors have. The idea behind this iteration is that a single community label can quickly become dominant in a densely connected group of nodes. Once the LPA reaches convergence, the algorithm stops, and the resulting node labels represent their communities.
+
+![image](https://github.com/user-attachments/assets/4f9000fe-c340-4638-8b0d-01e2d367aecb)
+
+## Ch. 8 Constructing a nearest neighbor similarity network
+
+### 8.1 Feature extraction
+
+Nodes with similar roles do not have to be next to one another in the network. For example, you could say users with a large following have a role in producing certain types of content. There could be multiple users with a large following, and they don’t have to follow one another or be close in the network, but they still hold a similar role. We can encode a node’s local neighborhood by counting its graphlets. A graphlet is a position of a node in a distinctly connected subgraph consisting of k nodes.
+
+**Motifs**: small, recurring subgraphs or patterns found within a larger graph that occur significantly more often than would be expected by random chance. These patterns typically involve a small number of nodes (usually 3-5 nodes) and specific connectivity
+
+**Graphlets**: small, induced subgraphs of a fixed size (usually up to 5 nodes) that represent all possible non-isomorphic configurations of nodes and edges in a graph. Unlike motifs, graphlets don't focus on the frequency of appearance but rather on the specific structure of the subgraphs
+
+![image](https://github.com/user-attachments/assets/69eacfbe-8b5a-4f51-9a75-4a5aeb8da3a2)
+
+What is the difference between the two? A motif is a distinctly connected subgraph, while a graphlet describes a node’s position in the motif. For example, if you look at motif 1, you can observe that it consists of three nodes and two relationships. With motifs, you only count how often this pattern occurs in a network. On the other hand, you can observe there are three options for a node position in this motif 1; therefore, there are three graphlets available. Motifs are used to characterize a network structure (Kim et al., 2011), while graphlets come in handy when you want to describe a local neighborhood of a node (Pržulj et al., 2004).
+
+#### Betweenness centrality
+
+An graph algorithm used to measure the importance or influence of a node in a network by calculating how often that node appears on the shortest paths between other nodes. It highlights nodes that serve as bridges or connectors within the network, playing a key role in the flow of information or resources.
+
+1. Projecting the in-memory graph that describes the follower network and includes all the precalculated node features
+
+```cypher
+CALL gds.graph.project('knnExample','User', 'FOLLOWS',
+ {nodeProperties:['tweetCount', 'retweetRatio', 'timeToRetweet', 'inDegree',
+  'outDegree', 'friendCount', 'graphlet5', 'graphlet8', 'graphlet11']})
+```
+
+2. Mutating the betweenness centrality algorithm
+
+```cypher
+CALL gds.betweenness.mutate('knnExample', {mutateProperty:'betweenness'})
+```
+
+#### Closeness centrality
+
+It is a measure that indicates how close a node is to all the other nodes in the network. The algorithm starts by calculating the shortest paths from a node to all the other nodes in the network. Once the shortest paths are calculated, the algorithm sums the distance to all the other nodes. By default, it returns an inverse of the distance sum so that a higher score means that a node has a higher closeness centrality rank. One can interpret closeness as the potential ability to reach all the other nodes as quickly as possible.
+
+1. Mutating the closeness centrality algorithm
+
+```cypher
+CALL gds.closeness.mutate('knnExample',
+  {mutateProperty:'closeness', useWassermanFaust: true})
+```
+
+### 8.2 Constructing the nearest neighbor graph
+
+After manually extracting features (above), the second step is to group or cluster users into segments
+
+#### Evaluating features
+
+1. We need to store the mutated properties to the database
+
+```cypher
+CALL gds.graph.writeNodeProperties('knnExample',
+  ['betweenness', 'closeness'])
+```
+
+2. Identifying the five most frequently correlating pairs of features
+
+```cypher
+WITH ['tweetCount', 'retweetRatio', 'timeToRetweet', 'friendCount',
+      'inDegree', 'outDegree', 'graphlet5', 'graphlet8',
+      'graphlet11', 'closeness', 'betweenness'] AS features
+MATCH (u:User)
+UNWIND features as feature1
+UNWIND features as feature2                                      #1
+WITH feature1,
+     feature2,
+     collect(u[feature1]) as vector1,
+     collect(u[feature2]) as vector2
+ 
+WHERE feature1 < feature2                                        #2
+RETURN feature1,
+       feature2,
+       gds.similarity.pearson(vector1, vector2) AS correlation   #3
+ORDER BY correlation DESC LIMIT 5
+```
+
+1. Uses two UNWINDs to compare each feature to all the others
+
+2. Avoids comparing a feature with itself and removes duplicates
+
+3. Calculates the correlation
+
+![image](https://github.com/user-attachments/assets/eb8b1118-0038-40a6-a083-1ecad0acb008)
+
+It appears that some of the features are highly correlated. For example, the friendCount highly correlates with graphlet5, graphlet11, and betweenness features. Also, the graphlet8 variable correlates with the outgoing degree. To remove some of the highly correlated pairs of features, you will ignore the friendCount, graphlet8, and graphlet5 features from your segmentation process.
+
+3. Mutating the hashtag co-occurrence network to the in-memory graph
+
+```cypher
+WITH ['tweetCount', 'retweetRatio', 'timeToRetweet','inDegree',
+      'outDegree', 'graphlet11', 'closeness', 'betweenness'] AS features
+MATCH (u:User)
+UNWIND features as feature
+WITH feature,
+     apoc.agg.statistics(u[feature],
+                        [0.5,0.75,0.9,0.95,0.99]) as stats
+RETURN feature,
+       round(stats.min,2) as min,
+       round(stats.max,2) as max,
+       round(stats.mean,2) as mean,
+       round(stats.stdev,2) as stdev,
+       round(stats.`0.5`,2) as p50,
+       round(stats.`0.75`,2) as p75,
+       round(stats.`0.9`,2) as p90,
+       round(stats.`0.95`,2) as p95,
+       round(stats.`0.99`,2) as p99
+```
+
+<img width="493" alt="image" src="https://github.com/user-attachments/assets/5cd4466c-a205-4cd9-baa9-acedfbc8594e">
+
+The first thing we can noticed is that more than 95% of users have a graphlet11 count of 0. We could drop the graphlet11 feature due to its low variance; however, will keep it in this example case. The closeness centrality scores range from 0.0 to 0.25 with an average of 0.04. On the other hand, the betweenness centrality is not normalized, so the scores are much higher, as it ranges from 0.0 to nearly 200,000.
+
+### 8.3 User segmentation with the community detection algorithm
+
+The last step in the user segmentation process is to execute a community detection algorithm to identify groups or segments of users.
+
+The Louvain algorithm can be used to detect communities or segments of users by grouping densely connected nodes in the network. Unlike the label propagation algorithm (LPA), which serves a similar function, Louvain uses a different mathematical approach to identify these communities
+
+Here is how Louvian can be used:
+
+```cypher
+CALL gds.louvain.mutate('knnExample',
+  {relationshipTypes:['SIMILAR'], mutateProperty:'userSegmentation'})
+```
+
+
+
+
+
+
+
+
+
 
 
 

From baab9003628fc82624a80aea8c34f8b01d755caa Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 30 Sep 2024 23:03:39 +0900
Subject: [PATCH 442/638] Update 2024-09-30-day-273.md

---
 _posts/2024-09-30-day-273.md | 42 ++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-09-30-day-273.md b/_posts/2024-09-30-day-273.md
index 4818b3b94..7716838b3 100644
--- a/_posts/2024-09-30-day-273.md
+++ b/_posts/2024-09-30-day-273.md
@@ -1,11 +1,13 @@
 ---
 layout: post
-title: (Day 273)
-categories: [theory,gnn]
+title: (Day 273) First stream on youtube and finishing chapter 9 of the graph algs book
+categories: [theory,gnn,applying-knowledge]
 ---
 
 # Hello :) Today is Day 273!
 A quick summary of today:
+* covered ch. 9 of [Graph Algs for DS](https://learning.oreilly.com/library/view/graph-algorithms-for/9781617299469/)
+* did my [1st youtube stream](https://www.youtube.com/watch?v=2h8c7rnWUXY)
 
 ## Ch. 4 Exploratory graph analysis
 
@@ -617,24 +619,60 @@ CALL gds.louvain.mutate('knnExample',
   {relationshipTypes:['SIMILAR'], mutateProperty:'userSegmentation'})
 ```
 
+## Ch. 9 Node embeddings and classification
 
+**Adjacency matrix**
 
+![image](https://github.com/user-attachments/assets/a607b8bf-f196-4f79-8795-167fcccf2e83)
 
+Suppose you want to train a machine learning model and, somehow, use the network structure information as an input feature. Let’s say you use an adjacency matrix as an input. There are a couple of problems with this approach:
 
+* There are too many input features
+* Machine learning models are dependent on the size of the graph
+* Overfitting can become a problem
 
+If we add or remove a single node from the graph, the size of the adjacency matrix changes and the model is no longer functional, as there is a different number of input features
 
+In practice, we would want to embed a node’s local representation to compare nodes with similar neighborhood topology, instead of using all relationships between nodes as a feature input. Node embedding techniques try to solve these issues by learning lower-dimensional node representation for any given network. The learned node representations or embeddings should automatically encode the network structure so that the similarity in the embedding space approximates the similarity in the network. _A key message is that the node representations are learned instead of manually engineered._ We can achieve this through node embedding models.
 
+A node embedding model takes the high-dimensional representation of a graph as an input and outputs a lower-dimensional representation. 
 
+![image](https://github.com/user-attachments/assets/6b514e61-7226-4d2b-b15c-9ba85339cf35)
 
+### 9.1 Node embedding models
 
+Node embedding models aim to produce lower-dimensional representations of nodes, while preserving network structure information. These lower-dimensional representations can then be used as feature inputs for various machine learning tasks, such as node classification, link prediction, and community detection, thereby simplifying the computational complexity and potentially improving the performance of models.
 
+#### Homophily vs. structural roles approach
 
+A common approach is to represent nodes in the embedding space so that neighboring nodes in the graph are close in the embedding space - this is designed under the **homophily assumption** that connected nodes tend to be similar or have similar labels in a downstream machine learning workflow
 
+![image](https://github.com/user-attachments/assets/9b8feaaa-cfc3-4b03-902f-a42e46c1b989)
 
+**Structural roles approach**
 
+![image](https://github.com/user-attachments/assets/90eb226f-081c-4118-b288-e1ba15a799bf)
 
+The node embedding models used depends on the downstream task you need to complete
 
+#### Inductive vs. transductive embedding models
 
+The difference between inductive and transductive node embedding models is in their ability to encode new unseen nodes during training
 
+_When dealing with a **transductive node embedding algorithm**, you cannot calculate embeddings for nodes not seen during the initial embedding calculation. You can consider transductive models as creating a vocabulary during initial computation, where the key of the vocabulary represents a node and its value represents the embedding. If a node was not seen during the initial computation, it is not present in the vocabulary; hence, you cannot simply retrieve the embeddings for the new unseen nodes. If you want to calculate the embeddings for the new nodes, you must calculate the embeddings for the whole graph, meaning all the previously observed nodes as well as the new nodes. Since the embeddings might change for existing nodes, you must also retrain the classification model._
 
+_On the other hand, **inductive node embedding models** can calculate embeddings for unseen nodes during the initial computation. For example, you can train a model based on the initial computation of node embeddings. When a new node is introduced, you can calculate the embedding for the new node without recalculating embeddings for the whole graph. Likewise, you don’t need to retrain the classification model for every new node. Encoding previously unseen nodes is a great advantage when dealing with growing or multiple separate graphs. For instance, you could train a classification model on a single graph and then use it to predict node labels for nodes of different separate graphs._
 
+### 9.2 and 9.3 Node classification task
+
+This part of the chapter introduced a task to predict the language of new streams on Twitch.tv. The premise is that if streams share audiences then we can predict the language of upcoming streamers. 
+
+I covered this part on stream today: [https://www.youtube.com/watch?v=2h8c7rnWUXY](https://www.youtube.com/watch?v=2h8c7rnWUXY)
+
+---
+
+I guess something else I can do on stream is continue with the Neo4j courses which are publicly free, and I don't have to worry about showing content that's behind a paywall. 
+
+That is all for today!
+
+See you tomorrow :)

From ffb8347297cf5796e95eac470ff36ac03bee42f8 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 1 Oct 2024 21:19:41 +0900
Subject: [PATCH 443/638] Create 2024-10-01-day-274.md

---
 _posts/2024-10-01-day-274.md | 302 +++++++++++++++++++++++++++++++++++
 1 file changed, 302 insertions(+)
 create mode 100644 _posts/2024-10-01-day-274.md

diff --git a/_posts/2024-10-01-day-274.md b/_posts/2024-10-01-day-274.md
new file mode 100644
index 000000000..98eb20c38
--- /dev/null
+++ b/_posts/2024-10-01-day-274.md
@@ -0,0 +1,302 @@
+---
+layout: post
+title: (Day 274) Finished the Graph Algorithms for Data Science book + stream
+categories: [theory,gnn,applying-knowledge]
+---
+
+# Hello :) Today is Day 274!
+A quick summary of today:
+* covered the last chapters of the graph algs for DS book
+* did a stream from 6 to ~7.30pm [here](https://youtu.be/yQM2EtERCAw)
+
+## Ch. 10 Link prediction ([link](https://learning.oreilly.com/library/view/graph-algorithms-for/9781617299469/))
+
+![image](https://github.com/user-attachments/assets/dd34cdcc-7d06-4fff-8c4c-1a0e8e675c19)
+
+Over time people might make new friends and cut ties with some. In the above case, Luke and Ravij might become friends given their common friend Surya. This is one case of a link prediction tasks.
+
+Another case would be in drugs and symptoms
+
+![image](https://github.com/user-attachments/assets/ff2e4949-83f7-4c94-8238-d33106b765dc)
+
+### 10.1 Link Prediction Workflow
+
+Networks often follow certain organizing principles, either intentionally or unintentionally, which can be leveraged to predict the likelihood of new links forming between node pairs. For instance, in a social network, similar-age people might be more likely to become friends. However, solely considering personal characteristics can be limiting, as relationships may depend on additional factors like shared social circles. Nodes with more common connections (ie. friends) or higher degrees of connectedness are more likely to form links.
+
+#### Key Approaches to Link Prediction:
+1. **Node Property Combination**: This involves using attributes like the age difference between nodes, node degrees, or cosine similarity of node embeddings. The properties can be concatenated or otherwise combined to improve prediction
+2. **Distance-Based Metrics**: Features such as the shortest path between nodes (number of hops) indicate how close two nodes are, with shorter distances suggesting a higher probability of a future connection
+3. **Neighborhood Overlap**: The more overlapping connections two nodes have, the higher the likelihood of them forming a link. For example, common friends can be quantified using the Jaccard similarity index
+
+![image](https://github.com/user-attachments/assets/b6940c2b-721b-4f3f-a5bc-fd8c4ae5e353)
+
+#### Prediction Methods:
+- **Unsupervised Approach**: Calculate link prediction metrics between node pairs and rank the most likely future links
+- **Classification Model**: Train a classification model on link prediction metrics. This method can identify patterns and assign weights to different metrics, capturing more complex relationships between node pairs that might be missed in an unsupervised approach
+
+Going back to being a DS at Twitch. If two channels share a significant audience, this information is used to provide recommendations to users in the "Users of This Channel Also Watch" section. One way to improve these recommendations is by predicting future audience overlap between channels and using these predictions to enhance recommendations. This would allow recommending both channels with existing overlaps and channels likely to share audiences in the future.
+
+![image](https://github.com/user-attachments/assets/98974499-b7a4-4b41-8a39-078b3d9391cb)
+
+The network of Twitch channels can be represented as a graph, where edges indicate audience overlap. Solid lines represent existing shared audiences and are used to populate current recommendations. As a data scientist, you can leverage these relationships to predict future audience overlap, which would then power the recommendation engine.
+
+#### Building a link prediction model to recommend Twitch channels
+
+![image](https://github.com/user-attachments/assets/6331d1ec-c6a9-4792-b7bc-4d5a986d62ae)
+
+### 10.2 Dataset split
+
+* Time-based split
+
+![image](https://github.com/user-attachments/assets/a741ad07-5b79-4eda-8e1f-a725cbd7b763)
+
+* Random split
+
+![image](https://github.com/user-attachments/assets/bd0ad63e-bd18-4cf6-a78f-4b69d65a2d1e)
+
+* Negative samples
+
+When training a binary classifier like the link prediction model, we should include both positive and negative examples in the training and test sets. Without negative examples, the model cannot learn to differentiate between the two outputs and might produce inaccurate predictions.
+
+For the coding exercises
+
+1. Create a feature set
+
+```cypher
+run_query("""
+MATCH (s1:Stream)-[:SHARED_AUDIENCE]->(s2:Stream)
+WITH s1, s2
+WHERE rand() <= 0.9
+MERGE (s1)-[:FEATURE_REL]->(s2);
+""")
+```
+
+2. Create train/test set from the remaining 10%
+
+```cypher
+train_test_size = run_query("""
+MATCH (s1)-[:SHARED_AUDIENCE]->(s2)
+WHERE NOT EXISTS {(s1)-[:FEATURE_REL]->(s2)}
+MERGE (s1)-[r:TEST_TRAIN]->(s2)
+RETURN count(r) AS result;
+""")
+```
+
+3. Create negative train/test
+
+```cypher
+run_query("""
+MATCH (s1:Stream),(s2:Stream)
+WHERE NOT EXISTS {(s1)-[:SHARED_AUDIENCE]-(s2)}
+      AND s1 < s2
+      AND rand() > 0.9
+WITH s1,s2
+LIMIT 13082
+MERGE (s1)-[:NEGATIVE_TEST_TRAIN]->(s2);
+""")
+```
+
+### 10.3 Network feature engineering
+
+Next, we will produce network features that capture the closeness or similarity of pairs of nodes in the network. The idea is that the closer or more similar a pair of nodes are given the network metrics, the more likely they are to form a future connection. The future connections will then be used to provide better recommendations to Twitch users.
+
+#### Network distance
+
+![image](https://github.com/user-attachments/assets/5eb0118a-2050-4adc-89f9-1e23f9df17a8)
+
+The idea behind the network distance is that the closer the two nodes are in the network, the more likely they are to form future connections
+
+_Calculating the network distance between pairs of nodes in the train and test sets_
+
+```cypher
+run_query("""
+MATCH (s1)-[r:TEST_TRAIN|NEGATIVE_TEST_TRAIN]->(s2)    #1
+MATCH p = shortestPath((s1)-[:FEATURE_REL*]-(s2))      #2
+WITH r, length(p) AS networkDistance                   #3
+SET r.networkDistance = networkDistance                #4
+""")
+```
+
+1. Matches all the pairs of nodes connected with the TEST_TRAIN or NEGATIVE_TEST_TRAIN relationships
+2. Identifies the shortest path between the pairs of nodes. The shortest path is constrained to be only allowed to traverse the FEATURE_REL relationships to prevent data leakage.
+3. Calculates the length of the shortest paths using the length() function
+4. Stores the network distance result as the relationship property
+
+#### Preferential attachment
+
+Another popular metric used in link prediction is preferential attachment. Preferential attachment is an underlying organizing principle occurring in real-world networks, where nodes with a higher number of relationships are more likely to make new relationships
+
+![image](https://github.com/user-attachments/assets/a3338b97-aee1-497f-b405-4ff806a04fbb)
+
+```cypher
+run_query("""
+MATCH (s1:Stream)-[r:TEST_TRAIN|NEGATIVE_TEST_TRAIN]->(s2)
+WITH r, count{ (s1)-[:FEATURE_REL]-() } *
+        count{ (s2)-[:FEATURE_REL]-() } AS preferentialAttachment
+SET r.preferentialAttachment = preferentialAttachment
+""")
+```
+
+#### Common neighbors
+
+The more common neighbors two nodes have, the higher the chance is of a link forming in the future
+
+```cypher
+run_query("""
+MATCH (s1:Stream)-[r:TEST_TRAIN|NEGATIVE_TEST_TRAIN]->(s2)
+OPTIONAL MATCH (s1)-[:FEATURE_REL]-(neighbor)-[:FEATURE_REL]-(s2)
+WITH r, count(distinct neighbor) AS commonNeighbor
+SET r.commonNeighbor = commonNeighbor
+""")
+```
+
+#### Adamic-Adar index
+
+The idea behind the Adamic-Adar index is that the smaller the node degree common neighbors between a pair of nodes have, the more likely it is that they will form a connection in the future
+
+![image](https://github.com/user-attachments/assets/8835a214-ef21-49e0-a62b-7d242bd23555)
+
+In case B, the nodes A and B are more likely to become friends, than in case A. 
+
+```cypher
+run_query("""
+MATCH (s1:Stream)-[r:TEST_TRAIN|NEGATIVE_TEST_TRAIN]->(s2:Stream)
+OPTIONAL MATCH (s1)-[:FEATURE_REL]-(neighbor)-[:FEATURE_REL]-(s2)
+WITH r, collect(distinct neighbor) AS commonNeighbors             
+UNWIND commonNeighbors AS cn
+WITH r, count{ (cn)-[:FEATURE_REL]-() } AS neighborDegree         
+WITH r, sum(1 / log(neighborDegree)) AS adamicAdar                
+SET r.adamicAdar = adamicAdar;                                    
+""")
+```
+
+#### Clustering coefficient of common neighbors
+
+A clustering coefficient measures the connectedness of the neighbors of a particular node, with the values ranging from 0 to 1. A value of 0 indicates the neighboring nodes have no connections with each other. On the other hand, a value of 1 indicates the network of neighbors forms a complete graph, where all the neighbors are connected.
+
+```cypher
+run_query("""
+MATCH (s1:Stream)-[r:TEST_TRAIN|NEGATIVE_TEST_TRAIN]->(s2:Stream)
+OPTIONAL MATCH (s1)-[:FEATURE_REL]-(neighbor)-[:FEATURE_REL]-(s2)
+WITH r, collect(distinct neighbor) AS commonNeighbors,
+        count(distinct neighbor) AS commonNeighborCount             #1
+OPTIONAL MATCH (x)-[cr:FEATURE_REL]->(y)                            #2
+WHERE x IN commonNeighbors AND y IN commonNeighbors
+WITH r, commonNeighborCount, count(cr) AS commonNeighborRels
+WITH r, CASE WHEN commonNeighborCount < 2 THEN 0 ELSE               #3
+  toFloat(commonNeighborRels) / (commonNeighborCount *
+                 (commonNeighborCount - 1) / 2) END as clusteringCoefficient
+SET r.clusteringCoefficient = clusteringCoefficient                 #4
+""")
+```
+
+1. Identifies and counts the number of common neighbors between a pair of nodes
+2. Identifies all existing relationships between common neighbors
+3. Calculates the clustering coefficient by dividing the number of existing relationships by the number of potential relationships
+4. Stores the results as a relationship property
+
+After all the features, here is an example link with its features
+
+<img width="787" alt="image" src="https://github.com/user-attachments/assets/e6759a55-b933-40a1-95ed-5d763f90525b">
+
+
+### 10.4 Link prediction classification model
+
+1. Retrieving link prediction features and class output
+
+```python
+data = run_query("""
+MATCH (s1)-[r:TEST_TRAIN|NEGATIVE_TEST_TRAIN]->(s2)
+WITH r.networkDistance AS networkDistance,
+     r.preferentialAttachment AS preferentialAttachment,
+     r.commonNeighbor AS commonNeighbor,
+     r.adamicAdar AS adamicAdar,
+     r.clusteringCoefficient AS clusteringCoefficient,
+     CASE WHEN r:TEST_TRAIN THEN 1 ELSE 0 END as output
+RETURN networkDistance, preferentialAttachment, commonNeighbor,
+       adamicAdar, clusteringCoefficient, output
+""")
+```
+
+<img width="966" alt="image" src="https://github.com/user-attachments/assets/7b5e5bc6-bca8-41cd-bdb1-eee502fc4a04">
+
+The `output` column is is a 0 or 1. Positive examples are tagged with the TEST_TRAIN relationship type and are represented with a value of 1, while the negative examples are marked with NEGATIVE_TEST_TRAIN and are represented as 0.
+
+<img width="1091" alt="image" src="https://github.com/user-attachments/assets/193a9036-2153-41c2-b84e-c881cb6e5433">
+
+2. Splitting the train and test sets and training the link prediction model
+
+```python
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import RandomForestClassifier
+ 
+X = data.drop("output", axis=1)
+y = data["output"].to_list()
+ 
+X_train, X_test, y_train, y_test = train_test_split(
+  X, y, test_size=0.2, random_state=0)
+rfc = RandomForestClassifier()
+rfc.fit(X_train, y_train)
+```
+
+3. Generating the classification report
+
+<img width="590" alt="image" src="https://github.com/user-attachments/assets/a3f7f1c2-412f-4b8b-8b60-6143540dbfd4">
+
+4. Evaluating feature importance
+
+<img width="324" alt="image" src="https://github.com/user-attachments/assets/ead34d27-114b-4384-9f3f-4b0d0153d3f6">
+
+## Ch. 11 Knowledge graph completion
+
+The difference between link prediction and completion is that the first is a workflow to predict future links, while the latter deals with predicting missing links
+
+The link prediction features used in chapter 10 do not differentiate between various node or relationship types. For example, the number of common neighbors does not differentiate between different relationship or node types. Therefore, the link prediction features used in chapter 10 work best with monopartite or homogeneous graphs
+
+### 11.1 Knowledge graph embedding model
+
+#### Triple
+
+Knowledge graph embedding models use triples to describe graphs. A triple consists of two nodes, known as a head (h) and tail (t), and a labeled-directed relationship (r).
+
+![image](https://github.com/user-attachments/assets/ef55bf57-24c6-4ec8-b2c7-389c61787437)
+
+#### TransE
+
+The objective of the TransE method is to calculate low-dimensional vector representations, also known as embeddings, for all the nodes and relationships in the graph. The TransE method is frequently used to demonstrate knowledge graph embeddings, as it is simple to illustrate and relatively cheap to calculate.
+
+![image](https://github.com/user-attachments/assets/0748f87b-3076-4129-ba28-78ff1c5cdb84)
+
+The TransE method tries to produce embeddings so that for every triple in the training set, it minimizes the distance between the sum of the head and the relationship to the tail embedding. This optimization score can be written as h + r ≈ t. On the other hand, if a relationship between the head and tail node does not exist, then the sum of head and relation embedding should not be close to the tail (h + r != t).
+
+**Limitations**
+
+1. Symmetric Relationships
+TransE struggles with **symmetric relationships**, where two entities are mutually related. For example:
+- **Tomaž, SIBLING, Blaž** and **Blaž, SIBLING, Tomaž**.
+  
+In TransE, relationships are represented as vectors. The issue arises because the same relationship vector cannot point in opposite directions, meaning that TransE cannot properly encode relationships that are symmetric by nature.
+
+2. Composition Relationships
+TransE can handle **composition relationships**, where one relationship can be derived by combining others. For example:
+- **John, MOTHER, Alicia**, **Alicia, SPOUSE, Michael**, leads to **John, FATHER, Michael**.
+
+In this case, the vectors for these relationships can be composed to represent such patterns, so TransE supports composite relations.
+
+3. 1-to-N Relationships
+TransE has significant limitations with **1-to-N relationships**, where one entity is related to multiple others, like:
+- **Surya, FRIEND, Rajiv** and **Surya, FRIEND, Jane**.
+
+To tackle these limitations, later PairRE was developed and it is capable of encoding symmetry, composition, and 1-to-N relationships.
+
+## [Streamed today](https://youtu.be/yQM2EtERCAw)
+
+This is a resource that I need to read again - [Accelerate Fraud Detection with Graph Databases](https://neo4j.com/whitepapers/accelerate-fraud-detection-graph-databases/)
+
+And I covered part 1 of [this blog series](https://neo4j.com/developer-blog/exploring-fraud-detection-neo4j-graph-data-science-summary/) going over using neo4j for fraud detection
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 407dc138901b54dfbfa19463e7797ac49ece7b62 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 1 Oct 2024 22:05:14 +0900
Subject: [PATCH 444/638] Update projects.md

---
 _pages/projects.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/_pages/projects.md b/_pages/projects.md
index aa82f1533..f7fd3557a 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -76,6 +76,33 @@ title: Projects
     <p><b>Tech: Docker, Mage orchestration, Google Cloud Platform, Terraform, dbt, Looker</b></p>
     <a href="https://github.com/divakaivan/lending-club-data-pipeline" target="_blank">View Project</a>
   </div>
+
+  <h2>Other</h2>
+  <div class="project-item">
+    <h4><a href="https://www.kaggle.com/divakaivan12/code" target="_blank">Notebook Expert on Kaggle</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://www.kaggle.com/code/divakaivan12/the-full-story-behind-multicollinearity" target="_blank">Diving into the full story behind multicollinearity</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://www.kaggle.com/code/divakaivan12/neural-network-epoch-by-hand" target="_blank">Doing a neural network epoch by hand</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/text2chart" target="_blank">text2chart - transforming natural language to charts</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/pdf-rag-from-scratch" target="_blank">PDF RAG from scratch</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/taxi-demand-video-models-paper" target="_blank">Using Video Generation Models for Taxi OD Demand Matrix Predictions</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/db2chat" target="_blank">db2chat - chat with your (SQLite) database</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/classify_hangul" target="_blank">CV model to classify Hangul characters</a></h4>
+  </div>
+  
 </div>
 
 <style>
@@ -96,6 +123,10 @@ title: Projects
     margin-top: 0;
   }
 
+  .project-item h4 {
+    margin-top: 0;
+  }
+
   .project-item a {
     display: inline-block;
     margin-top: 10px;

From aa434abd86567681a781d791dadbf722a5d85eab Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 1 Oct 2024 22:12:16 +0900
Subject: [PATCH 445/638] reorder projects

---
 _pages/projects.md | 48 +++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index f7fd3557a..d5a65fe0c 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -5,6 +5,30 @@ title: Projects
 ---
 
 <div id="projects">
+  <div class="project-item">
+    <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
+    <p>
+      <li>An AI-driven solution for real-time credit card fraud detection using MLOps techniques</li>
+      <li>Fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring</li>
+      <li>High fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models</li>
+    </p>
+    <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
+    <a href="https://github.com/divakaivan/kb_project" target="_blank">View on GitHub</a>
+  </div>
+
+  <div class="project-item">
+    <h3>Transaction Stream Data Engineering Pipeline</h3>
+    <p>
+      <li>Generate transaction data via Stripe's API</li>
+      <li>Stream data using Apache Kafka and process it in real-time with PySpark Structured Streaming</li>
+      <li>Store processed data in PostgreSQL</li>
+      <li>Manage data transformations and modeling using dbt</li>
+      <li>Visualize data using Grafana</li>
+    </p>
+    <p><b>Tech: PostgreSQL DB, Kafka, PySpark, dbt, Grafana</b></p>
+    <a href="https://github.com/divakaivan/transaction-stream-data-pipeline" target="_blank">View Project</a>
+  </div>
+  
   <div class="project-item">
     <h3>Your Personal Finance Voice Assistant</h3>
     <p>
@@ -16,17 +40,6 @@ title: Projects
     <p><b>Tech: SQLite, OpenAI, HuggingFace, LlamaIndex, Azire Pheonix monitoring, Streamlit</b></p>
     <a href="https://github.com/divakaivan/personal_finance_assistant" target="_blank">View on GitHub</a>
   </div>
-  
-  <div class="project-item">
-    <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
-    <p>
-      <li>An AI-driven solution for real-time credit card fraud detection using MLOps techniques</li>
-      <li>Fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring</li>
-      <li>High fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models</li>
-    </p>
-    <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
-    <a href="https://github.com/divakaivan/kb_project" target="_blank">View on GitHub</a>
-  </div>
 
   <div class="project-item">
     <h3>Glaswegian Audio Dataset and ASR model</h3>
@@ -50,19 +63,6 @@ title: Projects
     <a href="https://github.com/divakaivan/insurance-fraud-mlops-pipeline" target="_blank">View Project</a>
   </div>
 
-  <div class="project-item">
-    <h3>Transaction Stream Data Engineering Pipeline</h3>
-    <p>
-      <li>Generate transaction data via Stripe's API</li>
-      <li>Stream data using Apache Kafka and process it in real-time with PySpark Structured Streaming</li>
-      <li>Store processed data in PostgreSQL</li>
-      <li>Manage data transformations and modeling using dbt</li>
-      <li>Visualize data using Grafana</li>
-    </p>
-    <p><b>Tech: PostgreSQL DB, Kafka, PySpark, dbt, Grafana</b></p>
-    <a href="https://github.com/divakaivan/transaction-stream-data-pipeline" target="_blank">View Project</a>
-  </div>
-
   <div class="project-item">
     <h3>Lending Club Data Engineering Pipeline</h3>
     <p>

From 3662bb8b78bf313bff9911bc32c1f93631c4de68 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 1 Oct 2024 22:18:50 +0900
Subject: [PATCH 446/638] Update about.md

---
 _pages/about.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/_pages/about.md b/_pages/about.md
index 2981e78f0..c24b5d56f 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -10,4 +10,8 @@ On the 2nd of January 2024, I embarked on a, what I thought would be, 50 day jou
 
 On Day 100 I knew it was not over, so I continued.
 
-After a few days I decided to go for the full year.
\ No newline at end of file
+After a few days I decided to go for the full year.
+
+My studies have become a cover of math, stats, data engineering, classical machine learning, modern deep learning architectures, AI assistants, MLOps systems. So far this journey has been about the technical side of things, but crucially also about thinking as DataEng/DataScientist/MachineLearningEng/DataAnalyst and developing the soft skills needed for any of these roles. Also about learning how I learn, but others too which I will summarise by the end of this 1-year journey. 
+
+**These days I stream part of my daily studied on [my youtube channel](https://www.youtube.com/@ilearnthings123) - Monday to Thursday 10AM GMT+1/6PM KST**

From ebeb43d7bef4138c73301b64185dae57325c80c7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 3 Oct 2024 00:23:54 +0900
Subject: [PATCH 447/638] Create 2024-10-02-day-275.md

---
 _posts/2024-10-02-day-275.md | 182 +++++++++++++++++++++++++++++++++++
 1 file changed, 182 insertions(+)
 create mode 100644 _posts/2024-10-02-day-275.md

diff --git a/_posts/2024-10-02-day-275.md b/_posts/2024-10-02-day-275.md
new file mode 100644
index 000000000..cf1aee903
--- /dev/null
+++ b/_posts/2024-10-02-day-275.md
@@ -0,0 +1,182 @@
+---
+layout: post
+title: (Day 275) Finding new books to read + stream
+categories: [theory,math,nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 275!
+A quick summary of today:
+* showed my mini paper to my lab professor
+* read a bit of Deep Learning by Bishop&Bishop
+* found many new books to read
+* next steps in the movie review PEFT LLM project
+* [streamed again](https://www.youtube.com/watch?v=bez0EsQ30jc&t=246s)
+
+## Firstly about the [paper](https://github.com/divakaivan/taxi-demand-video-models-paper)
+
+I talked about it on stream, but tldr is: I had the paper living in my repo for 2-3 months and I read a lot about taxi demand matrices and predicting them but when I finished writing the literature review, and briefly the rest of the 'paper' I kind of lost hope that this will be a good paper so I gave up on submitting it and talking more about it with my lab professor. But after 2-3 months, my SO finally convinced me to send it so that I can get help and continue writing it. 
+
+## [Movie review project update](https://huggingface.co/divakaivan/llama3-finetuned-reviewer-kr)
+
+For this project I had LoRA fine-tuned llama3 and a t5 model. Among the two, llama3 performed way better and the code uses the unsloth library which specialises in faster fine-tuning. So going forward as per my professor's suggestion I will select a few models, along with a few base prompts and compare the performance across all combinations. A reminder, this movie review project is a cover-up for the actual project idea which I cannot share at the moment but is something similar to this idea. 
+
+## [DL by Bishop&Bishop](https://www.bishopbook.com/)
+
+Talk about it on stream but unfortunately I did not take notes and as I was in the lab I did not want to put my phone to film myself reading as I did not want to film my labmates. I read chapter 5: single layer networks for classification, chapter 6: deep neural nets, and then skimmed through some of the other chapters. 
+
+## New books
+
+Thanks to Bishop&Bishop I was motivated to go and find more books to read. But I wanted to read more foundation knowledge books - math, stats, probability (again). I ended up finding a lot:
+
+* [Before Machine Learning](https://www.amazon.com/dp/B0CLKVHK7Y?binding=paperback&ref=dbs_dp_rwt_sb_pc_tpbk) - 3 book series on linear algebra, calculus and stats&probs
+* [The 100-page ML book](https://themlbook.com/) and [MLE](https://leanpub.com/MLE) by Andryi Burkov 
+
+Not foundational knowledge, but seem interesting:
+* [Machine Learning for High-Risk Applications](https://learning.oreilly.com/library/view/machine-learning-for/9781098102425/)
+* [Machine Learning for Financial Risk Management with Python](https://learning.oreilly.com/library/view/machine-learning-for/9781492085249/)
+
+## After stream I rested for a bit, and read the 1st book of the Before Machine Learning series
+
+It is *very* beginner friendly and written in a casual way.
+
+[Here is the link to it on o'Reilly](https://learning.oreilly.com/library/view/before-machine-learning/9781836208952/). 
+
+## Chapter 3
+
+### What Is a Vector?
+
+A vector is essentially a list of numbers where the order of the elements matters, often used in machine learning to represent data points like height and weight. For instance, a two-dimensional vector could represent a student as (height, weight). Each vector has both direction and magnitude, with its origin typically at (0,0) in a coordinate system.
+
+<img width="255" alt="image" src="https://github.com/user-attachments/assets/7a29a5c2-55d3-4f5c-955c-3b1eb1c5f5f0">
+<img width="306" alt="image" src="https://github.com/user-attachments/assets/dd1fda3d-2070-4dbf-b144-82d138d9278b">
+
+
+Vectors can have multiple dimensions; for example, a three-dimensional vector might be represented as (x, y, z). Understanding vectors involves visualizing their components: the starting point (origin), direction (based on coordinates), and magnitude (the length of the vector).
+
+<img width="292" alt="image" src="https://github.com/user-attachments/assets/28aa4476-807f-4430-bd99-0ece09405748">
+
+Vectors can describe real-world phenomena like wind by representing direction and speed. They also allow for mathematical operations, such as addition, which can represent paths or movements. Overall, vectors are foundational elements in mathematics and have wide applications in various fields.
+
+### Vector Addition
+
+<img width="282" alt="image" src="https://github.com/user-attachments/assets/2410ce2b-09eb-4cc5-b9d4-c487bbcde341">
+<img width="347" alt="image" src="https://github.com/user-attachments/assets/93b9ea63-af61-477f-beaf-0989fab316df">
+<img width="359" alt="image" src="https://github.com/user-attachments/assets/c6a7acc7-2747-4672-96ba-633d533e480d">
+
+### Scalar Multiplication
+
+λ ∈ ℝ
+
+A scalar is any real number, such as π, which can be used to scale vectors. The symbol ∈ means "belongs to," and ℝ represents the set of real numbers. When multiplying a vector by a scalar (λ), there are four possible outcomes based on λ's value. If λ is greater than 1, the vector stretches while maintaining direction. If 0 < λ < 1, the vector shrinks without changing direction. If λ is less than -1, the vector stretches but reverses direction. If -1 < λ < 0, the vector shrinks and reverses direction. Multiplying a vector by a scalar alters its length and, in some cases, its direction. Finally, vector multiplication has not yet been covered.
+
+### The Dot Product
+
+The dot product is a method of multiplying two vectors that results in a scalar value. It is calculated by multiplying corresponding elements of two vectors and summing the results. For example, given two vectors, v = (2, 3) and w = (2, 1), their dot product is 2 * 2 + 3 * 1 = 7.
+
+In linear algebra, the dot product has a geometric interpretation. It measures how much one vector aligns with another by projecting one vector onto the other and multiplying their magnitudes. The angle \(\theta\) between vectors determines this relationship, where the dot product \( \mathbf{v} \cdot \mathbf{w} = |\mathbf{v}| |\mathbf{w}| \cos(\theta) \).
+
+If two vectors point in the same direction, the dot product is positive; if they are perpendicular, the dot product is zero; if they point in opposite directions, it is negative.
+
+In machine learning, the dot product is used to measure similarity. For example, when recommending movies based on user preferences, you can compare movie vectors by calculating the dot product or using cosine similarity, which focuses on vector direction rather than magnitude. Cosine similarity is preferred when the magnitude is not crucial, as in text analysis or machine learning tasks. 
+
+Both the dot product and cosine similarity are valuable tools for comparing and manipulating vectors, with their applications depending on whether vector length or direction is more relevant.
+
+### The Vector Space
+
+A **vector space** is a structured set of vectors that follow specific rules (axioms) for operations like vector addition and scalar multiplication. These operations must produce results that remain within the vector space. For any collection of vectors to qualify as a vector space, they must adhere to these axioms, ensuring operations like vector addition and scalar multiplication behave as expected:
+
+<img width="715" alt="image" src="https://github.com/user-attachments/assets/d36962b0-a92a-418a-a76d-6235a6c54393">
+
+The **basis** of a vector space is a set of linearly independent vectors that can represent all vectors within that space through a combination of scalar multiples. Changing the basis provides a new perspective on the vectors, useful in different contexts such as machine learning, where altering the basis can reveal patterns or simplify computations. For example, changing the basis for data like house features can help better understand trends such as the total number of rooms or the balance between bedrooms and bathrooms.
+
+## Chapter 4: Matrices
+
+Maybe because I was exhausted, but the topic is math are there was *so much* over-the-top casual explanations that kind of confused me as to why are they there. I am not sure if this make sense, but in its attempts to explain things too simply it added maybe a bit too much abstractions and deviations from the main point that at some points I needed to re-read 😆 even though I know the material. 
+
+### Linear Transformations
+
+<img width="459" alt="image" src="https://github.com/user-attachments/assets/056ef628-348a-4ceb-b703-fc78bb59b553">
+
+L : ℝn  →  ℝm 
+
+It must satisfy:
+
+L(v + u) = L(v) + L(u)
+L(c ⋅ v) = c ⋅ L(v)
+
+where v and u are vectors that belon to ℝn, and c is a scalar that is a real number
+
+### The Eigen 'Stuff'
+
+The book tries, but [2blue1brown](https://www.youtube.com/watch?v=PFDu9oVAE-g&t=530s&pp=ygUXMmJsdWUxYnJvd24gZWlnZW52ZWN0b3I%3D)'s explanation is the best by far. 
+
+To be honest when I was reading this part, and after remembering 2blue1brown's channel, I remembered how great that youtube channel is and that it explains everything so nicely. 
+
+### Matrix decomposition
+
+![image](https://github.com/user-attachments/assets/306f3953-690a-44b3-a6d1-b301c021249a)
+
+Where:
+
+U represents a rotation (it preserves the angles and lengths of vectors)
+Σ is the scaling matrix (a diagonal matrix containing the singular values, which represent scaling factors)
+V^T is another rotation (the transpose of an orthogonal matrix V which can also be interpreted as a rotation or reflection. It defines the basis for the column space of the original matrix)
+
+This transformation applied to a circle represents the operation:
+
+![image](https://github.com/user-attachments/assets/fefa2d9a-3522-4b47-8fc3-240673b3d573)
+
+### Principal Component Analysis
+
+The principal component analysis will create a new set of axes called the principal axes, where we will project the data and get these so-called principal components. These are a linear combination of the original features that will be equipped with outstanding characteristics. Characteristics which are not only uncorrelated, but the first components also capture most of the variance in the data, which makes this methodology a good technique for reducing the dimensions of complex data sets.
+
+Start with some data:
+
+<img width="646" alt="image" src="https://github.com/user-attachments/assets/0733ad9d-9f83-4d02-b518-d578b9c5d310">
+
+Normalise it using the mean and st.d.:
+
+<img width="689" alt="image" src="https://github.com/user-attachments/assets/08df1117-8c16-4386-b710-455a3c1ea49c">
+
+If we imagine the table as a matrix A^T, multiply it by A and divide by N (8, normalising it) and we get:
+
+<img width="339" alt="image" src="https://github.com/user-attachments/assets/ed372de4-d42b-46ad-b3a0-df7da38cccb9">
+
+Next we need to find the eigenvectors and eigenvalues. That can be done through eigen decomposition
+
+Why?
+
+1. Eigenvectors define the principal components (the directions of max variance):
+   - eigenvectors represent the directions in which the data varies the most
+   - when you perform PCA, you aim to transform your data into a new coordinate system where the axes (called principal components) correspond to these directions of maximum variance
+   - the eigenvectors of the covariance matrix (or the correlation matrix) point in these directions of maximum variance
+   - these eigenvectors become the principal components in PCA, where each principal component is a linear combination of the original features, and they are orthogonal to each other
+
+2. Eigenvalues represent the magnitude of variance:
+   - eigenvalues correspond to the amount of variance (or the 'importance') along each principal component (eigenvector)
+   - larger eigenvalues indicate that the corresponding eigenvector (principal component) captures a larger amount of the data's variance
+   - by sorting the eigenvalues in descending order, we can rank the principal components by how much variance they explain. This helps determine which principal components are most important for dimensionality reduction
+
+<img width="621" alt="image" src="https://github.com/user-attachments/assets/485285fc-95c2-4932-b3bf-5667fd95ca70">
+
+P is a matrix with the eigenvectors, and this will be where we find our principal axes. It happens to be already sorted by eigenvalue magnitude. On the other hand, in the matrix Σ we have all the eigenvalues. These represent what can be called the explainability of the variance, how much of the variance present in the data is ”captured” by each component.
+
+The sum of the eigenvalues is 5, and by dividing each of them by 5 we find the individual percentage of variance that each distinct eigenvalue explains
+
+<img width="422" alt="image" src="https://github.com/user-attachments/assets/d0e36ad4-6e08-4114-aa06-5024fea25bc1">
+
+The 1st two components explain ~76% of the variance so we can use the two for as our principal components to reduce our dims. 
+
+<img width="242" alt="image" src="https://github.com/user-attachments/assets/3942d50e-c1a0-4daa-9990-ff16634c99fc">
+
+So now we just need to multiply our original 8x5 data matrix by this 5x2 matrix to get the resulting dimenson reduced feature set:
+
+<img width="424" alt="image" src="https://github.com/user-attachments/assets/976bf4ca-071e-4933-8f47-821e84c6e771">
+
+These components are also called latent or hidden variables; relationships that are hidden in the data and are the result of linear combinations.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 50868be7d28cca56bde176989ae7a85d4faccf52 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 3 Oct 2024 12:51:45 +0900
Subject: [PATCH 448/638] fix math format

---
 _posts/2024-10-02-day-275.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-02-day-275.md b/_posts/2024-10-02-day-275.md
index cf1aee903..6a9e58e5c 100644
--- a/_posts/2024-10-02-day-275.md
+++ b/_posts/2024-10-02-day-275.md
@@ -73,7 +73,7 @@ A scalar is any real number, such as π, which can be used to scale vectors. The
 
 The dot product is a method of multiplying two vectors that results in a scalar value. It is calculated by multiplying corresponding elements of two vectors and summing the results. For example, given two vectors, v = (2, 3) and w = (2, 1), their dot product is 2 * 2 + 3 * 1 = 7.
 
-In linear algebra, the dot product has a geometric interpretation. It measures how much one vector aligns with another by projecting one vector onto the other and multiplying their magnitudes. The angle \(\theta\) between vectors determines this relationship, where the dot product \( \mathbf{v} \cdot \mathbf{w} = |\mathbf{v}| |\mathbf{w}| \cos(\theta) \).
+In linear algebra, the dot product has a geometric interpretation. It measures how much one vector aligns with another by projecting one vector onto the other and multiplying their magnitudes. The angle theta between vectors determines this relationship, where the dot product v dot w = \mathbf{v} \mathbf{w} \cos(\theta) \).
 
 If two vectors point in the same direction, the dot product is positive; if they are perpendicular, the dot product is zero; if they point in opposite directions, it is negative.
 

From 78c9fc1d8c2070bfe04b65d2bcd35aef670fb393 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 3 Oct 2024 23:32:54 +0900
Subject: [PATCH 449/638] Create 2024-10-03-day-276.md

---
 _posts/2024-10-03-day-276.md | 107 +++++++++++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 _posts/2024-10-03-day-276.md

diff --git a/_posts/2024-10-03-day-276.md b/_posts/2024-10-03-day-276.md
new file mode 100644
index 000000000..a7cbf5c61
--- /dev/null
+++ b/_posts/2024-10-03-day-276.md
@@ -0,0 +1,107 @@
+---
+layout: post
+title: (Day 276) 4.5hr stream learning about intermediate neo4j queries
+categories: [gnn,applying-knowledge]
+---
+
+# Hello :) Today is Day 276!
+A quick summary of today:
+* [here is the stream](https://youtu.be/vA5JVrNazJA)
+* completed [Intermediate Cypher Queries](https://graphacademy.neo4j.com/courses/cypher-intermediate-queries/) course by Neo4j
+
+![461499389_1795582970975098_6880455522022025897_n](https://github.com/user-attachments/assets/7ae7200a-d85b-49e7-b9af-9488d4ea7520)
+
+After the stream my voice was exhausted 😆 I talked non-stop throughout the whole stream. Anyway ~ here is just a summary of what was covered in the course, but everything is in the video.
+
+# 01. Filtering Queries
+**15 Lessons**
+- Basic Cypher Queries
+- Testing Equality
+- Querying for Null Values
+- Checking for Null Values
+- Range Query
+- Testing List Inclusion
+- Testing Strings
+- Case Insensitive Search
+- Using the CONTAINS Predicate
+- Query Patterns and Performance
+- Testing Patterns
+- Profiling
+- Multiple MATCH Clauses
+- Retrieve Movies and Reviewers
+- OPTIONAL MATCH
+
+# 02. Controlling Results Returned
+**10 Lessons**
+- Ordering Returned Results
+- Ordering Results
+- Viewing the Ordered Results
+- Ordering Multiple Values
+- Limiting or Counting Results Returned
+- Limiting Results
+- Eliminating Duplicates
+- Map Projections to Return Data
+- Changing Results Returned
+- Conditionally Returning Data
+
+# 03. Working with Cypher Data
+**7 Lessons**
+- Aggregating Data
+- Counting Results
+- Creating Lists
+- How Many Actors?
+- Working with Dates and Times
+- Duration
+- Duration in Minutes
+
+# 04. Graph Traversal
+**7 Lessons**
+- Graph Traversal
+- Traversing the Graph
+- Relationships Traversed
+- Varying Length Traversal
+- Actors 2 Hops Away
+- Actors 4 Hops Away
+- Actors Up to 4 Hops Away
+
+# 05. Pipelining Queries
+**11 Lessons**
+- Scoping Variables
+- Highest Revenue Movies
+- Highest for Tom Hanks
+- Top Movies
+- Unique Genres
+- Pipelining Queries
+- Highest Rated Tom Hanks Movie
+- Highest Average Rating
+- Unwinding Lists
+- UK Movies
+- Taiwan Movies
+
+# 06. Reducing Memory
+**3 Lessons**
+- Subqueries
+- Using Subqueries
+- Combining Results
+
+# 07. Using Parameters
+**4 Lessons**
+- Parameters in Cypher
+- Setting Parameters
+- Using Parameters
+- Application Examples Using Parameters
+
+---
+
+From this [Neo4j Graph Data Science](https://graphacademy.neo4j.com/courses/gds-certification/) certification series 2 courses are left
+
+* Introduction to Neo4j Graph Data Science
+* Neo4j Graph Data Science Fundamentals
+
+Which are supposed to take ~1.5hrs. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 4b366b48132f271c2b2a301f3172df84e814c978 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 5 Oct 2024 01:37:22 +0900
Subject: [PATCH 450/638] Create 2024-10-04-day-277.md

---
 _posts/2024-10-04-day-277.md | 65 ++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 _posts/2024-10-04-day-277.md

diff --git a/_posts/2024-10-04-day-277.md b/_posts/2024-10-04-day-277.md
new file mode 100644
index 000000000..6f2fdc9de
--- /dev/null
+++ b/_posts/2024-10-04-day-277.md
@@ -0,0 +1,65 @@
+---
+layout: post
+title: (Day 277) Neo4j Professional Certificate attempt 1
+categories: [gnn,traditional-machine-learning,statistics]
+---
+
+# Hello :) Today is Day 277!
+A quick summary of today:
+* reading a bit of The Elements of Statistical Learning
+* attempting the [Neo4j Professional Certificate](https://graphacademy.neo4j.com/certifications/neo4j-certification/)
+
+## Reading a bit of The Elements of Statistical Learning
+
+I think this is the book I was looking for. Not sure why I am starting to read it 10 months into my journey but it is what I was looking for. Going in-depth on 'classical' models - linear regression and classificaion, trees, boosting. 
+
+I went to a cafe to read it, but I could not film myself 😆. 
+
+![image](https://github.com/user-attachments/assets/2e88b9a5-9bb5-41a2-95fa-e17dc8dec6e6)
+
+One thing I noticed was the usage of the below two pics
+
+![image](https://github.com/user-attachments/assets/94256e79-fd0c-42e8-b83b-04a2f26e9aa2)
+![image](https://github.com/user-attachments/assets/e1c5a71c-70d5-4e72-9789-3fc3ce046443)
+
+To showcase the difference between a simple OLS and a KNN, this and DL by Bishop&Bishop used those pics, and most likely other books have used them as well. They are good pics and a nice representation of the clear different between the two methods. 
+
+I had a more in-depth look into chapter 3
+
+![image](https://github.com/user-attachments/assets/9ae863a2-82d9-4925-9460-46f59df844be)
+
+### 1. Linear Regression Model
+- models the relationship between input X and the response Y as a linear combination
+- the goal is to estimate coefficients beta that minimize the residual sum of squares (RSS), representing the difference between predicted and actual values
+
+### 2. Least Squares Estimation
+- the OLS method minimizes RSS to estimate parameters beta
+- OLS is unbiased and computationally efficient but may struggle with multicollinearity or large sets of predictors
+
+### 3. Subset Selection
+- Techniques for selecting a subset of input features to reduce model complexity while maintaining accuracy:
+  - best subset selection
+  - forward selection
+  - backward selection
+
+### 4. Shrinkage Method
+- **Ridge Regression**: Adds a penalty based on the size of the coefficients, shrinking them toward zero to reduce variance at the cost of a small bias
+- **Lasso**: Similar to ridge, but uses the absolute value of the coefficients, making it useful for feature selection as some coefficients may be shrunk to zero
+
+### 5. Principal Components Regression (PCR) and Partial Least Squares (PLS)
+- **PCR**: Reduces dimensionality by transforming predictors into principal components and performing regression on these components
+- **PLS**: Similar to PCR but also takes the response variable into account, often leading to better predictive performance
+
+I need to read and take notes for this book. It reminds me of [ISLR](https://www.statlearning.com/)
+
+## Attempting the [Neo4j Professional Certificate](https://graphacademy.neo4j.com/certifications/neo4j-certification/)
+
+<img width="778" alt="image" src="https://github.com/user-attachments/assets/f7e61630-66e0-4636-a8b5-052bad9381bc">
+
+There is no proctor or screen checking so I was surprised I could just check the docs or google for answers. I tried to minimise that just to see what I actually know, and above the the result haha. I didn't pass however the feedback is great so now I know what I need to focus on to improve and hopefully pass next time.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 1b8be9b8999da074c1aa9f8eee7cf852079b280e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 5 Oct 2024 23:42:36 +0900
Subject: [PATCH 451/638] Create 2024-10-05-day-278.md

---
 _posts/2024-10-05-day-278.md | 881 +++++++++++++++++++++++++++++++++++
 1 file changed, 881 insertions(+)
 create mode 100644 _posts/2024-10-05-day-278.md

diff --git a/_posts/2024-10-05-day-278.md b/_posts/2024-10-05-day-278.md
new file mode 100644
index 000000000..f3f703bc0
--- /dev/null
+++ b/_posts/2024-10-05-day-278.md
@@ -0,0 +1,881 @@
+---
+layout: post
+title: (Day 278) Before Machine Learning Volume 2 - Calculus + Neo4j GDS
+categories: [gnn,traditional-machine-learning,math]
+---
+
+# Hello :) Today is Day 278!
+A quick summary of today:
+* started [volume 2](https://www.mldepot.co.uk/) of the before ML book series by Jorge Brasil
+* covered the Neo4j [graph data science courses](https://graphacademy.neo4j.com/courses/graph-data-science-fundamentals/)
+
+After reading the 1st volume on linear algebra I decided to continue as I love going back to basics and seeing how different books introduce them. Also I like the titles of these books - Before Machine Learning which are straight to the point and clear. 
+
+## Chapter 1 - Why Calculus?
+
+In 1 sentence: It is the 'magic' power behind the optimization and functionality of machine learning algorithms
+
+## Chapter 2 - Pointing Fingers and Crossing Lines: The last breath of just linearity
+
+It starts with describing the point (yes, a single point), a small dot that can be assigned coordinates in any space. With two points, we can define a line, and by calculating the slope (rise over run), we arrive at the equation y=mx+b, where m is the slope and b is the y-intercept. This formula gives us a way to describe any straight line and sets the stage for exploring more complex concepts.
+
+## Chapter 3 - Changing Times and Tangent Lines: The Derivative
+
+The derivative measures how much a function’s output changes in response to a small change in its input.
+
+<img width="473" alt="image" src="https://github.com/user-attachments/assets/990ff695-6824-497a-88b6-b01af085adb1">
+
+We want to understand what happens to the rate of change when h is getting very close to, but is not, 0, meaning that h is a finitely small number. 
+
+<img width="322" alt="image" src="https://github.com/user-attachments/assets/69349a0c-3831-4052-9877-3d0e2ce8fff9">
+
+Then, it introduced some of the rules of calculus:
+
+* constant rule
+* product rule
+* quotient rule
+* sum rule
+* substraction rule
+* power rule
+* chain rule (a bit later in the chapter)
+
+### It introduced limits and rules for them:
+
+<img width="1415" alt="image" src="https://github.com/user-attachments/assets/734d1960-541c-4c68-9635-fdf0127a80b3">
+
+### L'Hopital's rule
+
+This is a technique in calculus used to find the limit of indeterminate forms, such as 0/0 or ∞/∞. The rule says that if you get one of these forms when calculating a limit, you can take the derivative of the numerator and the derivative of the denominator separately, and then calculate the limit again.
+
+If you have a limit that looks like this:
+
+`lim (x → c) [f(x) / g(x)] = 0/0 or ∞/∞`
+
+You can apply L'Hopital's Rule by taking the derivatives:
+
+`lim (x → c) [f(x) / g(x)] = lim (x → c) [f'(x) / g'(x)]`
+
+If the result is still indeterminate, you can repeat the process by taking more derivatives.
+
+For example, to find the limit of:
+
+`lim (x → 0) [sin(x) / x]`
+
+At x = 0, it results in 0/0, so we apply L'Hopital's Rule:
+
+The derivative of sin(x) is cos(x).
+The derivative of x is 1.
+Now the limit becomes:
+
+`lim (x → 0) [cos(x) / 1] = cos(0) = 1`
+
+So, the limit is 1.
+
+### Second derivative
+
+We know that the first derivative gives the slope of the tangent line at any given point on the function. However, the 2nd derivative is about understanding the variation of this slope along the function’s path. To think of it visually, imagine drawing tangent lines at various points along a curve. The slope of these lines could change: they might get steeper, remain constant, or become flatter. The second derivative is a tool that quantifies this change. If it’s positive, the slopes of these tangent lines are getting steeper, indicating that the function is bending or curving upward. On the contrary, if it’s negative, the slopes are becoming less steep or more negative, which signifies that the function is curving downward
+
+### Taylor series
+
+An infinite series allows us to represent or approximate functions using polynomials. Among these, the Taylor series stands out. It enables us to create precise polynomial approximations of functions around a specific point.
+
+At its core, these series are straightforward; each term is added to the sum of the preceding terms
+
+<img width="125" alt="image" src="https://github.com/user-attachments/assets/84ad9635-592d-4408-a6df-b94c9e4b4291">
+
+The Taylor series is useful especially for solving differential equations, approximating difficult functions or analyzing the behavior of functions around a specific point.
+
+### Critical points
+
+#### The Newton-Raphson method
+
+<img width="241" alt="image" src="https://github.com/user-attachments/assets/3ad132aa-6383-4241-8cbe-3d8830b70f56">
+
+The next iteration is the result of the subtraction of the previous point and the ratio between the values of both the function and the derivative at the same location
+
+For a more intuitive view, let's consider the ratio. The term `f(x_{n-1})` gives the value of `f(x)` at that point, and if we are looking for the root, this value indicates how far we are from it. The derivative at this point, which represents the slope of the tangent line at `x_{n-1}`, acts like a guide, showing us the direction to move. In essence, we are 'normalizing our error' (how far off our current guess is) by considering the “rate of change” (how quickly the function is changing). This ratio helps determine how much we should adjust our guess to move closer to the root.
+
+Put simply, a root point in Newton's method is the value where the function equals zero
+
+To successfully find the root of a function, the function must be:
+
+* differentiable
+* the derivative can’t be zero at the approximation
+
+- if f'(x) is decreasing, i.e, f''(x) < 0, then f'(a) = 0 is a maximum point at a, and f(x) has the shape of a ∩, where a belongs to the domain of the function
+
+- if f'(x) is increasing, i.e f''(x) > 0, then f'(a) = 0 is a minimum point at a, and f(x) has the shape of a ∪, where a belongs to the domain of the function
+
+To understand the significance of the second derivative f''(x), we need to explore its roots. By plotting f(x) and f''(x), we see that where f''(x) crosses the x-axis marks critical points where the tangent to f(x) is horizontal, potentially indicating local maxima or minima.
+
+<img width="389" alt="image" src="https://github.com/user-attachments/assets/f939b003-3544-424b-b098-6113f358b6a7">
+
+These points where f''(x) = 0 are important because they signal changes in the curvature of f(x), which could mean the function is switching between a bowl (concave up) and a dome (concave down). These are called inflection points
+
+To confirm whether f''(x) = 0 is an inflection point:
+- confirmed inflection point: if f''(x) changes sign around x = a, then x = a is an inflection point, showing a change in curvature
+- not an inflection point: if f''(x) does not change sign around x = a, then x = a is not an inflection point, meaning the curvature remains unchanged
+
+The sign of f''(x) around the root helps determine whether the curvature of f(x) changes at that point.
+
+#### Gradient descent
+
+1. Select a starting point
+2. Pick a learning rate
+3. Update the value of the estimate x_n
+4. Check the stopping conditions
+
+I found that the series has an accompanying github repo with sample code showcasing some of the concepts. And part of this chapter 3 is showcased [through code as well](https://github.com/mldepot/beforeMachineLearning/blob/main/Vol2%20-%20Calculus/CritcalPointsWithGradientDescentAndNewtons.ipynb).
+
+### Code for chapter 3
+
+First, we define the functions and derivatives:
+
+```python
+def f(x):
+    return x**4 - 4*x**3 + 4*x**2
+def df(x):
+    return 4*x**3 - 12*x**2 + 8*x
+    
+def df2(x):  # Second derivative
+    return 12*x**2 - 24*x + 8
+```
+
+#### 1. Newton's Method for Root Finding:
+
+**Function and Derivative:**
+
+The function we're considering is:
+
+f(x) = x^4 + 4x^3 + 4x^2
+
+Its derivative is:
+
+f'(x) = 4x^3 - 12x^2 + 8x
+
+**Method Overview:**
+
+The Newton-Raphson method updates our estimate for the root using the formula:
+
+x_{n+1} = x_n - \frac{f(x_n)}{f'(x_n)}
+
+This method iteratively refines the root approximation.
+
+**Stopping Criteria:**
+
+We use two stopping criteria:
+
+1. When the absolute value of f(x) is less than 10^{-5} indicating proximity to a root
+2. After 10 iterations, to prevent endless loops in case the method doesn't converge
+
+```python
+def newton_method(x0, tol=1e-5, max_iter=10):
+    """
+    Newton's method for root finding.
+    
+    Parameters:
+    - x0: Initial guess
+    - tol: Tolerance level (default is 1e-5)
+    - max_iter: Maximum number of iterations (default is 10)
+
+    Returns:
+    - Root of the function f(x) if found within max_iter iterations and tolerance, else returns the latest estimate
+    """
+    x = x0
+    for i in range(max_iter):
+        # Compute the next approximation using Newton's method formula
+        x_next = x - f(x)/df(x)
+        
+        # Check the stopping criterion
+        if abs(f(x_next)) < tol:
+            print(f"Root found at x = {x_next}, after {i+1} iterations.")
+            return x_next
+        
+        # Update x for the next iteration
+        x = x_next
+        
+    print(f"Stopped after {max_iter} iterations. Latest estimate for root is x = {x}.")
+    return x
+```
+
+Using the below:
+
+```python
+x0 = 0.1
+root = newton_method(x0)
+```
+
+We get: `Root found at x = 0.001407967242655117, after 6 iterations.`
+
+#### 2. Minimization with Gradient Descent
+
+**Method Overview:**
+
+Gradient Descent updates our estimate for the local minimum using the rule:
+
+x_{n+1} = x_n - alpha f'(x_n)
+
+Where alpha is the learning rate. This method iteratively shifts the current estimate in the direction of the steepest decrease of the function
+
+**Stopping Criteria:**
+
+The algorithm stops when the difference between the function values in consecutive steps is smaller than 10^{-5}. If this doesn't occur, the method will stop after 10,000 iterations as a fallback.
+
+```python
+def gradient_descent_minimize(learning_rate=0.01, tol=1e-5, x0=0.1):
+    """
+    Function to perform minimization using gradient descent.
+    
+    Parameters:
+    - learning_rate: Step size for each iteration.
+    - tol: Tolerance value for stopping criterion.
+    - x0: Initial guess.
+    
+    Returns:
+    - x: Approximate minimum point.
+    - errors: List containing value of f(x) for each iteration.
+    """
+    x = x0
+    errors = [f(x)]
+    
+    # Run gradient descent until change in f(x) value is smaller than tol
+    while True:
+        x_new = x - learning_rate * df(x)
+        if np.abs(f(x_new) - f(x)) < tol:
+            break
+        x = x_new
+        errors.append(f(x))
+    
+    return x, errors
+```
+
+From:
+
+```python
+min_gd, gd_errors = gradient_descent_minimize()
+print(f"Approximate minimum point using Gradient Descent: x = {min_gd}")
+```
+
+We get: `Approximate minimum point using Gradient Descent: x = 0.00387025495013399`
+
+#### 3. Minimization with Newton's Method
+
+**Method Overview:**
+
+Newton's method for optimization updates our estimate for the local minimum using the rule:
+
+x_{n+1} = x_n - \frac{f'(x_n)}{f''(x_n)}
+
+This method tries to approximate the function locally by a quadratic function and then jumps straight to the minimum of that quadratic.
+
+**Stopping Criteria:**
+
+The algorithm stops when the difference between the function values in consecutive steps is smaller than 10^{-5}. If this doesn't occur, the method will stop after 10 iterations as a fallback.
+
+```python
+def newton_method_optimize(tol=1e-5, x0=0.1):
+    """
+    Function to perform minimization using Newton's method.
+    
+    Parameters:
+    - tol: Tolerance value for stopping criterion.
+    - x0: Initial guess.
+    
+    Returns:
+    - x: Approximate minimum point.
+    - errors: List containing value of f(x) for each iteration.
+    """
+    x = x0
+    errors = [f(x)]
+    
+    # Run Newton's method until change in f(x) value is smaller than tol
+    while True:
+        x_new = x - df(x)/df2(x)
+        if np.abs(f(x_new) - f(x)) < tol:
+            break
+        x = x_new
+        errors.append(f(x))
+    
+    return x, errors
+```
+
+```python
+# Initial guess
+x0 = 0.1
+min_point, errors = newton_method_optimize(x0=x0)
+print(f"Approximate minimum point using Newton's method for optimization: x = {min_point:.5f}")
+```
+
+We get: `Approximate minimum point using Newton's method for optimization: x = -0.00055`
+
+#### 4. Comparison between Gradient Descent and Newton’s Method for Optimization
+
+```python
+min_gd, gd_errors = gradient_descent_minimize()
+min_newton, newton_errors = newton_method_optimize()
+
+# Compare number of iterations taken to converge
+print(f"Gradient Descent took {len(gd_errors)} iterations.")
+print(f"Newton's method took {len(newton_errors)} iterations.")
+
+# Plot the convergence of the methods
+plt.figure(figsize=(10, 6))
+plt.plot(gd_errors, label='Gradient Descent', linestyle='--')
+plt.plot(newton_errors, label="Newton's Method", linestyle='-')
+plt.title("Convergence of Gradient Descent vs Newton's Method")
+plt.xlabel("Iterations")
+plt.ylabel("Value of f(x)")
+plt.legend()
+plt.grid(True)
+plt.show()
+```
+
+<img width="792" alt="image" src="https://github.com/user-attachments/assets/e2ffb815-b3e0-4ad0-a208-4e2a7c440115">
+
+Gradient Descent took 42 iterations.
+
+Newton's method took 3 iterations.
+
+While Newton's method can be faster in terms of iterations, its computational complexity, memory requirements, and sensitivity make Gradient Descent a more practical choice for many applications, particularly in high-dimensional spaces or when dealing with non-convex functions.
+
+## Chapter 4 Cleaning Up The Derivatives Debris: The Integral
+
+How can we calculate the area under a function
+
+<img width="319" alt="image" src="https://github.com/user-attachments/assets/86dfb6b6-44ce-4720-ac53-08affa74fade">
+
+We can use rectangles to approximate it
+
+<img width="356" alt="image" src="https://github.com/user-attachments/assets/ab1293c7-3a32-427b-8968-6cacebeedace">
+
+And the base is the derivative of x (dx) and the height is f(x). To get a better approximation we can increase the amount of rectangles:
+
+<img width="353" alt="image" src="https://github.com/user-attachments/assets/3bcc81eb-c2ed-4d7a-b5e5-2f04a17f11c0">
+
+And even more
+
+<img width="334" alt="image" src="https://github.com/user-attachments/assets/c9143610-4490-46a3-9c82-145ccb6664a5">
+
+Now the height of each of the rectangles is f(x)
+
+### Fundamental theorem of calculus
+
+<img width="262" alt="image" src="https://github.com/user-attachments/assets/dd4fb9bc-6810-4aca-a099-145e3fc3f162">
+
+Where F is an anti-derivative with formula:
+
+<img width="162" alt="image" src="https://github.com/user-attachments/assets/3a15c410-7129-4af3-afa3-efa377cd3efa">
+
+Integration and differentiation are inverse operations; integrating a function and then differentiating the result returns the original function.
+
+---
+
+Chapter 5 (last chapter) is for tomorrow:
+
+<img width="493" alt="image" src="https://github.com/user-attachments/assets/22789eb9-c95d-4a2e-90f8-f729f9c20279">
+<img width="462" alt="image" src="https://github.com/user-attachments/assets/a43fc0ab-c75d-4ad8-9e84-c26fc1b6a23a">
+
+## [Introduction to Neo4j Graph Data Science](https://graphacademy.neo4j.com/courses/gds-product-introduction/)
+
+Today I did this course. It was very short and not that much info. 
+
+<img width="292" alt="image" src="https://github.com/user-attachments/assets/e224ee1b-bcad-435f-8998-37b23eed70f6">
+
+The main thing I think is Native vs Cypher graph projections
+
+In Neo4j, both **Native** and **Cypher Projections** refer to different methods of working with graph data, especially when it comes to accessing and manipulating it. Here’s a breakdown of each:
+
+### 1. Native
+
+**Native** refers to the core functionality provided by Neo4j, which operates directly on the underlying graph database structure. This includes:
+
+- Data Model: Native access utilizes the property graph model where nodes and relationships can have properties associated with them.
+- Performance: Accessing data natively can be faster for certain operations since it leverages Neo4j’s optimized internal storage and indexing mechanisms.
+- API Access: You can interact with the graph directly using the Neo4j Java API or other drivers for various programming languages. This approach is usually more complex but offers high performance for large datasets and complex queries.
+- Lower-Level Operations: Native access allows for more control over transaction management, caching, and other low-level operations, which can be beneficial for advanced use cases.
+
+### 2. Cypher Projections
+
+Cypher Projections are a more abstracted way of working with Neo4j data through the Cypher query language. Projections allow you to define a subset of data and work with it in a more user-friendly manner:
+
+- Declarative Syntax: cypher uses a more intuitive, SQL-like syntax, making it easier for users to query and manipulate graph data without needing to understand the underlying data structures
+- Data Retrieval: we can create projections that define which parts of the graph to retrieve and how to structure the results. This is useful for getting specific views of the data without the overhead of fetching everything
+- Read-Only: Cypher projections are generally read-only, meaning you can retrieve and manipulate the data in-memory but can't directly modify the underlying graph through projections
+- Flexibility: Projections allow you to transform the data in various ways, such as aggregating properties, filtering nodes, and modifying relationships, all in a single query
+
+### Use Cases
+
+- Native might be more suitable for applications that require high performance and involve complex transactions or batch processing
+- Cypher Projections are often preferred for data exploration, analytics, and scenarios where ease of use and readability are more important than raw performance
+
+
+## [Graph Data Science Fundamentals](https://graphacademy.neo4j.com/courses/graph-data-science-fundamentals/)
+
+### Algorithm Tiers and Execution Modes
+
+**Tiers**
+
+GDS algorithms are classified into three tiers: alpha, beta, and production.
+
+* Production-quality: Indicates that the algorithm has been tested in regard to stability and scalability. Algorithms in this tier are prefixed with gds.<algorithm>.
+
+* Beta: Indicates that the algorithm is a candidate for the production-quality tier. Algorithms in this tier are prefixed with gds.beta.<algorithm>.
+
+* Alpha: Indicates that the algorithm is experimental and might be changed or removed at any time. Algorithms in this tier are prefixed with gds.alpha.<algorithm>.
+
+**Execution Modes**
+
+GDS algorithms have 4 executions modes which determine how the results of the algorithm are handled.
+
+* stream: Returns the result of the algorithm as a stream of records.
+
+* stats: Returns a single record of summary statistics, but does not write to the Neo4j database or modify any data.
+
+* mutate: Writes the results of the algorithm to the in-memory graph projection and returns a single record of summary statistics.
+
+* write: Writes the results of the algorithm back the Neo4j database and returns a single record of summary statistics.
+
+**Memory Estimation**
+
+To learn more about the used algorithms we can use `.estimate` to estimate the memory that is required to support the run analytics/flows.
+
+**Overall Algorithm Syntax**
+
+```cypher
+CALL gds[.<tier>].<algorithm>.<execution-mode>[.<estimate>](
+	graphName: STRING,
+	configuration: MAP
+)
+```
+
+### Centrality and Importance
+
+Centrality algorithms are used to determine the importance of distinct nodes in a graph.
+
+Common use cases of centrality include:
+
+* Recommendations: Identify and recommend the most influential or popular items in your content or product offering catalog
+
+* Supply chain analytics: find the most critical node in your supply chain, whether it be a supplier in a network, a raw material that is part of a manufactured product, or a port in a route
+
+* Fraud & Anomaly Detection: Find users with many shared identifiers or who otherwise act as a bridge between many communities
+
+**Degree Centrality**
+
+Degree centrality is one of the most ubiquitous and simple centrality algorithms. It counts the number of relationships a node has. In the GDS implementation, we specifically calculate out-degree centrality which is the count of outgoing relationships from a node
+
+To use a GDS algorithm, you must first create a graph projection. A projection is an in-memory graph you can quickly query and manipulate.
+
+Create a graph projection of Actor and Movie nodes:
+
+```cypher
+CALL gds.graph.project(
+  'proj',
+  ['Actor','Movie'],
+  'ACTED_IN'
+  );
+```
+
+Then stream the degree centrality to find the actors who have acted in the most movies:
+
+```cypher
+//get top 5 most prolific actors (those in the most movies)
+//using degree centrality which counts number of `ACTED_IN` relationships
+CALL gds.degree.stream('proj')
+YIELD nodeId, score
+RETURN
+  gds.util.asNode(nodeId).name AS actorName,
+  score AS numberOfMoviesActedIn
+ORDER BY numberOfMoviesActedIn DESCENDING, actorName LIMIT 5
+```
+
+We get:
+
+<img width="473" alt="image" src="https://github.com/user-attachments/assets/a50bd15c-8e52-470f-818e-9ab86177ab67">
+
+**PageRank**
+
+PageRank is a good algorithm for measuring the influence of nodes in a directed graph, particularly where the relationships imply some form of flow of movement such as in payment networks, supply chain and logistics, communications, routing, and graphs of website and links.
+
+In summary, PageRank estimates the importance of a node by counting the number of incoming relationships from neighboring nodes weighted by the importance and out-degree centrality of those neighbors. The underlying assumption is that more important nodes are likely to have proportionately more incoming relationships from other important nodes.
+
+First, create the graph projection. We can use a Cypher projection to create an in-memory graph with :DIRECTED_ACTOR relationships between two (:Person) nodes. This graph can be traversed to understand the influence across directors and actors.
+
+```cypher
+// create Cypher projection for network of people directing actors
+// filter to recent high grossing movies
+MATCH (source:Person)-[:DIRECTED]->(m:Movie)<-[:ACTED_IN]-(target)
+WHERE m.year >= 1990 AND m.revenue >= 10000000
+WITH source, target, count(*) as actedWithCount
+WITH gds.graph.project(
+  'proj',
+  source,
+  target,
+  {
+    relationshipType: "DIRECTED_ACTOR"
+  }
+) as g
+RETURN
+  g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels
+```
+
+Next stream PageRank to find the top 5 most influential people in director-actor network.
+
+```cypher
+CALL gds.pageRank.stream('proj')
+YIELD nodeId, score
+RETURN
+  gds.util.asNode(nodeId).name AS personName,
+  score AS influence
+ORDER BY influence DESCENDING, personName LIMIT 5
+```
+
+<img width="480" alt="image" src="https://github.com/user-attachments/assets/a5c3933a-a63d-4cd9-aaed-7fdf8e186b7f">
+
+**Other Centrality Algorithms**
+
+Other GDS production tier centrality algorithms include:
+
+* Betweenness Centrality: Measures the extent to which a node stands between the other nodes in a graph. It is often used to find nodes that serve as a bridge from one part of a graph to another.
+
+* Eigenvector Centrality: Measures the transitive influence of nodes. Similar to PageRank, but works only on the largest eigenvector of the adjacency matrix so does not converge in the same way and tends to more strongly favor high degree nodes. It can be more appropriate in certain use cases, particularly those with undirected relationships.
+
+* Article Rank: A variant of PageRank which assumes that relationships originating from low-degree nodes have a higher influence than relationships from high-degree nodes.
+
+### Path Finding
+
+Path finding algorithms find the shortest path between two or more nodes or evaluate the availability and quality of paths.
+
+Common use cases of path finding are:
+
+* Supply chain analytics: Identifying the fastest path between an origin and a destination or between a raw material and a finished product
+
+* Customer Journey: Analyzing the events that make up a customer’s experience. In healthcare for example, this can be the experience of an in-patient from admission to discharge.
+
+**Dijkstra Source-Target Shortest Path**
+
+A common, industry standard, path finding algorithm is Dijkstra. It computes the shortest path between a source and a target node. Like many other path finding algorithms in GDS, Dijkstra supports weighted relationships to account for distance or another cost property when comparing paths.
+
+First, create the graph projection.
+
+```cypher
+CALL gds.graph.project('proj',
+    ['Person','Movie'],
+    {
+        ACTED_IN:{orientation:'UNDIRECTED'},
+        DIRECTED:{orientation:'UNDIRECTED'}
+    }
+);
+```
+
+Then we can run Dijkstra’s shortest path.
+
+```cypher
+MATCH (kevin:Actor{name : 'Kevin Bacon'})
+MATCH (denzel:Actor{name : 'Denzel Washington'})
+
+CALL gds.shortestPath.dijkstra.stream(
+    'proj',
+    {
+        sourceNode:kevin,
+        TargetNode:denzel
+    }
+)
+
+YIELD sourceNode, targetNode, path
+RETURN sourceNode, targetNode, nodes(path) as path;
+```
+
+Resulting path:
+
+![image](https://github.com/user-attachments/assets/39046494-0426-4fdc-830b-8601452f330a)
+
+**Other Path Finding Algorithms**
+
+Other GDS production tier Path Finding algorithms can be split into a few different subcategories that are listed below:
+
+Shortest path between two nodes:
+
+* A* Shortest Path: An extension of Dijkstra that uses a heuristic function to speed up computation.
+
+* Yen’s Algorithm Shortest Path: An extension of Dijkstra that allows you to find multiple, the top k, shortest paths.
+
+Shortest path between a source node and multiple other target nodes:
+
+* Dijkstra Single-Source Shortest Path: Dijkstra implementation for shortest path between one source and multiple targets.
+
+* Delta-Stepping Single-Source Shortest Path: Parallelized shortest path computation. Computes faster than Dijkstra single-source shortest Path but uses more memory.
+
+General path search between a source node and multiple other target nodes:
+
+* Breadth First Search: Searches paths in order of increasing distance from the source node on each iteration.
+
+* Depth First Search: Searches as far as possible along a single multi-hop path on each iteration.
+
+### Community Detection
+
+Community detection algorithms are used to evaluate how groups of nodes may be clustered or partitioned in the graph. Much of the community detection functionality in GDS is focused on distinguishing and assigning ids to these node groups for downstream analytics, visualization, or other processing.
+
+Common use cases of community detection include:
+
+* Fraud detection: Finding fraud rings by identifying accounts that have frequent suspicious transactions and/or share identifiers between one another.
+
+* Customer 360: Disambiguating multiple records and interactions into a single customer profile so an organization has an aggregated source of truth for each customer.
+
+* Market segmentation: dividing a target market into approachable subgroups based on priorities, behaviors, interests, and other criteria.
+
+**Louvain Community Detection**
+
+A common community detection algorithm is Louvain. Louvain maximizes a modularity score for each community, where the modularity quantifies the quality of an assignment of nodes to communities. This means evaluating how much more densely connected the nodes within a community are, compared to how connected they would be in a random network.
+
+Louvain optimizes this modularity with a hierarchical clustering approach that recursively merges communities together. There are multiple parameters that can be used to tune Louvain to control its performance and the number and size of communities produced. This includes the maximum number of iterations and hierarchical levels to use as well as the tolerance parameter for assessing convergence/stopping conditions. 
+
+An additional important consideration is that Louvain is a stochastic algorithm. As such, the community assignments may change a bit when re-run. When the graph does not have a naturally well-defined community structure the changes between runs can become more significant. Louvain includes a seedProperty parameter which can be used to assign initial community ids and help with consistency between runs. Also, if consistency is important, other community detection algorithms, such as Weakly Connected Components (WCC), take a more deterministic partitioning approach to assigning communities and thus will not change between runs.
+
+First create a graph projection with movies, actors, and directors. Project the relationships with an UNDIRECTED orientation as that works best with the Louvain algorithm.
+
+```cypher
+CALL gds.graph.project('proj', ['Movie', 'Person'], {
+    ACTED_IN:{orientation:'UNDIRECTED'},
+    DIRECTED:{orientation:'UNDIRECTED'}
+});
+```
+
+Then we can run Louvain. Here we will run Louvain in mutate mode to save community Ids and return high level statistics on the community counts, distribution, modularity score, and information for how Louvain processed the graph.
+
+```cypher
+CALL gds.louvain.mutate('proj', {mutateProperty:'communityId'})
+```
+
+Here is the output:
+
+```
+mutateMillis
+nodePropertiesWritten
+modularity
+modularities
+ranLevels
+communityCount
+communityDistribution
+postProcessingMillis
+preProcessingMillis
+computeMillis
+configuration
+0
+28172
+0.7080752619422075
+[0.4735378349269174, 0.610543949698786, 0.6984121667978503, 0.707715829688635, 0.7080752619422075]
+5
+504
+{
+  "min": 1,
+  "p5": 3,
+  "max": 2752,
+  "p999": 2752,
+  "p99": 986,
+  "p1": 1,
+  "p10": 4,
+  "p90": 154,
+  "p50": 6,
+  "p25": 6,
+  "p75": 8,
+  "p95": 310,
+  "mean": 55.8968253968254
+}
+27
+0
+13469
+{
+  "mutateProperty": "communityId",
+  "jobId": "2a32b9a2-1489-41ed-9afd-9429bbe9a7f7",
+  "sudo": false,
+  "maxIterations": 10,
+  "maxLevels": 10,
+  "seedProperty": null,
+  "logProgress": true,
+  "nodeLabels": [
+    "*"
+  ],
+  "concurrency": 4,
+  "includeIntermediateCommunities": false,
+  "relationshipTypes": [
+    "*"
+  ],
+  "tolerance": 0.0001,
+  "consecutiveIds": false
+}
+```
+
+We can verify the communityId node properties in the projection with a stream operation.
+
+```cypher
+CALL gds.graph.nodeProperty.stream('proj','communityId', ['Person'])
+YIELD nodeId, propertyValue
+WITH gds.util.asNode(nodeId) AS n, propertyValue AS communityId
+WHERE n:Person
+RETURN n.name, communityId LIMIT 10
+```
+
+<img width="561" alt="image" src="https://github.com/user-attachments/assets/d99d717b-27f7-4acd-a6a9-a311dd47747d">
+
+Where we can see the node's name and the id of the community it belongs to.
+
+**Other Community Detection Algorithms**
+
+Below are some of the other production tier community detection algorithms. A full list of all community detection algorithms can be found in the Community Detection algorithms documentation.
+
+* Label Propagation: Similar intent as Louvain. Fast algorithm that parallelizes well. Great for large graphs.
+
+* Weakly Connected Components (WCC): Partitions the graph into sets of connected nodes such that: a) Every node is reachable from any other node in the same set; b) No path exists between nodes from different sets
+
+* Triangle Count: Counts the number of triangles for each node. Can be used to detect the cohesiveness of communities and stability of the graph.
+
+* Local Clustering Coefficient: Computes the local clustering coefficient for each node in the graph which is an indicator for how the node clusters with its neighbors.
+
+### Node Embedding
+
+The goal of node embedding is to compute low-dimensional vector representations of nodes such that similarity between vectors (eg. dot product) approximates similarity between nodes in the original graph. These vectors, also called embeddings, can be extremely useful for exploratory data analysis, similarity measurements, and machine learning.
+
+![image](https://github.com/user-attachments/assets/c0eba0bb-eb32-49c2-b05a-5a25596ff858)
+
+Of course, in real-world problems node embeddings will usually be larger than 2 dimensions, often ending up in the hundreds or larger, especially when applied to bigger graphs with millions or billions of nodes. Node embedding also doesn’t have to base similarity strictly on node proximity in the graph. While similarity based on distance in relationship hops and common neighbors is perhaps most common in application, node embedding can also consider node properties and other "global-view" node attributes when calculating embedding vectors.
+
+**Use Cases**
+
+* EDA
+* Similarity measurement
+* Features for ML
+
+**FastRP**
+
+GDS offers a custom implementation of a node embedding technique called Fast Random Projection
+
+FastRP leverages probabilistic sampling techniques to generate sparse representations of the graph allowing for extremely fast calculation of embedding vectors that are comparative in quality to those produced with traditional random walk and neural net techniques such as Node2vec and GraphSage. This makes FastRP a great choice for getting started with exploring embedding on a graph in GDS.
+
+**Other Node Embedding Algorithms**
+
+GDS has also implemented Node2Vec, which computes a vector representation of a node based on random walks in the graph, and GraphSage, which is an inductive modeling approach for computing node embeddings using node properties and graph structure.
+
+### Similarity
+
+Similarity algorithms, as the name implies, are used to infer similarity between pairs of nodes. In GDS these algorithms run over the graph projection in bulk. When similar node pairs are identified according to the user specified metric and threshold, a relationship with a similarity score property is drawn between the pair. Depending on which execution mode is used when running the algorithm, these similarity relationships can be streamed, mutated to the in-memory graph, or written back to the database.
+
+Common use cases for similarity include:
+
+* Fraud detection: finding potential fraud user accounts by analyzing how similar a set of new user accounts is to flagged accounts
+
+* Recommendation Systems: In an online retail store, identifying items that pair to the one currently being viewed by a user to inform impressions and increase rate of purchase
+
+* Entity Resolution: Identify nodes that are similar to one another based on activity or identifying information in the graph
+
+**Similarity Algorithms in GDS**
+
+* Node Similarity: Determines similarity between nodes based on the relative proportion of shared neighboring nodes in the graph. Node Similarity is a good choice where explainability is important, and you can narrow down the universe of comparisons to a subset of your data. Examples of narrowing down include focusing on just single communities, newly added nodes, or nodes within a specific proximity to a subgraph of interest.
+
+* K-Nearest Neighbor (KNN): Determines similarity based off node properties. The GDS KNN implementation can scale well for global inference over large graphs when tuned appropriately. it can be used in conjunction with embeddings and other graph algorithms to determine the similarity between nodes based on proximity in the graph, node properties, community structure, importance/centrality, etc.
+
+**Choice of Similarity Metric**
+
+Both Node Similarity and KNN provide choices between different similarity metrics. Node Similarity has choices between Jaccard and Overlap similarity. KNN choice of metric is driven by the node property types. List of integers are subject to Jaccard and Overlap, list of floating point numbers to Cosine Similarity, Pearson, and Euclidean. Using different metrics will of course alter the similarity score and change the interpretation slightly.
+
+**Controlling Scope of Comparisons**
+
+Comparing every node to every other node in the graph is a computationally expensive task of roughly O(n^2) complexity. The GDS implementations for both Node Similarity and KNN have internal mechanisms to intelligently select node pairs for comparison allowing them to work faster and scale better. They also have parameters that can be adjusted by the user to tune how node pairs are sampled and selected for comparison.
+
+**Controlling Scope of Results**
+
+For similarity comparisons we may also want to control the number of results returned to only consider the most relevant node pairs. Both Node Similarity and KNN have a topK parameter to limit the number similarity comparisons returned per node. With node similarity there is also the capability to limit the results globally as opposed to just a per node basis.
+
+**Applied Example with KNN**
+
+Create a projection
+
+```cypher
+CALL gds.graph.project('proj', ['Movie', 'Person'], {
+    ACTED_IN:{orientation:'UNDIRECTED'},
+    DIRECTED:{orientation:'UNDIRECTED'}
+});
+```
+
+Run FastRP in mutate mode so the embeddings will be saved in the projection
+
+```cypher
+CALL gds.fastRP.mutate('proj',  {
+    embeddingDimension:64,
+    randomSeed:7474,
+    mutateProperty:'embedding'
+})
+```
+
+After that we can run similarity. We will use the default cosine metric. 
+
+```cypher
+CALL gds.knn.stream('proj', {nodeLabels:['Person'], nodeProperties:['embedding'], topK:1})
+YIELD  node1, node2, similarity
+RETURN gds.util.asNode(node1).name AS actorName1,
+    gds.util.asNode(node2).name AS actorName2,
+    similarity
+LIMIT 10
+```
+
+Here is a sample out of the 10
+
+```
+actorName1
+actorName2
+similarity
+"François Lallement"
+"Gian Maria Volontè"
+0.701564371585846
+"Jules-Eugène Legris"
+"Giacomo Baessato"
+0.7108281850814819
+```
+
+**Similarity Functions**
+
+In addition to the node similarity and KNN algorithms, GDS also provides a set of functions that can be used to calculate similarity between two arrays of numbers using various similarity metrics including jaccard, overlap, pearson, cosine similarity and a few others.
+
+### Machine Learning Overview
+
+GDS focuses on offering managed pipelines for end-to-end ML workflows. Data selection, feature engineering, data splitting, hyperparameter configuration, and training steps are coupled together within the pipeline object to track the end-to-end steps needed.
+
+There are currently two supported types of ML pipelines:
+
+* Node Classification Pipelines: Supervised binary and multi-class classification for nodes
+
+* Link Prediction Pipelines: Supervised prediction for whether a relationship or "link" should exist between pairs of nodes
+
+### Node Classification Pipeline
+
+**Node Classification Pattern in GDS**
+
+![image](https://github.com/user-attachments/assets/39a043f5-6980-4b7e-ab1e-8e93ea50c453)
+
+In practice, steps 1-6 will be executed automatically by the pipeline. We will just be responsible for providing configuration and hyperparameters for them. So at a high level, the workflow will look like the following for node classification, this will be the same for link prediction as well:
+
+1. Project a graph and configure the pipeline (the order doesn’t matter).
+
+2. Execute the pipeline with a train command.
+
+3. Predict on a projected graph with the predict command. The predictions can then be written back to the database if desired using graph write operations.
+
+### Link Prediction
+
+GDS currently offers a binary classifier where the target is a 0-1 indicator, 0 for no link, 1 for a link. This type of link prediction works really well on an undirected graph where you are predicting one type of relationship between nodes of a single label, such as for social network and entity resolution problems.
+
+**Link Prediction Pattern in GDS**
+
+![image](https://github.com/user-attachments/assets/37b37efc-c424-4022-b435-35969cbbb9e3)
+
+there is an additional feature-input set in the relationship splits which now comes before node property and feature generation steps. In short, this is to handle data leakage issues, whereby model features are calculated using the relationships you are trying to predict. Such a situation would allow the model to use information in the features that would normally not be available, resulting in overly optimistic performance metrics.
+
+<img width="1034" alt="image" src="https://github.com/user-attachments/assets/1d12d3d5-0d8b-4247-b8c5-cedb0ffc5cf8">
+
+Creating a pipeline in code seem to have a lot of steps but it's something I need to explore further by meself.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 53c3ff7825137cfc3f938c071348f6a5dc06baa0 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 6 Oct 2024 23:23:18 +0900
Subject: [PATCH 452/638] Create 2024-10-06-day-279.md

---
 _posts/2024-10-06-day-279.md | 95 ++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)
 create mode 100644 _posts/2024-10-06-day-279.md

diff --git a/_posts/2024-10-06-day-279.md b/_posts/2024-10-06-day-279.md
new file mode 100644
index 000000000..a7090932f
--- /dev/null
+++ b/_posts/2024-10-06-day-279.md
@@ -0,0 +1,95 @@
+---
+layout: post
+title: (Day 279) Neo4j Graph Data Science Certification - SUCCESS!
+categories: [gnn,traditional-machine-learning,math]
+---
+
+# Hello :) Today is Day 279!
+A quick summary of today:
+
+## [Neo4j Graph Data Science Certification](https://graphacademy.neo4j.com/c/c66662a0-256b-4fa7-98cb-898a6fca6d49/) ✔️
+
+Technically this should count for yesterday, because after I posted yesterday's post and rested a bit I decided to take the certification which was around midnight. 
+
+<img width="1032" alt="image" src="https://github.com/user-attachments/assets/0e078551-36b6-46c9-a5d3-112b82513736">
+
+And I got it 🥳
+
+## [The 100-page ML book](https://themlbook.com/)
+
+![image](https://github.com/user-attachments/assets/6a701954-c032-473b-8772-fc250fa16950)
+
+Wow! This book goes into my hall of fame 😆
+
+While it is ~150 pages (short) it covers the material very well and if I was a beginner, I would use this book as a guide for what to study next. I am not sure if this book is a precursor for the [MLE book](https://www.mlebook.com/wiki/doku.php) by the same author or if it's a complimentary piece, but it is so nice and easy to read (compared to what I read next below)
+
+I got the MLE book as well and I will keep it on my to-read list.
+
+## Before ML Vol 2 - Chapter 5 - A Free Upgrade: More Dimensions
+
+### Scalar Functions
+
+<img width="183" alt="image" src="https://github.com/user-attachments/assets/e3e3b3f7-07d0-40eb-aded-228cbf39c6dd">
+
+It is a type of function that takes inputs from a two-dimensional space (represented by R^2) and maps them to a single-dimensional output (represented by f : R^2 → R)
+
+When dealing with functions of two variables, such as f(x, y), we employ the concept of partial derivatives. This involves differentiating with respect to one variable while holding the other constant.
+
+**Gradient Descent**
+
+<img width="231" alt="image" src="https://github.com/user-attachments/assets/ee9de107-fa91-4a45-9a80-96eaf2dd4557">
+
+* X = (x₁, x₂, ..., xₙ) represents a vector with *n* dimensions
+* η is the learning rate
+* ∇f(Xₙ) is the gradient of the function *f* with respect to the parameters **Xₙ** - the GPS that indicates the direction of steepest descent
+
+**The Hessian**
+
+The Hessian matrix is crucial in understanding the curvature of scalar functions and determining convexity. It's a square matrix of second-order partial derivatives. To prove convexity, the function must satisfy a specific inequality involving the Hessian.
+
+By using a second-order Taylor expansion, we relate the Hessian to the curvature properties of the function. If the Hessian is positive semi-definite (its eigenvalues are non-negative), the function is convex. This guarantees that any identified minimum is a global minimum, which is vital in optimization processes.
+
+**Neural networks**
+
+This part introduced some of the basic concepts behind NNs and went through a sample NN calculation
+
+* activation functions
+* loss functions
+* backprop and chain rule
+
+The author [shares a notebook](https://github.com/mldepot/beforeMachineLearning/blob/main/Vol2%20-%20Calculus/NeuralNetwork.ipynb) where a neural net is written from scratch and a comparison is shown for different activation functions as well.
+
+### Vector Functions
+
+Taking multiple inputs and producing multiple outputs
+
+An example curvature:
+
+<img width="380" alt="image" src="https://github.com/user-attachments/assets/01bc90f9-40b8-4680-89e3-d9c0107632aa">
+
+It’s definitely not convex!
+
+We must understand how to classify critical point other than the minima
+
+- **Positive Eigenvalues (Local Minimum):**
+  - If all the eigenvalues of the Hessian matrix at a critical point are positive, then the function has a local minimum at that point. This is because the positive eigenvalues indicate that the function is concave up along all directions in the vicinity of that point, forming a bowl shape or an upward-facing paraboloid.
+  
+- **Negative Eigenvalues (Local Maximum):**
+  - If all the eigenvalues of the Hessian matrix at a critical point are negative, then the function has a local maximum at that point. This represents a situation where the function is concave down in all directions at that point, resembling an inverted bowl or a downward-facing paraboloid.
+  
+- **Mixed Sign Eigenvalues (Saddle Point):**
+  - If the Hessian matrix at a critical point has eigenvalues with mixed signs (some positive and some negative), then the function has a saddle point at that location. A saddle point represents a situation where the function is concave up in some directions and concave down in others, resembling a saddle shape on the graph.
+
+The global or local minima concept here is slightly different than with a scalar function. Two functions dictate the output of f: The function d represents the distance to the food source whereas the dance angle is depicted by that θ guy. Therefore, selecting one or more of these critical points depends on the context of the problem we are dealing with.
+
+And again showed an example neural net calculation. 
+
+2nd book of this Before ML series, and I still feel this book has too much abstractions and side stories. Maybe it deserves a 2nd read to change my opinion. 
+
+I found that the 3rd book on probability and stats is only available on amazon's kindle, but when I try to buy the Kindle edition it says it is not available right now :( I guess I have to wait. 
+
+---
+
+That is all for today!
+
+See you tomorrow :) 

From 9d153eaf737f3e54aa5f9b88f7ccc32bcf1f06db Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 7 Oct 2024 10:30:46 +0900
Subject: [PATCH 453/638] forgot to add summary of the day

---
 _posts/2024-10-06-day-279.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_posts/2024-10-06-day-279.md b/_posts/2024-10-06-day-279.md
index a7090932f..9e3abe4b4 100644
--- a/_posts/2024-10-06-day-279.md
+++ b/_posts/2024-10-06-day-279.md
@@ -6,6 +6,8 @@ categories: [gnn,traditional-machine-learning,math]
 
 # Hello :) Today is Day 279!
 A quick summary of today:
+* passed the Neo4j GDS certification exam
+* read The 100-page ML book
 
 ## [Neo4j Graph Data Science Certification](https://graphacademy.neo4j.com/c/c66662a0-256b-4fa7-98cb-898a6fca6d49/) ✔️
 

From ce7b60c0c8a7c5b34baf010071b87c29f383f2cf Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 8 Oct 2024 00:14:19 +0900
Subject: [PATCH 454/638] Create 2024-10-08-day-280.md

---
 _posts/2024-10-08-day-280.md | 402 +++++++++++++++++++++++++++++++++++
 1 file changed, 402 insertions(+)
 create mode 100644 _posts/2024-10-08-day-280.md

diff --git a/_posts/2024-10-08-day-280.md b/_posts/2024-10-08-day-280.md
new file mode 100644
index 000000000..098bf66e6
--- /dev/null
+++ b/_posts/2024-10-08-day-280.md
@@ -0,0 +1,402 @@
+---
+layout: post
+title: (Day 280) Reading scikit-learn docs on stream + reading Andryi Burkov's MLE book
+categories: [traditional-machine-learning,theory,mlops,data-eng,statistics]
+---
+
+# Hello :) Today is Day 280!
+A quick summary of today:
+* started the [MLE](https://mlebook.com/) book by Andriy Burkov
+* streamed and read some of scikit-learn's docs
+
+## MLE by Andryi Burkov
+
+In the foreword, written by Cassie Kozyrkov - Chief Decision Scientist at Google, I found about [Making Friends with Machine Learning](https://youtu.be/1vkb7BCMQd0) - a course by Google, and I will mark it to check later. 
+
+Also she compliments this book a lot as one of the very few 'applied' machine learning books, compared to the horde of research ML books out there, and she finishes the foreword with:
+
+_If you intend to use machine learning to solve business problems at scale, I’m delighted you got your hands on this book. Enjoy!_
+
+## Chapter 1: Introduction
+
+### Notation and definitions
+
+* scalar
+* vector
+* matrix
+* capital sigma
+* Euclidean norm and distance
+
+### What is ML
+
+_Machine learning is a subfield of computer science that is concerned with building algorithms that, to be useful, rely on a collection of examples of some phenomenon. These examples can come from nature, be handcrafted by humans, or generated by another algorith_
+
+* supervised learning (features, targets, regression, classification)
+* unsupervised learning (unlabeled examples, clustering, dimensionality reduction, outlier detection)
+* semi-supervised learning (both labeled and unlabeled examples)
+* reinforcement learning
+
+### Data and ML terminology
+
+* data used directly and indirectly
+* raw and tidy data
+* training and holdout sets
+* baseline models
+* ML pipelines
+* parameters vs hyperparameters
+* classification vs regression
+* model-based (i.e. SVM with params that are applied on new data) vs instance-based learning (i.e. KNN which uses the nearby neighbourhood of the input to make a prediction)
+* shallow vs deep learning
+
+### When to use ML
+
+* when the problem is too complex for coding
+* when the problem is constantly changing
+* when it is a perceptive problem (i.e. speech, image, video recognition)
+* when it is an unstudied phenomenon
+* when the problem has a simple objective
+* when it is cost-effective
+
+### When *not* to use ML
+
+* every action of the system or a decision made by it must be explainable
+* every change in the system’s behavior compared to its past behavior in a similar situation must be explainable
+* the cost of an error made by the system is too high
+* you want to get to the market as fast as possible
+* getting the right data is too hard or impossible
+* you can solve the problem using traditional software development at a lower cost
+* a simple heuristic would work reasonably well
+* the phenomenon has too many outcomes while you cannot get a sufficient amount of examples to represent them (like in video games or word processing software)
+* you build a system that will not have to be improved frequently over time
+* you can manually fill an exhaustive lookup table by providing the expected output for any input (that is, the number of possible input values is not too large, or getting outputs is fast and cheap)
+
+### What is MLE
+
+Machine Learning Engineering (MLE) focuses on building, deploying, and maintaining machine learning models in production environments. It involves data collection, preprocessing, feature engineering, model training, optimization, and ensuring model stability and scalability. MLEs work to integrate machine learning models into systems, handling issues like performance degradation and system failures over time.
+
+Thanks to DataTalksClub's mlops-zoomcamp I managed to get a very good insight into the world of MLE.
+
+### ML project life cycle
+
+![image](https://github.com/user-attachments/assets/635db062-c2f4-4c01-a4e6-a6535f722d47)
+
+## Chapter 2 - Before the project starts
+
+* the project must be prioritised as there could be many items in the backlog, so we need to figure out which ones are more important for an MVP (or something like that)
+* the project myst have a well-defined goal so that resources can be allocated appropriately
+
+### Prioritisation of a ML project
+
+The key considerations are impact and cost
+
+* impact is high when 1) ML can replace a complex part if an engineering project, or 2) there's a great benefit in getting inexpensive (but probably imperfect) predictions
+* three cost highly influence the cost: the difficulty of the problem; the cost of data; the need for accuracy
+  * in terms of the problem's difficulty, the main considerations are: whether an implemented algorithm or a software library capable of solving the problem is available; and whether significant computation power is needed to build the model or to run it in the production environment
+  * in terms of cost of data, the main considerations are: can data ba generated automatically; what is the cost of manual annotation of the data; how many examples are needed
+  * in terms of accuracy: how costly is each wrong prediction; and what is the lowest accuracy level below which the model becomes impractical
+ 
+### Estimating complexity of a ML project
+
+There is no set standard, but just a comparison with other projects.
+
+#### The unknowns
+
+* whether the required quality is attainable in practice
+* how much data you will need to read the required quality
+* what features and how many features are necessary so that the model can learn and generalise sufficiently
+* how large the model should be
+* how long will it take to train one model and how many experiemtns will be required to reach the desired level of performance
+
+#### Simplifying the problem
+
+One way to make a more educated guess is to simplify the problem and solve a simpler problem first.
+
+#### Nonlinear progress
+
+The prediction error typically decreases rapidly at first, but progress slows down over time. Sometimes, there may be no improvement, prompting you to add new features that rely on external databases or knowledge sources. While developing these features or labeling more data (possibly through outsourcing), the model's performance may stagnate.
+
+Due to this nonlinear progress, it's important to ensure that the product owner or client understands the limitations and risks involved. Keep detailed logs of every activity and track the time spent, which will aid in both reporting and estimating the complexity of future projects.
+
+### Defining the goal of a ML project
+
+_The goal of a machine learning project is to build a model that solves, or helps solve, a business problem._
+
+#### What a model can do
+
+* automate (for example, by taking action on the user’s behalf or by starting or stopping a specific activity on a server)
+* alert or prompt (for example, by asking the user if an action should be taken or by asking a system administrator if the traffic seems suspicious)
+* organize, by presenting a set of items in an order that might be useful for a user (for example, by sorting pictures or documents in the order of similarity to a query oraccording to the user’s preferences)
+* annotate (for instance, by adding contextual annotations to displayed information, or by highlighting, in a text, phrases relevant to the user’s task)
+* extract (for example, by detecting smaller pieces of relevant information in a larger input, such as named entities in the text: proper names, companies, or locations)
+* recommend (for example, by detecting and showing to a user highly relevant items in a large collection based on item’s content or user’s reaction to the past recommendations)
+* classify (for example, by dispatching input examples into one, or several, of a predefined set of distinctly-named groups)
+* quantify (for example, by assigning a number, such as a price, to an object, such as a house)
+* synthesize (for example, by generating new text, image, sound, or another object similar to the objects in a collection)
+* answer an explicit question (for example, “Does this text describe that image?” or “Are these two images similar?”)
+* transform its input (for example, by reducing its dimensionality for visualization purposes, paraphrasing a long text as a short abstract, translating a sentence into another language, or augmenting an image by applying a filter to it)
+* detect a novelty or an anomaly
+
+#### Properties of a successful model
+
+* it respects the input and output specifications and the performance requirements
+* it benefits the organisation (measured via cost reduction, increased sales or profit)
+* it helps the user (measured via productivity, engagement, and sentiment)
+* it is scientifically rigorous
+
+### Structuring a ML team
+
+#### Two cultures
+
+1. Collaborative specialization: 
+   - data analysts and software engineers work closely together
+   - engineers are not required to have deep machine learning expertise, but they need to understand the terminology
+   - advocates argue that each team member should excel in their specific area. Data analysts should master machine learning techniques, while engineers focus on efficient, maintainable code
+
+2. Full-stack expertise:
+   - every engineer in the team possesses both machine learning and software engineering skills
+   - supporters of this approach claim that scientists often prioritize accuracy over practicality, leading to solutions that may not be viable in production. Additionally, scientists may not write well-structured, efficient code, creating challenges for engineers to rewrite it for production
+
+Each culture has its pros and cons, with one emphasizing deep specialization and the other encouraging a hybrid skill set.
+
+#### Members of a ML team
+
+A machine learning team may consist of various experts, including:
+
+1. Data engineers:
+   - responsible for ETL (Extract, Transform, Load) processes and creating automated data pipelines
+   - design the structure and integration of data from various sources
+   - provide fast access to data through APIs or queries for data analysts and consumers
+   - typically do not need machine learning knowledge
+   - in large companies, data engineers usually work in separate teams from machine learning engineers
+
+2. Data labeling experts:
+   - handle labeling of data according to specifications from analysts, build labeling tools, manage labelers, and validate data quality
+   - in large companies, labeling teams may include local, outsourced labelers, and engineers responsible for tool development
+   - collaboration with **domain experts** is encouraged for better feature engineering, aligning model predictions with business needs
+
+3. DevOps engineers:
+   - collaborate with machine learning engineers to automate model deployment, monitoring, and maintenance
+   - in smaller companies, a DevOps engineer may be part of the machine learning team, while in larger organizations, they work in a broader DevOps team
+   - some companies have introduced the **MLOps role**, dedicated to deploying and upgrading machine learning models and managing data pipelines involving these models
+
+### Why ML projects fail
+
+* lack of experienced talent
+* lack of support by the leadership
+* lack of data infrastructure
+* data labeling challenges
+* siloed organisations and lack of collaboration
+* technically infeasible projects
+* lack of alignment between tech and business teams
+
+## Chapter 3 - Data collection and preparation
+
+Before any machine learning activity can start, the analyst must collect and prepare the data. The data available to the analyst is not always “right” and is not always in a form that a machine learning algorithm can use.
+
+![image](https://github.com/user-attachments/assets/e72037a6-dfbd-4c12-b342-1b10244941c4)
+
+### Questions about the data
+
+* is the data accessible
+* is the data sizeable (enough for our project)
+* is the data useable
+* is the data understandable
+* is the data reliable
+
+### Common problems with data
+
+* high cost
+* bad quality
+* noise
+* bias (selection bias, self-selection bias, omitted variable bias, sponsorship or funding bias, sampling bias, prejudice or stereotype bias, systematic value distortion, experimenter bias, labeling bias)
+* low predictive power
+* outdated examples
+* outliers
+* data leakage
+
+#### Ways to avoid bias
+
+Avoiding bias in data is challenging but crucial for building fair models. Here are several strategies:
+
+1. Question the data: 
+   - investigate who created the data, why, and how. Examine research methods to ensure they don’t introduce bias
+
+2. Avoid selection bias: 
+   - systematically question the choice of data sources. Using only current customer data for predictions, for example, can lead to overly optimistic results
+
+3. Manage self-selection bias:
+   - keep surveys short and offer incentives for quality responses. Pre-select respondents to reduce bias, such as using expert references instead of asking entrepreneurs directly if they are successful
+
+4. Minimize omitted variable bias:
+   - use all available features, even unnecessary ones, and rely on regularization to determine their importance. If a key feature is missing, consider using a proxy
+
+5. Reduce sponsorship bias:
+   - investigate the incentives of data sources, especially in areas like tobacco or pharmaceuticals where bias may be introduced by sponsors
+
+6. Avoid sampling bias:
+   - research real-world data proportions and ensure similar proportions in your training data
+
+7. Control prejudice or stereotypebias bias:
+   - ensure balanced data representation by adjusting for under- or over-represented groups during training
+
+8. Address systematic value distortion:
+   - use multiple measuring devices or trained humans to reduce bias in measurements
+
+9. Prevent experimenter bias:
+   - let multiple people validate survey questions and opt for open-ended questions. If using multiple-choice, include an "Other" option
+
+10. Avoid labeling bias:
+   - use multiple labelers to label the same examples and compare their decisions. Investigate frequent document skipping by labelers
+
+11. Human involvement:
+   - keep humans involved in data gathering and preparation, as models trained on biased data will produce biased results
+
+### What is good data
+
+* good data is informative: it contains enough information that can be used for modeling
+* goot  data has good coverage of what you want to do with the model
+* good data reflects real inputs
+* good data is as unbiased as possible
+* good data is not a result of a feedback loop (is no the result of the model itself)
+* good data has consistent labels
+* good data is big enough
+
+### Dealing with interaction data
+
+Interaction data is the data you can collect from user interactions with the system our model supports.
+
+Good interaction data contains information on 3 aspects:
+
+* context of interaction
+* action of the user in that context
+* outcome of interaction
+
+### Causes of data leakage
+
+* target is a function of a feature
+* features hides the target
+* features from the future
+
+### Data partitioning
+
+To obtain good partitions (train, validation, test) if a dataset, partitioning has to satisfy several conditions:
+
+1. Split was applied to raw data
+2. Data was randomised before the split (unless time-series cases)
+3. Validation and test sets follow the same distribution
+4. Leakage during the split was avoided
+
+Group leakage - imagine you have magnetic resonance images of the brains of multiple patients. Each image is labeled with certain brain disease, and the same patient may be represented by several images taken at different times. If you apply the partitioning technique discussed above (shuffle, then split), images of the same patient might appear in both the training and holdout data.
+
+### Dealing with issing attributes
+
+* removing the examples with missing attributes from the dataset (if the data is big enough)
+* using a learning algorithm that can deal with missing attribute values (such as decision trees)
+* using data imputation
+
+If we use the mean or something similar for data imputation, data leakage might appear, however this type of leakage is not as significant as other types from above.
+
+### Data augmentation
+
+* images
+* text
+
+### Dealing with imbalanced data
+
+![image](https://github.com/user-attachments/assets/72043107-7d5c-4168-9561-ab003c2eb7b0)
+
+#### Oversampling
+
+1. Copying examples: 
+   - multiple copies of minority class examples are created, effectively increasing their weight
+
+2. Creating synthetic examples: 
+   - synthetic examples are generated by combining feature values from multiple minority class examples. Two well-known algorithms for this purpose are:
+   - Synthetic Minority Oversampling Technique (SMOTE): for a given minority class example, it selects a number of nearest neighbors. A new synthetic example is generated by taking the original example and adding a fraction of the difference between it and one of its randomly chosen neighbors
+   - Adaptive Synthetic Sampling Method (ADASYN): similar to SMOTE, but the number of synthetic examples created for each original example is proportional to the number of neighbors that are not from the minority class. This approach generates more synthetic examples in areas where minority class examples are sparse
+
+![image](https://github.com/user-attachments/assets/2b8d5c2f-7151-4eca-9cda-e68c8ee02e39)
+
+#### Undersampling
+
+1. The undersampling can be done randomly - the examples to remove from the majority class can be chosen at random
+2. Property-Based selection - examples can be selected for removal based on specific properties. One such property involves Tomek links. A Tomek link exists between two examples from different classes if no other example is closer to either of them than they are to each other. Closeness can be defined using metrics like cosine similarity or Euclidean distance. Removing examples based on Tomek links can help establish a clear margin between classes.
+3. Cluster-Based - first, decide how many examples you want to retain in the majority class after undersampling. This number is denoted as k. Then, run a centroid-based clustering algorithm on the majority class examples, using k as the desired number of clusters. After clustering, replace all examples in the majority class with the k centroids. An example of a centroid-based clustering algorithm is k-nearest neighbors.
+
+![image](https://github.com/user-attachments/assets/f0479ee1-31c2-4aa8-9f8d-5c7f1624965c)
+
+Hyprid strategies, combining over- and undersampling exist as well.
+
+### Data sampling strategies
+
+When we have a large dataset, it’s not always practical or necessary to work with the entire data asset. Instead, we can draw a smaller data sample that contains enough information for learning.
+
+There are two main strategies: probability sampling and nonprobability sampling
+* In probability sampling, all examples have a chance to be selected. These techniques involve randomness
+* Nonprobability sampling is not random. To build a sample, it follows a fixed deterministic sequence of heuristic actions. This means that some examples don’t have a chance of being selected, no matter how many samples you build.
+
+#### Sample random sampling
+
+It is the most straightforward method. Here, each example from the entire dataset is chosen purely by chance; each example has an equal chance of being selected.
+
+#### Systematic (or interval) sampling
+
+We create a list containing all examples. From that list, randomly select the first example x_start from the first k elements on the list. Then, you select every k-th item on the list starting from x_start. You choose such a value of k that will give you a sample of the desired size. 
+
+An advantage of this method over the simple random sampling is that it draws examples from the whole range of values. However, systematic sampling is inappropriate if the list of examples has periodicity or repetitive patterns. In the latter case, the obtained sample can exhibit a bias. However, if the list of examples is randomised, then systematic sampling often results in a better sample than simple random sampling.
+
+#### Stratified sampling
+
+If we know about the existence of several groups (e.g. gender, location, or age) in our data, we should have examples from each of those groups in your sample. In stratified sampling, we first divide your dataset into groups (called strata) and then randomly select examples from each stratum, like in simple random sampling. The number of examples to select from each stratum is proportional to the size of the stratum.
+
+If we don't know how to define the strata, we can use clustering algorithms where we need to decide on the amount of clusters. 
+
+Stratified sampling is the slowest of the three methods due to the additional overhead of working with several independent strata. However, its potential benefit of producing a less biased sample typically outweighs its drawbacks.
+
+### Storing Data
+
+* data formats (CSV, XML, JSON, PARQUET, etc.)
+* data storage levels (filesystems, object storage, database, data lake)
+* data versioning:
+  * level 0: data is unversioned
+  * level 1: data is versioned as a snapshot at training time
+  * level 2: both data and code are versioned as one asset
+  * level 3: using or building a specialised data versioning solution
+
+ * docs and metadata: what that data means, how it was collected/created, the partitioning details, preprocessing steps, was any data excluded, format used to store the data, types of attributes/features, number of examples
+
+### Data manipulation best practices
+
+#### Reproducability
+
+Reproducibility is crucial in all aspects of data collection and preparation. To ensure reproducibility, avoid manual data transformations and reliance on ad hoc tools, such as regular expressions or quick commands in text editors and command line shells.
+
+Data collection and transformation typically involve multiple stages, including:
+
+- downloading data from web APIs or databases
+- replacing multiword expressions with unique tokens
+- removing stop-words and noise
+- cropping and unblurring images
+- imputation of missing values
+
+Each stage should be implemented as a software script, such as in Python or R, detailing inputs and outputs. This approach helps you track all changes made to the data. If an issue arises during any stage, you can simply fix the script and rerun the entire data processing pipeline from the beginning.
+
+Manual interventions, on the other hand, can be challenging to reproduce, especially when working with updated datasets or larger volumes of data. Automating the process ensures scalability and consistency in data handling.
+
+#### Data first, Algorithm second
+
+Focus most of your effort and time on getting more data of wide variety and high quality, instead of trying to squeeze the maximum out of a learning algorithm
+
+Data augmentation, when implemented well, will most likely contribute more to the quality of the model than the search for the best hyperparameter values or model architecture.
+
+## Streamed for 1.5hr today
+
+[Here](https://youtu.be/GFg2eaVcngc)
+
+I read a few different posts on scikit-learn's documentation because they are very well-written and have good examples. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 012a2b9229803dfa97ea5639aa62f408448ebc16 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 8 Oct 2024 21:52:30 +0900
Subject: [PATCH 455/638] Create 2024-10-08-day-281.md

---
 _posts/2024-10-08-day-281.md | 295 +++++++++++++++++++++++++++++++++++
 1 file changed, 295 insertions(+)
 create mode 100644 _posts/2024-10-08-day-281.md

diff --git a/_posts/2024-10-08-day-281.md b/_posts/2024-10-08-day-281.md
new file mode 100644
index 000000000..77d614e16
--- /dev/null
+++ b/_posts/2024-10-08-day-281.md
@@ -0,0 +1,295 @@
+---
+layout: post
+title: (Day 281) Read chapter 4 of Andriy Burkov's MLE book
+categories: [traditional-machine-learning,theory,mlops,data-eng]
+---
+
+# Hello :) Today is Day 281!
+A quick summary of today:
+* chapter 4 of Andriy Burkov's MLE book
+
+## Chapter 4 - Feature Engineering
+
+*After data collection and preparation, feature engineering is the second most important activity in machine learning*
+
+<img width="980" alt="image" src="https://github.com/user-attachments/assets/7b3a8204-d432-413a-a393-d1e9a0c3ed1a">
+
+Feature engineering is a process of first conceptually and then programmatically transforming a raw example into a feature vector.
+
+### How to engineer features
+
+**Feature engineering in text**
+
+There are two common techniques: one-hot encoding and bag-of-words
+
+**Categorical features to numbers**
+
+OHE is not the only method. There is also:
+
+* mean encoding (also called bin counting or feature calibration)
+
+The **odds ratio** (OR) is used to quantify the association between two events, like a feature and a binary label. In binary classification, it helps measure how the presence of a feature (e.g., a word in an email) is associated with the likelihood of a certain outcome (e.g., spam).
+
+The **odds ratio** compares the odds of an event happening in the presence of some condition to the odds of the event happening in the absence of that condition.
+
+Key concept:
+
+- **Odds of an event** = P(A happens)/P(A doesn’t happen)
+  
+To calculate the **odds ratio** for two events, A (the feature being present, e.g., the word 'infected') and B (the positive class, e.g., spam):
+
+OR = Odds of B when A is present / Odds of B when A is absent
+
+|                     | Spam (B) | Not Spam (¬B) | Total |
+|---------------------|----------|---------------|-------|
+| Contains “infected” (A)   | 145      | 8             | 153   |
+| Doesn't contain “infected” (¬A) | 346      | 2909          | 3255  |
+| **Total**           | 491      | 2917          | 3408  |
+
+- Odds of spam when 'infected' is present = 145/8 = 18.125
+- Odds of spam when 'infected' is absent = 346/2909 = 0.119
+
+Now, calculate the odds ratio:
+
+OR = (145/8)/(346/2909) = 152.4
+
+The odds ratio of 152.4 means that emails containing the word “infected” are 152.4 times more likely to be spam compared to emails that don’t contain the word.
+
+Log-Odds Ratio
+
+The log-odds ratio is the logarithm of the odds ratio, and it helps in managing extremely large or small values, which can arise when the odds ratio is very high or very low. It is easier to work with log values in machine learning models.
+
+Here’s the formula for the log-odds ratio:
+
+log(OR) = log(145) - log(8) - log(346) + log(2909) = 2.2
+
+This log-odds ratio value of **2.2** can be used to replace the categorical value 'infected' in the model as a numeric feature.
+
+* sine-cosine transformation
+
+When categorical features are cyclical, integer encoding does not work well. For example, try converting Monday through Sunday to the integers 1 through 7. The difference between Sunday and Saturday is 1, while the difference between Monday and Sunday is −6. However, our reasoning suggests the same difference of 1, because Monday is just one day past Sunday
+
+Instead, we can use sine-consine transformation which converts a cyclical feature into two synthetic features. If *p* is the feature of interest, we can replace it with two following two values:
+
+<img width="492" alt="image-1" src="https://github.com/user-attachments/assets/d5770155-63b7-4af4-9dde-975e60c0e8d9">
+
+The below table shows the days of the week converted using this transformation
+
+| p  | \( p_{sin} \) | \( p_{cos} \) |
+|----|--------------|--------------|
+| 1  | 0.78         | 0.62         |
+| 2  | 0.97         | -0.22        |
+| 3  | 0.43         | -0.9         |
+| 4  | -0.43        | -0.9         |
+| 5  | -0.97        | -0.22        |
+| 6  | -0.78        | 0.62         |
+| 7  | 0            | 1            |
+
+<img width="838" alt="image-2" src="https://github.com/user-attachments/assets/eec55963-8907-4ab2-81dd-b72571b60b51">
+
+If we visualise the points from the table, we can clearly see the cyclical nature
+
+In our data, we can replace monday with [0.78, 0.62], and the same for the rest of the week and this representation might work better than simple ordinal encoding. 
+
+**Feature hashing**
+
+Feature hashing converts text data, or categorical attributes with many values, into a feature vector of arbitrary dimensionality. 
+
+BOW and OHE are good for handling categorical features but might result in adding too many dimensions to the dataset.
+
+The hashing trick is a method to manage large datasets by converting categorical attributes (or document tokens) into feature vectors of a fixed dimensionality. The process works as follows:
+
+1. Decide on dimensionality: choose the size of the feature vectors (like 5)
+2. Hash function: apply a hash function to each value (e.g., words), converting them into non-negative integers
+3. Modulo operation: use the modulo of the dimensionality to assign each hashed value to an index in the feature vector
+
+Commonly used hash functions are MurmurHash3, Jenkins, CityHash, and MD5
+
+**Topic modelling**
+
+Topic modeling is a family of techniques that uses unlabeled data, typically in the form of natural language text documents. The model learns to represent a document as a vector of topics. For example, in a collection of news articles, the five major topics could be sports, politics, entertainment, finance, and technology. Then, each document could be represented as a five-dimensional feature vector, one dimension per topic:
+
+`[0.04, 0.5, 0.1, 0.3, 0.06]`
+
+Latent Semantic Analysis (LSA) and Latent Dirichlet Allocation (LDA) are examples of topic modelling algorithms. 
+
+### Properties of good features
+
+* high predictive power
+* fast computability
+* reliability
+* uncorrelatedness
+
+### Feature selection
+
+If the feature vector is very wide (contains thousands or millions of features), the training time can become prohibitively long. Furthermore, the overall size of the training data can become too large to fit in the RAM of a conventional server.
+
+#### Cutting the long tail
+
+Typically, if a feature contains information (e.g., a non-zero value) only for a handful of examples, such a feature could be removed from the feature vector. In bag-of-words, you can build a graph with the distribution of token counts, and then cut off the so-called long tail
+
+The decision on a threshold for defining the long tail is somewhat subjective. You can set it as a hyperparameter for your problem and discover the optimal value experimentally
+
+<img width="1047" alt="image-3" src="https://github.com/user-attachments/assets/44f5e67f-6ca9-4915-9fb4-bc0590108ac3">
+
+#### Boruta
+
+This is popularly used in Kaggle (apparently). 
+
+Boruta iteratively trains random forest models and runs statistical tests to identify features as important and unimportant
+
+One useful feature of the random forest is its built-in capability to estimate the importance of each feature
+
+The algorithm works in two stages. First, it classifies all training examples from the original training set. Each decision tree in the random forest model votes only on the classification of examples that weren’t used to build that tree. After a tree is tested, the number of correct predictions is recorded for that tree.
+
+At the second stage, the values of a certain feature are randomly permuted across examples, and the tests are repeated. The number of correct predictions is once again recorded for each tree. The importance of the feature for a single tree is then computed as the difference between the number of correct classifications between the original and permuted setting, divided by the number of examples. To obtain the feature importance score, the feature importance measures for individual trees are averaged. While not strictly necessary, it’s convenient to use z-scores instead of the raw importance scores.
+
+The underlying idea of Boruta is simple: we first extend the list of features by adding a randomized copy of each original feature, and then build a classifier based on this extended dataset. To assess the importance of an original feature, we compare it to all randomized features. Only features for which the importance is higher than that of the randomized features — and statistically significant — are considered truly important.
+
+It is important to note that Boruta is a heuristic and there are no theoretical guarantees for its performance.  If we want to be sure that Boruta doesn’t harm, run it multiple times and make sure that the feature selection is stable
+
+#### L1-regularization
+
+L1 (Lasso) in particular allows the algorithm to learn to ignore features - it penalizes the model for being too complex. Since it 0s down features, it produces a sparse model, and it implicitly performs feature selection. 
+
+#### Task-specific feature selection
+
+When working with text, removing *stop words* is common - a, the, propositions, pronouns, etc. 
+
+Sometimes it might even be worth to replace rare (infrequent) words with a special token like RARE_WORD if we want to reduce the resulting dimensionality.
+
+### Synthesizing features
+
+#### Feature discretisation
+
+Binning, also known as bucketing, is a popular technique that allows transforming a numerical feature into a categorical one by replacing numerical values in a specific range by a constant categorical value.
+
+There are three typical approaches to binning:
+
+* uniform binning
+* k-means-based binning
+* quantile-based binning
+
+After binning, the bins must be transformed back to numerical values by using a technique like OHE.
+
+#### Synthesizing features from the data
+
+One technique commonly used to synthesize one or more additional features is clustering (by using k-means for example)
+
+#### Synthesizing features from other features
+
+Neural networks are notorious for their ability to learn complex features by combining simple features in unordinary ways. They combine simple features by letting their values undergo several levels of nested nonlinear transformations. If you have data in abundance, you can train a deep multilayer perceptron model that will learn to cleverly combine the basic unitary features it receives as input.
+
+In practice, the most common way to obtain new features from the existing features is to apply a simple transformation to one or a pair of existing features. Three typical simple transformations that apply to a numerical feature j in example i are 1) discretization of the feature, 2) squaring the feature, and 3) computing the sample mean and the standard deviation of feature j from k-nearest neighbors of the example i found by using some metric like Euclidean distance or cosine similarity
+
+### Learning features from data
+
+Learning features from data is especially effective when we can get access to large collections of relevant labeled or unlabeled data, such as text corpora or collections of images from the Web.
+
+* word embeddings
+* document embeddings
+
+How to choose embedding dimension?
+
+`d = fourth root of the number of categories`
+
+A more principled approach to choose the embedding dimensionality is to treat it as a hyperparameter tuned on a downstream task.
+
+### Dimensionality reduction
+
+* PCA
+* UMAP
+
+### Scaling features
+
+Feature scaling is bringing all your features to the same, or very similar, ranges of values or distributions. Multiple experiments demonstrated that a learning algorithm applied to scaled features might produce a better model. While there’s no guarantee that scaling will have a positive impact on the quality of your model, it’s considered a best practice
+
+Also, scaling reduces the risk of numerical overflow, the problem that computers have when working with very small or very big numbers.
+
+* normalisation
+* standardisation 
+
+### Data leakage in feature engineering
+
+#### Possible Problems
+
+Data leakage can occur if we perform feature engineering on the entire dataset **before** splitting it into training, validation, and test sets. This results in:
+
+- the training set benefiting from knowledge of the holdout sets, leading to overly optimistic model performance
+- for text data, if **bag-of-words** is used before splitting the data, the model may learn features (tokens) present only in the holdout sets, inflating its performance
+
+#### Solutions
+
+To prevent data leakage:
+
+- **first split the dataset** into training and holdout sets
+- perform feature engineering, such as scaling, binning, or mean encoding, **only on the training data**
+
+### Storing and documenting features
+
+It’s advised to design a schema file that provides a description of the features’ expected properties
+
+A schema file is a document that describes features. This file is machine-readable, versioned, and updated each time someone makes significant updates to features. It should contain:
+
+* names of features
+* for each feature:
+    * its type
+    * the fraction of examples that are expected to have that feature present
+    * min and max values
+    * sample mean and variance
+    * whether it allows zeros
+    * whether it allows undefined values
+
+#### Feature store
+
+It is a central repository used to store, document, reuse, and share features across teams in large organizations. It helps address several challenges:
+
+- feature reuse: same features are often re-implemented by different teams
+- varying feature definitions: different teams define features differently, often without accessible documentation
+- computationally intensive features: some real-time models require complex features that are computationally expensive, making them hard to use in real-time applications
+- inconsistency between training and serving: features might differ between training (historical data) and serving (real-time data), leading to inconsistent results
+- feature expiration: it's unclear when features need to be recomputed, leading to unnecessary recalculations
+
+Key components:
+
+- feature name: a unique identifier for each feature (e.g. 'average_session_length')
+- description: a natural language explanation of the feature (e.g. 'the average length of a user's session')
+- metadata: information such as the feature's purpose, input/output types, caching duration, and whether it is available for online or offline use
+- feature definition: versioned code that computes the feature's value
+
+Benefits:
+
+- online and offline availability: features can be read from cache or computed in real-time, supporting both online and offline use
+- versioning: feature values are versioned, allowing models to be retrained with the same feature values as previous versions
+- reproducibility: previous feature values are retained and timestamped, enabling consistent model building
+
+Use Cases:
+
+- online feature store: updated in real-time with live data (e.g. 'restaurant’s average meal preparation time over the last hour')
+- offline feature store: updated in batch mode with historical data (e.g. 'restaurant’s average meal preparation time over the last seven days')
+
+Feature stores sync between **online** and **offline** modes, ensuring consistency across real-time and batch applications.
+
+<img width="1039" alt="image-4" src="https://github.com/user-attachments/assets/15a2ff54-b5b6-4604-a39f-7ffae7782b0c">
+
+### Feature engineering best practices
+
+* start by generating many simple features
+* reuse legacy systems
+* use IDs as features when needed  (ID as in location for example)
+* reduce cardinality (number of distinct values) when possible (i.e. group similar values, group the long tail, remove the feature)
+* use features based on counts with caution
+* re-do feature selection if necessary
+* test the code carefully
+* keep code, model, and data in sync
+* isolate feature extraction code so that individual feature code can be updated without affecting other features, pipelines, or the model
+* serialise together model and feature extractor
+* log the values of features
+
+---
+
+I was on a uni trip today so I'm glad I managed to finish this chapter while riding on the bus.
+
+That is all for today!
+
+See you tomorrow :)

From 63496517287cd8bbccbbe1d8d11d658e7c80500c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 9 Oct 2024 22:46:12 +0900
Subject: [PATCH 456/638] Create 2024-10-09-day-282.md

---
 _posts/2024-10-09-day-282.md | 225 +++++++++++++++++++++++++++++++++++
 1 file changed, 225 insertions(+)
 create mode 100644 _posts/2024-10-09-day-282.md

diff --git a/_posts/2024-10-09-day-282.md b/_posts/2024-10-09-day-282.md
new file mode 100644
index 000000000..d5116b564
--- /dev/null
+++ b/_posts/2024-10-09-day-282.md
@@ -0,0 +1,225 @@
+---
+layout: post
+title: (Day 282) Chapter 5 from MLE by Andriy Burkov
+categories: [traditional-machine-learning,theory,mlops]
+---
+
+# Hello :) Today is Day 282!
+A quick summary of today:
+* continued reading the MLE book while travelling
+
+## Chapter 5 - Supervised Model Training (part 1)
+
+Model training (or modeling) is the fourth stage in the machine learning project life cycle
+
+<img width="968" alt="image" src="https://github.com/user-attachments/assets/3858c0d6-416f-4002-b7e4-03c089185108">
+
+
+Model training is one of the most overrated activities in machine learning. On average, a machine learning engineer spends only 5 - 10% of their time on modeling, if at all. Successful data collection, preparation, and feature engineering are more important
+
+In this first part, the topics covered are: learning preparation, choosing the learning algorithm, a shallow learning strategy, assessing model performance, bias-variance tradeoff, regularization, the concept of the machine learning pipeline, and hyperparameter tuning.
+
+### Before you start working on the model
+
+Before working on the model, we should validate schema conformity, define an achievable level of performance, choose a performance metric, and make several other decisions.
+
+#### Validate schema conformity
+
+Ensure the data conforms to the schema, as defined by the schema file
+
+Even if we initially prepared the data, it's likely that the original and current datasets are no longer identical. Several factors could explain this difference, including:
+
+- an error in the method used to persist the data, whether to a hard drive or a database
+- an error in the method used to read the data from where it was stored
+- changes made by someone else to the data or schema without informing us
+
+These schema errors must be detected, identified, and corrected, similar to how programming errors are handled. If necessary, the entire data collection and preparation pipeline should be rerun from scratch.
+
+#### Define an achievable performance level
+
+This is very important as it gives us an idea of when to stop trying to improve the model
+
+* if a human can label examples without too much effort, math, or complex logic derivations, then you can hope to achieve human-level performance with your model
+* if the information needed to make a labeling decision is fully contained in the features, you can expect to have near-zero error
+* if the input feature vector has a high number of signals (such as pixels in an image, or words in a document), you can expect to come close to near-zero error
+* if you have a computer program solving the same classification or regression problem, you can expect your model to perform at least as well. Often the machine learning model performance can improve as more labeled data comes in
+* if you observe a similar, but different system, you can expect to get a similar, but different machine learning model performance
+
+#### Choose a performance metric
+
+The metrics depend on the project. It is recommended to choose one, and only one, performance metric before we start working on the model. Then, compare different models and track the overall progress by using this one metric.
+
+#### Choose the right baseline
+
+A baseline is a model or an algorithm that provides a reference point for comparison.
+
+Having a baseline gives an analyst confidence that the machine-learning-based solution works. If the value of the performance metric for the machine learning model is better than the value obtained using the baseline, then machine learning provides value.
+
+A baseline doesn’t have to be the result of any learning algorithm. It can be a rule-based or heuristic algorithm, a simple statistic applied to the training data, or something else.
+
+The two most commonly used baseline algorithms are:
+
+* random prediction: makes a prediction by randomly choosing a label from the collection of labels assigned to the training examples. In the classification problem, it corresponds to randomly picking one class from all classes in the problem. In the regression problem it means selecting from all unique target values in the training data
+* zero rule: predicts the most common class in classification or the average value in regression
+
+Baselines can also be more complex, like linear models or simple neural networks, and can even involve human performance, such as using services like Amazon Mechanical Turk for predictions. Baselines help guide model development by providing a standard to measure against.
+
+#### Split data into three sets
+
+1. training set
+2. validation set
+3. test set
+
+Key considerations when splitting the data:
+
+- the validation and test sets should come from the same statistical distribution, resembling the data expected in production
+- distribution shifts can occur if training data differs from the validation/test sets. We have to be mindful of this when selecting data, as it can lead to overly optimistic performance estimates and result in suboptimal models being deployed
+
+#### Preconditions for supervised learning
+
+1. You have a labeled dataset
+2. You have split the dataset into three subsets: training, validation, and test
+3. Examples in the validation and test sets are statistically similar
+4. You engineered features and filled missed values using only the training data
+5. You converted all examples into numerical feature vectors
+6. You have selected a performance metric that returns a single number
+7. You have a baseline
+
+### Representing labels for ML
+
+* multiclass classification: using OHE
+* multilabel classification (i.e. in an image we identify both cats and dogs): we can use BOW
+
+### Selecting the learning algorithm
+
+#### Main properties of a learning algorithm
+
+* explainability
+* in-memory vs out-of-memory
+* number of features and examples
+* nonlinearity of the data
+* training speed
+* prediction speed
+
+#### Algorithm spot-checking
+
+Shortlisting candidate learning algorithms for a given problem is sometimes called algorithm spot-checking. For the most effective spot-checking, it is recommended to:
+
+* select algorithms based on different principles (sometimes called orthogonal), such as instance-based algorithms, kernel-based, shallow learning, deep learning, ensembles
+* try each algorithm with 3 - 5 different values of the most sensitive hyperparameters
+* use the same training/validation split for all experiments
+* if the learning algorithm is not deterministic (such as the learning algorithms for neural networks and random forests), run several experiments, and then average the results
+* once the project is over, note which algorithms performed the best, and use this information when working on a similar problem in the future
+
+If we don't know our problem well, we can try to solve it using as many orthogonal approaches as possible, rather than spending a lot of time on the most promising approach. It is generally a better idea to spend time experimenting with new algorithms and libraries, rather than trying to squeeze the maximum from the one with which you have the most experience. If we don’t have time to carefully spot-check algorithms, one simple 'hack' is to find an efficient implementation of a learning algorithm or a model that most modern papers claim to beat, when applied to a problem similar to yours, and use it for solving our problem.
+
+There is this nice 'cheat-sheet' that was also in 'The 100-page ML book'
+
+![image-1](https://github.com/user-attachments/assets/2aceb62c-69ed-4c2a-ac03-839f0c0ae935)
+
+### Building a pipeline
+
+<img width="1057" alt="image-2" src="https://github.com/user-attachments/assets/adb500a3-fa45-4008-9b73-aa3a24e0f65c">
+
+(It's a relatively short part haha)
+
+### Assessing model performance
+
+#### Performance metrics for regression
+
+* mean squared error (MSE)
+* median absolute error (MAE)
+* almost correct predictions error rate (ACPER)
+
+#### Performance metrics for classification
+
+* precision-recall
+* accuracy
+* cost-sensitive accuracy
+* area under the ROC curve (AUC)
+* F-score
+
+A new one I learned is:
+
+* Cohen’s kappa statistic: it is a performance metric that applies to both multiclass and imbalanced learning problems. The advantage of this metric over accuracy is that Cohen’s kappa tells you how much better your classification model is performing, compared to a classifier that randomly guesses a class according to the frequency of each class
+
+<img width="155" alt="image-3" src="https://github.com/user-attachments/assets/04a31915-3da0-4f68-9cb5-238077746bc1">
+
+where p_o is called the observed argument, and p_e is the expected agreement
+
+<img width="807" alt="image-4" src="https://github.com/user-attachments/assets/a30b9e18-ea36-4028-983a-e64426af3d31">
+
+The value of Cohen’s kappa is always less than or equal to 1. Values of 0 or less indicate that the model has a problem. While there is no universally accepted way to interpret the values of Cohen’s kappa, it’s usually considered that values between 0.61 and 0.80 indicate that the model is good, and values 0.81 or higher suggest that the model is very good.
+
+#### Performance metrics for ranking
+
+Precision and recall can be naturally applied to the ranking problem. It’s convenient to think of these two metrics as measuring the quality of document search results. Precision is the proportion of relevant documents actually found in the list of all returned documents. The recall is the ratio of the relevant documents returned by the search engine, compared to the total number of the relevant documents that should have been returned.
+
+The drawback of measuring the quality of ranking models with precision and recall is that these metrics treat all retrieved documents equally. When a human looks at search results, the few top-most results matter more than the results shown at the bottom of the list.
+
+Discounted Cumulative Gain (DCG) is a metric that evaluates the relevance of documents in a ranked search result, with higher relevance scores discounted at lower positions. 
+
+- Cumulative Gain (CG) sums the relevance of all results without considering position
+- DCG adjusts this by reducing the gain for results appearing lower in the ranking, assuming that highly relevant documents are more useful when they appear earlier
+  
+There is also the Normalized Discounted Cumulative Gain (nDCG) which is calculated by comparing the DCG of the actual result to the DCG of the ideal ranking (IDCG). This normalization makes nDCG a useful metric for comparing performance across different queries.
+
+The metric is commonly used in search engine performance evaluation and model comparison, particularly in ranking tasks.
+
+### Hyperparameter tuning
+
+* grid search: it is the simplest hyperparameter tuning technique. It’s used when the number of hyperparameters and their range is not too large. The technique consists of discretizing each of the two hyperparameters, and then evaluating each pair of discrete values
+* random search: here, we provide a statistical distribution for each hyperparameter from which values are randomly sampled
+* coarse-to-fine search: analysts often use a combination of grid search and random search called coarseto-fine search. This technique uses a coarse random search to first find the regions of high potential. Then, using a fine grid search in these regions, one finds the best values for hyperparameters
+* other techniques: bayesian techniques, which use past evaluation results to choose the next values to evaluate; gradient-based techniques, evolutionary optimization techniques, and other algorithmic hyperparameter tuning methods
+* cross-validation
+
+### Shallow model training
+
+Shallow models make predictions based directly on the values in the input feature vector. Most popular machine learning algorithms produce shallow models. The only kind of deep models commonly used are deep neural networks
+
+1. Define a performance metric P
+2. Shortlist learning algorithms
+3. Choose a hyperparameter tuning strategy T
+4. Pick a learning algorithm A
+5. Pick a combination H of hyperparameter values for algorithm A using strategy T
+6. Use the training set and train a model M using algorithm A parametrized with hyperparameter values H
+7. Use the validation set and calculate the value of metric P for model M.
+8. Decide:
+    * if there are still untested hyperparameter values, pick another combination H of hyperparameter values using strategy T and go back to step 6
+    * otherwise, pick a different learning algorithm A and go back to step 5, or proceed to step 9 if there are no more learning algorithms to try
+9. Return the model for which the value of metric P is maximized
+
+### Bias-Variance tradeoff
+
+<img width="1075" alt="image-5" src="https://github.com/user-attachments/assets/23edfb73-ffa6-454a-8c95-91fc8fd3013e">
+
+<img width="711" alt="image-6" src="https://github.com/user-attachments/assets/cfec277e-aef9-42cb-b767-610f16f69f31">
+
+### Regularization
+
+**L1 (lasso)**
+
+<img width="558" alt="image-7" src="https://github.com/user-attachments/assets/9edd7067-80b4-4059-8960-562efd032b33">
+
+Where C is a hyperparameter that controls the importance of regularization. If we set C to zero, the model becomes a standard non-regularized linear regression model. On the other hand, if we set C to a high value, the learning algorithm will try to set most w to a value close or equal to zero to minimize the objective. The model will become very simple, which can lead to underfitting
+
+**L2 (ridge)**
+
+<img width="600" alt="image-8" src="https://github.com/user-attachments/assets/f05d7c69-2cd8-4277-a0e9-7a735738e21f">
+
+In practice, L1 regularization produces a sparse model, assuming the value of hyperparameter C is great enough. This is a model where most of its parameters equal exactly zero. So, as discussed in the previous chapter, L1 implicitly performs feature selection by deciding which features are essential for prediction, and which are not. This property of L1 regularization is useful when we want to increase model explainability. However, if our goal is to maximize the model performance on the holdout data, then L2 usually gives better results.
+
+**Elastic net**
+
+If we combine L1 and L2, we get a method called elastic net regularization.
+
+For neural nets there are also dropout, batch norm, also non-math methods like data augmentation and early stopping. 
+
+---
+
+I could not stream yesterday and today as I was travelling. Today I also had time to read while riding on the bus/train. I am home now and tomorrow I will stream
+
+That is all for today!
+
+See you tomorrow :)

From 4d5cf45103f4832ebc846170c56d6bba8b727638 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 9 Oct 2024 22:47:37 +0900
Subject: [PATCH 457/638] fixed date

---
 _posts/{2024-10-08-day-280.md => 2024-10-07-day-280.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2024-10-08-day-280.md => 2024-10-07-day-280.md} (100%)

diff --git a/_posts/2024-10-08-day-280.md b/_posts/2024-10-07-day-280.md
similarity index 100%
rename from _posts/2024-10-08-day-280.md
rename to _posts/2024-10-07-day-280.md

From c3975eda6ce923421a216cb47c1fa6b6572ea730 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 11 Oct 2024 00:43:26 +0900
Subject: [PATCH 458/638] Create 2024-10-10-day-283.md

---
 _posts/2024-10-10-day-283.md | 420 +++++++++++++++++++++++++++++++++++
 1 file changed, 420 insertions(+)
 create mode 100644 _posts/2024-10-10-day-283.md

diff --git a/_posts/2024-10-10-day-283.md b/_posts/2024-10-10-day-283.md
new file mode 100644
index 000000000..89f081f9d
--- /dev/null
+++ b/_posts/2024-10-10-day-283.md
@@ -0,0 +1,420 @@
+---
+layout: post
+title: (Day 283) Reading more of the MLE book + fine-tuning llama models
+categories: [traditional-machine-learning,theory,mlops,nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 283!
+A quick summary of today:
+* read ch. 6 and 7 of the MLE book by Andriy Burkov
+* worked on the movie review LLM project and streamed
+
+
+## Chapter 6 - Supervised model training (part 2)
+
+### Deep model training strategy
+
+We can train a model from scratch, or use a pre-trained model
+
+For images: VGG16/19, InceptionV3, ResNet50
+
+For text: BERT, ELMo
+
+We can use pre-trained models in two ways:
+
+1) use its learned parameters to initialize your own model
+2) use the pre-trained model as a feature extractor for your model
+
+#### Performance metric and cost function
+
+Performance metrics depend on the project. And popular cost functions are MSE (regression), categorical cross-entropy and binary cross-entropy (for classification).
+
+#### Parameter-initialisation strategies
+
+* all to 1
+* all to 0
+* random normally distributed values
+* random uniformly distributed values
+* Xavier normal - params are initialised to values sample from a truncated normal distribution, centred on 0, with st.d. equal to sqrt(2/(in+out)), where 'in' is the number of units in the preceding layer to which the current unit is connected, and 'out' is the number of units on the subsequent layer to which the current uni is connected
+* Xavier uniform - params are initialised to values sampled from a uniform distribution within [-limit, limit], where 'limit' is sqrt(6/(in+out))
+
+There are other methods as well
+
+#### Optimisation algorithms
+
+* gradient descent
+* stochastic GD
+* mini-batch SGD
+
+#### Learning rate decay schedules
+
+Learning rate decay consists of gradually reducing the value of the learning rate alpha as the epochs progress. Consequently, the parameter updates become finer
+
+* time-based
+* step-based
+* exponential-based
+
+#### Regularisation
+
+* L1 and L2 (not neural net specific)
+* dropout
+* early stopping
+* batch-norm
+
+#### Network size search and hyperparam tuning
+
+![image](https://github.com/user-attachments/assets/224d03f3-838b-48f1-a581-decf583ce651)
+
+### Stacking models
+
+![image](https://github.com/user-attachments/assets/c621b876-60b7-40eb-baab-6ac0594f9891)
+
+In model stacking there is also the risk of data leakage. 
+
+To avoid data leakage, we can create a synthetic training set following a process similar to cross-validation. First, split all training data into ten or more blocks. The more blocks the better, but the process of training the model will be slower.
+
+Temporarily exclude one block from the training data, and train the base models on the remaining blocks. Then apply the base models to the examples in the excluded block. Obtain the predictions, and build the synthetic training examples for the excluded block by using the predictions from the base models. Repeat the same process for each of the remaining blocks, and you will end up with the training set for the stacking model. The new synthetic training set will be of the same size as that of the original training set.
+
+### Dealing with distribution shift
+
+**Types of shift**
+
+* covariate shift - shift in the values of features
+* prior probability shift - shift in the values of the target
+* concept drift - shift in the relationship between the features and the labels
+
+We may know your data is affected by a distribution shift, but we don’t usually know what type of shift it is.
+
+If we have a large test set compared to the train set, we can randomly pick values and transfer them. However, often we have a very high number of training examples and relatively few test examples. In that case, a more effective approach is to use adversarial validation
+
+**Adversarial validation**
+
+Adversarial validation is a technique used to distinguish between training and test sets. The assumption is that feature vectors in both training and test examples contain the same number of features and represent similar information.
+
+1. Split the train set: 
+   - divide the original training set into two subsets: Training Set 1 and Training Set 2
+
+2. Transform train set 1:
+   - add the original label of each example in Training Set 1 as an additional feature
+   - assign a new label "Training" to each example
+
+3. Transform the test set:
+   - add the original label of each example in the test set as an additional feature
+   - assign a new label "Test" to each example
+
+4. Create a synthetic train set:
+   - merge the Modified Training Set 1 and the Modified Test Set to form a new Synthetic Training Set
+   - this new set is used to solve a binary classification problem, distinguishing between "Training" and "Test" examples
+
+5. Train a binary classifier:
+   - train a binary classifier on the Synthetic Training Set to return a prediction score
+   - the classifier will predict whether an example is from the training set or test set
+
+6. Apply the binary classifier:
+   - use the binary classifier on Training Set 2 and identify the examples predicted as "Test" with high certainty. These examples serve as validation data for the original problem
+   - remove examples from Training Set 1 that the model predicts as "Training" with the highest certainty
+
+7. Final training:
+   - use the remaining examples in Training Set 1 as the training data for your original problem
+
+8. Experimentation:
+   - experiment to find the ideal way to split the original training set and how many examples to use for training and validation
+
+### Handling imbalanced datasets
+
+* class weighting: some algorithms allow the data analyst to provide weights for each class. The loss in the cost function is typically multiplied by the weight. The data analyst may, for example, provide greater weight to the minority class. This makes it harder for the learning algorithm to disregard examples of the minority class, because it would result in much higher cost than without class weighting
+
+![image](https://github.com/user-attachments/assets/b5a367e3-027e-47ad-bdf5-2522269c9ca8)
+
+* ensamble of resampled datasets: the analyst randomly chunks majority examples into H subsets, then creates H training sets. After training H models, the analyst then makes predictions by averaging (or taking the majority) of the outputs of H models 
+
+![image](https://github.com/user-attachments/assets/9e9f899e-04ea-46cb-abfc-fd4f3d2d354a)
+
+* other: if SGD is used the class imbalance can be tackled in several ways. First, we can have different learning rates for different classes: a lower value for the examples of the majority class, and a higher value otherwise. Second, we can make several consecutive updates of the model parameters each time you encounter an example of a minority class
+
+### Model calibration
+
+A model is considered well-calibrated when the predicted score for an input example corresponds to the true probability of that example belonging to a particular class. For instance, in a binary classifier, if the model predicts a score of 0.8, then approximately 80% of the examples with that score should belong to the positive class.
+
+Most machine learning models are not well-calibrated. Calibration plots help visualize this. In binary classification, these plots show predicted scores on the X-axis (grouped in bins) and the fraction of positive examples in each bin on the Y-axis. 
+
+For multiclass classification, a one-versus-rest approach is used, where multiple binary classifiers are trained, each focusing on distinguishing one class from the rest. Calibration plots are generated for each class.
+
+In a well-calibrated model, the calibration plot aligns closely with the diagonal line, meaning predicted probabilities match actual outcomes. Logistic regression is often well-calibrated, while models like support vector machines and random forests tend to show a sigmoid shape in their calibration plots, indicating poorer calibration.
+
+![image](https://github.com/user-attachments/assets/7d831e70-6e48-46b0-a5d2-d8c0aeaf60ef)
+
+### Troubleshooting and error analysis
+
+#### Reasons for poor model behaviour
+
+If the model performs poorly on the train data, common reasons are:
+
+* the model architecture or learning algorithm are not expressive enough (try more advanced learning algorithm, an ensemble method, or a deeper neural network)
+* you regularize too much (reduce regularization)
+* you have chosen suboptimal values for hyperparameters (tune hyperparameters)
+* the features you engineered don’t have enough predictive power (add more informative features)
+* you don’t have enough data for the model to generalize (try to get more data, use data augmentation, or transfer learning)
+* you have a bug in your code (debug the code that defines and trains the model)
+
+If the model performs well on the train data, but poorly on the holdout data, common reasons are:
+
+* you don't have enough data for generalisation
+* the model is under-regularised
+* the train data distribution is different from the holdout data distribution
+* you have chosen suboptimal values for hyperparams
+* your features have low predictive power
+
+#### Iterative model refinement
+
+1. Train the model using the best values of hyperparameters identified so far
+2. Test the model by applying it to a small subset of the validation set (100−300 examples)
+3. Find the most frequent error patterns on that small validation set. Remove those examples from the validation set, because your model will now overfit to them
+4. Generate new features, or add more training data to fix the observed error patterns
+5. Repeat until no frequent error patterns are observed (most errors look dissimilar)
+
+Iterative model refinement is a simplified version of error analysis
+
+#### Error analysis
+
+Errors can be:
+
+* uniform, and appear with the same rate in all use cases, or
+* focused, and appear more frequently in certain types of use cases
+
+To fix and error pattern, we can use one or a combination of techniques:
+
+* preprocessing the input
+* data augmentation
+* labeling more training examples
+* engineering new features that would allow the learning algorithm to distinguish between 'hard' cases
+
+#### Using sliced metrics
+
+If the model will be applied to different segments of the use cases, it should be separately tested for each segment. For example, if you want to predict the solvency of borrowers, you would want your model to be equally accurate for both male and female borrowers. To achieve that, you can split your validation data into several subsets, one subset per segment. Then compute the performance metric by separately applying your model to each subset. Alternatively, you can separately evaluate the model on each class by applying precision and recall metrics.
+
+#### Fixing wrong labels
+
+When humans label the training examples, the assigned labels can be wrong
+
+Here is a simple way to identify the examples that have wrong labels. Apply the model to the training data from which it was built, and analyze the examples for which it made a different prediction as compared to the labels provided by humans. If you see that some predictions are indeed correct, change those labels.
+
+If time is limited, we can examine cases that were close to the decision boundary.
+
+#### Troubleshooting DL
+
+![image](https://github.com/user-attachments/assets/c7d831aa-53db-457c-b135-448a5a11ac93)
+
+Common issues and most common causes of problems with getting to overfit one minibatch by a neural network model.
+
+| Sign              | Probable causes                                                                 |
+|-------------------|---------------------------------------------------------------------------------|
+| Error goes up      | Flipped the sign of the loss function or gradient                               |
+|                   | Learning rate too high                                                          |
+|                   | Softmax taken over wrong dimension                                              |
+| Error explodes     | Numerical issue                                                                |
+|                   | Learning rate too high                                                          |
+| Error oscillates   | Data or labels corrupted (e.g. zeroed or incorrectly shuffled)                  |
+|                   | Learning rate too high                                                          |
+| Error plateaus     | Learning rate too low                                                          |
+|                   | Gradient not flowing through the whole model                                    |
+|                   | Too much regularization                                                         |
+|                   | Incorrect input to loss function                                                |
+|                   | Data or labels corrupted                                                        |
+
+### Best practices
+
+* deliver a good model
+* trust popular open source implementations
+* optimise a business-specific performance measure
+* upgrade from scratch
+* avoid correction cascades
+* use model cascading with caution (using the output of one model as the input to another)
+* write efficient code, compile, and parallelise
+* test on both newer and older data
+* more data beats cleverer algorithms
+* new data beats cleverer features
+* embrace tiny progress
+* facilitate reproducability
+
+## Chapter 7 - Model evaluation
+
+Depending on the model’s applicative domain and the organization’s business goals and constraints, model evaluation may include the following tasks:
+
+* Estimate legal risks of putting the model in production. For example, some model predictions may indirectly communicate confidential information. Cyber attackers or competitors may attempt to reverse engineer the model’s training data. Additionally, when used for prediction, some features, such as age, gender, or race, might result in the organization being considered as biased or even discriminatory
+* Study the main properties of distributions of the training data versus the production data. By comparing the statistical distribution of examples, features, and labels, in both training and production data, is how distribution shift is detected. A significant difference between the two indicates a need to update the training data, and retrain the model
+* Evaluate the performance of the model. Before the model is deployed in production, its predictive performance must be evaluated on the external data, that is, data not used for training. The external data must include both historical and online examples from the production environment. The context of evaluation on the real-time, online data should closely resemble the production environment
+* Monitor the performance of the deployed model. The model’s performance may degrade over time. It is important to be able to detect this and, either upgrade the model by adding new data, or train an entirely different model. Model monitoring must be a carefully designed automated process, and might include a human in the loop
+
+### Offline and Online evaluation
+
+An offline model evaluation happens when the model is being trained by the analyst. The analyst tries out different features, models, algorithms, and hyperparameters. Tools like confusion matrix and various performance metrics, such as precision, recall, and AUC, allow comparing candidate models, and guide the model training in the right direction.
+
+Online model evaluation is testing and comparing models in production by using online data. The difference between offline and online model evaluation, as well as the placement of each type of evaluation in a machine learning system
+
+![image](https://github.com/user-attachments/assets/57cfa4ad-ab1b-4857-ae07-82d3baf47bf3)
+
+Why do we evaluate both offline and online? The offline model evaluation reflects how well the analyst succeeded in finding the right features, learning algorithm, model, and values of hyperparameters. In other words, the offline model evaluation reflects how good the model is from an engineering standpoint.
+
+Online evaluation, on the other hand, focuses on measuring business outcomes, such as customer satisfaction, average online time, open rate, and click-through rate. This information may not be reflected in historical data, but it’s what the business really cares about. Furthermore, offline evaluation doesn’t allow us to test the model in some conditions that can be observed online, such as connection and data loss, and call delays.
+
+The performance results obtained on the historical data will hold after deployment only if the distribution of the data remains the same over time. In practice, however, it’s not always the case. Typical examples of a distribution shift include the ever-changing interests of the user of a mobile or online application, instability of financial markets, climate change, or wear of a mechanical system whose properties the model is intended to predict. As a consequence, the model must be continuously monitored once deployed in production. When a distribution shift happens, the model must be updated with new data and re-deployed. One way of doing such monitoring is to compare the performance of the model on online and historical data. If the performance on online data becomes significantly worse, as compared to historical, it’s time to retrain the model.
+
+A common scenario is to monitor user behavior in response to different versions of the model. One popular technique used in this scenario is A/B testing. We split the users of a system into two groups, A and B. The two groups are served the old and the new models, respectively. Then we apply a statistical significance test to decide whether the performance of the new model is better than the old one.
+
+Multi-armed bandit (MAB) is another popular technique of online model evaluation. Similar to A/B testing, it identifies the best performing models by exposing model candidates to a fraction of users. Then it gradually exposes the best model to more users, by keeping gathering performance statistics until it’s reliable.
+
+### A/B testing
+
+A/B testing is a widely used statistical technique for evaluating models in production, commonly applied to online systems like websites and mobile apps. It helps answer questions like, "Does the new model (mB) perform better than the existing model (mA)?" In A/B testing, live traffic is split into two groups: Group A (control) uses the old model, while Group B (experiment) uses the new model. Performance is compared through statistical hypothesis testing to determine if the new model improves business metrics, such as user engagement or sales.
+
+The test uses a null hypothesis (no significant change) and an alternative hypothesis (significant change) to measure whether the new model has a statistically significant impact on a specific metric. Though A/B testing involves a variety of tests depending on the metric being measured, the core principle of splitting traffic and assessing statistical significance remains constant.
+
+#### G-Test
+
+The G-test is a statistical method used in A/B testing, particularly when evaluating metrics based on yes/no questions. This test is suitable for any scenario where only two possible answers exist, such as "Did the user buy the recommended product?" or "Did the user renew the subscription?"
+
+In the G-test, users are split into two groups: A (control) and B (experiment). Group A uses the old model, while group B uses the new model. The actions of each user are recorded as either "yes" or "no," and the data is placed into a contingency table. The G-test compares the observed results with the expected results assuming both models (A and B) are equivalent.
+
+The formula for the G-test measures the difference between the two groups, and if the calculated G-value is high enough, it suggests that one model is likely performing better than the other. The significance of the difference is determined by the p-value. If the p-value is less than a threshold (e.g., 0.05), the null hypothesis (both models perform equally) is rejected, indicating a significant performance difference between the two models.
+
+To calculate the p-value in Python, you can use `scipy.stats.chi2`. For R, the equivalent function is `pchisq`. The test is considered valid when there are enough "yes" and "no" results in both groups, typically at least 100 of each per group.
+
+In cases where there are more than two models or multiple possible answers (e.g., "yes," "no," "maybe"), the G-test can still be applied, but the complexity increases, and interpreting the results becomes more challenging. For simplicity, it’s often recommended to compare just two models with binary outcomes.
+
+If sample sizes are small, Monte Carlo simulations can be used as an alternative to approximate the p-value.
+
+#### Z-Test
+
+The second formulation of A/B testing uses the Z-test when the question is "How many?" or "How much?"—for example, how much time a user spent on a website or how much money a user spent in a month.
+
+In this test, users are again split into two groups: A (control) using the old model, and B (experiment) using the new model. The goal is to determine if users in group B spend significantly more time on the website than those in group A. The null hypothesis assumes no difference between the two groups, while the alternative hypothesis assumes that users of version B spend more time than those of version A.
+
+To calculate the Z-test value, you first compute the sample mean and sample variance for both groups. The Z-value indicates how likely it is that the difference between A and B is significant. If the calculated Z is large, it suggests a performance difference between the two models. Under the null hypothesis (if A and B are equivalent), Z follows a normal distribution.
+
+To determine statistical significance, the p-value is computed, which represents the probability of observing a Z-value as extreme as the one calculated, assuming no difference between the models. For example, a Z-value of 2.64 corresponds to a p-value of 0.05, meaning there's a 5% chance that the observed difference is due to random variation. If the p-value is less than the chosen significance level (commonly 0.05), the null hypothesis is rejected, indicating that the new model performs better.
+
+However, if the p-value is greater than or equal to the significance level, the null hypothesis is not rejected, meaning there's no evidence that the new model is better. It's important to note that not rejecting the null hypothesis is not the same as proving the models are identical; it simply means no significant evidence was found. Continuing to gather evidence until the p-value drops below the threshold is not scientifically sound.
+
+In practice, a significance level of 0.05 or 0.01 is commonly used, though the appropriate value may vary depending on the application. A lower threshold requires more evidence to reject the null hypothesis.
+
+#### Warnings
+
+The above two are commonly used, but may not always be suitable for real-world business problems. Cassie Kozyrkov, Chief Decision Scientist at Google, points out that these tests often demonstrate that two models are different but fail to quantify the practical significance of that difference—i.e., whether the difference is "at least x." This is crucial when the cost or risk of replacing an old model with a new one is high. In such cases, a more tailored test is needed, and consulting a statistician is recommended.
+
+It's also important to ensure the accuracy of the A/B test’s programming implementation. Mistakes in the code could invalidate the model evaluation, as broken tests don’t reveal their faults. Additionally, measurements for groups A and B should be taken simultaneously, as user behavior varies with time of day, day of the week, and other factors like country, internet speed, or browser version. This ensures the integrity of the experiment.
+
+### Multi-Armed Bandit (MAB)
+
+Multi-armed bandit (MAB) is a more advanced and efficient method for online model evaluation compared to A/B testing. One major drawback of A/B testing is that it requires a large number of test results, meaning many users are exposed to suboptimal models for extended periods. MAB addresses this by minimizing the exposure to suboptimal models while still ensuring sufficient data is gathered to evaluate both models reliably. 
+
+This creates the **exploration-exploitation dilemma**: we need to explore enough to understand each model's performance (exploration) while routing users to the better-performing model as often as possible (exploitation).
+
+In probability theory, the multi-armed bandit problem involves allocating limited resources (users) between competing options (models or "arms") to maximize the expected reward (a business performance metric like average session duration or purchase rate). The **UCB1 (Upper Confidence Bound)** algorithm solves this by dynamically choosing which arm to "play" based on past performance and how well the algorithm knows each arm's performance.
+
+The UCB1 algorithm:
+1. Tracks how often each model (arm) is used and the average reward.
+2. Plays the arm with the highest **UCB value**, a balance between its past reward and how much uncertainty remains about its performance.
+
+The algorithm is mathematically proven to converge to the optimal solution, eventually favoring the best-performing model most of the time.
+
+
+### Statistical bounds on the model performance
+
+When reporting model performance, it is important to include not only the metric value but also the **statistical bounds** (or **statistical interval**). The terminology varies in different contexts, where "confidence interval" and "credible interval" are sometimes used interchangeably, but they have distinct meanings in frequentist and Bayesian statistics. For simplicity, in this context, a statistical interval can be understood as indicating that there is a specific probability (e.g., 95%) that the estimated parameter lies within the defined bounds.
+
+### Techniques for Establishing Statistical Bounds
+
+1. Statistical interval for classification error:
+   - for classification models, the error ratio (err) can be reported with a statistical interval. Given a test set size N, a 99% statistical interval can be calculated as: `[err - \delta, err + \delta]`, where delta is determined by the size of the test set and a value z_N that corresponds to the desired confidence level (e.g., 2.58 for 99%)
+
+2. Bootstrapping statistical interval:
+   - bootstrapping is a method used for reporting statistical intervals applicable to both classification and regression. This technique involves creating multiple random samples (with replacement) from the dataset, training a model, and computing the performance metric for each sample. By sorting the resulting metrics, analysts can establish a statistical interval based on the desired confidence level (e.g., 80%, 95%, or 99%)
+
+3. Bootstrapping prediction interval for regression:
+   - for regression models, bootstrapping can also be employed to compute prediction intervals for specific input feature vectors. Similar to the previous method, multiple regression models are built using bootstrap samples from the training set. Predictions for the input feature vector are obtained from these models, and a tight interval is established around these predictions to indicate where the predicted value is expected to lie with a specified confidence level.
+
+In practice, confidence levels of 95% or 99% are commonly used, with the number of bootstrap samples typically set to 100 to ensure a reliable estimation of model performance.
+
+### Evaluation of test set adequacy
+
+The test examples used to evaluate the model must also be designed in such a way that they allow the discovery of the model’s defective behavior before the model reaches production.
+
+#### Neuron coverage
+
+When we evaluate a neural network, especially one to be used in a mission-critical scenario, such as a self-driving car or a space rocket, our test set must have good coverage. Neuron coverage of a test set for a neural network model is defined as the ratio of the units (neurons) activated by the examples from the test set, to the total number of units. A good test set has close to 100% neuron coverage.
+
+A technique for building such a test set is to start with a set of unlabeled examples, and all units of the model uncovered. Then, iteratively, we
+
+1. randomly pick an unlabeled example i and label it,
+2. send the feature vector xi to the input of the model,
+3. observe which units in the model were activated by xi
+4. if the prediction was correct, mark those units as covered,
+5. go back to step 1; continue iterating until the neuron coverage becomes close to 100%.
+
+A unit is considered activated when its output is above a certain threshold. For ReLU, it’s usually zero; for a logistic sigmoid, it’s 0.5.
+
+#### Mutation testing
+
+In SE, this is done by doing small but crucial changes in the tested code. Like changing a + to a -, or removing an else.
+
+In modelling, to create a mutant statistical model, instead of modifying the code, we modify the training data. If the model is deep, we can also randomly remove or add a layer, or remove or replace an activation function. The training data can be modified by:
+
+* adding duplicated examples
+* falsifying the labels of some examples
+* removing some examples
+* adding random noise to the values of some features
+
+We say that we kill a mutant if at least one test example gets a wrong prediction by that mutant statistical model.
+
+### Evaluation of model properties
+
+When we measure the quality of the model according to some performance metric, such as accuracy or AUC, we evaluate its correctness property. Besides this commonly evaluated property of the model, it can be appropriate to evaluate other properties of the model, such  as robustness and fairness.
+
+#### Robustness
+
+This refers to the stability of a machine learning model's performance when noise is added to the input data. A robust model's performance degrades proportionally to the level of noise introduced.
+
+To measure robustness, the following approach is used:
+- modify input features randomly by replacing values with zeros, ensuring the Euclidean distance between the original and modified inputs is within a set threshold (δ)
+- the model is considered **ε-robust** to a δ-perturbation if the difference in model predictions on the original and perturbed inputs remains within ε
+
+To determine robustness:
+- perturb the test set using δ-perturbations
+- gradually increase δ until the model's prediction difference exceeds an acceptable threshold (ε̂). The model with the greatest δ before exceeding ε̂ is considered the most robust and suitable for production deployment
+
+#### Fairness
+
+Fairness in ML involves ensuring that models do not exhibit bias against protected attributes such as race, gender, or age. Biases can arise from human-collected data and can lead to unfair models.
+
+Two key concepts of fairness are:
+
+- **Demographic Parity**: ensures that the model gives positive predictions (e.g., university acceptance, loan approval) at equal rates across different groups. For example, the model should predict a positive outcome for women and men at the same rate. Mathematically, it requires the probability of a positive prediction to be the same for different groups based on a sensitive attribute
+
+- **Equal Opportunity**: ensures that qualified individuals from different groups have the same chance of receiving a positive prediction. This means that, for those who qualify, the model should predict a positive outcome at the same rate, regardless of group membership. It focuses on equalizing the true positive rate (TPR) across different groups.
+
+Despite excluding protected attributes from the feature set, models may still exhibit bias due to correlations with other features, making fairness a complex and domain-specific challenge.
+
+## PEFT LLMs for movie review
+
+Today I worked on the movie review project, and I talked about it more in-depth [on stream.](https://www.youtube.com/watch?v=7aQ-zeRmulc&t=4s)
+
+A quick summary of evaluations:
+
+```
+rmse_llama3.1_8b    1.581458
+
+rmse_llama3.2_3b    1.693664
+
+llama_3.1_8b: {'rouge1': 0.7066951125849534, 'rouge2': 0.3931419071408453, 'rougeL': 0.6032890188826542, 'rougeLsum': 0.7037531656430065}
+
+llama_3.2_3b : {'rouge1': 0.6371560616916654, 'rouge2': 0.2941605420908827, 'rougeL': 0.511882508004283, 'rougeLsum': 0.6224696637390135}
+```
+
+The models are on [my huggingface](https://huggingface.co/divakaivan)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From e7b9ae2aacd58816e57d02ebe4047feb1bb096dc Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 11 Oct 2024 00:44:25 +0900
Subject: [PATCH 459/638] Update 2024-10-10-day-283.md

---
 _posts/2024-10-10-day-283.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/_posts/2024-10-10-day-283.md b/_posts/2024-10-10-day-283.md
index 89f081f9d..5605866dd 100644
--- a/_posts/2024-10-10-day-283.md
+++ b/_posts/2024-10-10-day-283.md
@@ -402,10 +402,14 @@ Today I worked on the movie review project, and I talked about it more in-depth
 A quick summary of evaluations:
 
 ```
+Ratings:
+
 rmse_llama3.1_8b    1.581458
 
 rmse_llama3.2_3b    1.693664
 
+Text:
+
 llama_3.1_8b: {'rouge1': 0.7066951125849534, 'rouge2': 0.3931419071408453, 'rougeL': 0.6032890188826542, 'rougeLsum': 0.7037531656430065}
 
 llama_3.2_3b : {'rouge1': 0.6371560616916654, 'rouge2': 0.2941605420908827, 'rougeL': 0.511882508004283, 'rougeLsum': 0.6224696637390135}

From af48bd898354b7037c91407d31bd36fe654628ce Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 12 Oct 2024 02:18:55 +0900
Subject: [PATCH 460/638] Create 2024-10-11-day-284.md

---
 _posts/2024-10-11-day-284.md | 53 ++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 _posts/2024-10-11-day-284.md

diff --git a/_posts/2024-10-11-day-284.md b/_posts/2024-10-11-day-284.md
new file mode 100644
index 000000000..d236c3c46
--- /dev/null
+++ b/_posts/2024-10-11-day-284.md
@@ -0,0 +1,53 @@
+---
+layout: post
+title: (Day 284) PEFT LLMs - Gemma, Mistral, Llama, Qwen
+categories: [nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 284!
+A quick summary of today:
+* all-day fune-tuning [movie reviewer LLMs](https://huggingface.co/divakaivan)
+
+It is almost 3am, I had to wait for the final model (mistral) to finish fine-tuning. 
+
+I had to re-calculate the results from yesterday - rougeLSum for the summary and RMSE for the ratings because I was using the wrong ground truth data.
+
+I made a table in excel that I can keep track of my progress and my professor can easily check-in to check my progress
+
+**Prompt v1**
+
+*Your task is to summarise. You are a helpful assistant that helps me evaluate Korean reviews. For each movie you are given 50 reviews. Analyze the reviews, and for the movie itself return a score(1 to 10) and explanation for each of the following criteria: Emotional, Characters, Plot, Visuals, Pacing. Return the review in Korean.*
+
+Train size	185
+
+Evaluation size	15
+
+Evaluation is done using the ground truth dataset and the evaluation dataset
+
+The same hyperparameters were used for all models
+
+| Model            | Train time*     | Text RougeLSum | Ratings RMSE | Link  |
+|------------------|-----------------|----------------|--------------|-------|
+| llama-3.1-8b     | 2hr 58min       | 0.7037         | 1.4665       | [here](https://huggingface.co/divakaivan/Meta-Llama-3.1-8B-Instruct-KR-movie-reviewer) |
+| llama-3.2-3b     | 1hr 42min       | 0.6776         | 1.4550       | [here](https://huggingface.co/divakaivan/Meta-Llama-3.2-3B-Instruct-KR-movie-reviewer) |
+| llama-3.1-70b    |                 |                |              |       |
+| qwen-2-7b        | 2hr 40min       | 0.6666         | 1.8658       | [here](https://huggingface.co/divakaivan/Qwen2-7B-Instruct-KR-movie-reviewer) |
+| qwen-2.5-7b      |                 |                |              |       |
+| qwen-2.5-72b     |                 |                |              |       |
+| gemma-2-9b       | 4hr 9min        | 0.67602        | 1.6392       | [here](https://huggingface.co/divakaivan/gemma-2-9b-it-KR-movie-reviewer) |
+| gemma-2-27b      |                 |                |              |       |
+| mistral-7b       | 3hr 10min       |      tomorrow          |    tomorrow          |  [here](https://huggingface.co/divakaivan/mistral-7b-instruct-v0.3-KR-movie-reviewer) |
+| mistral-small-2409 |                 |                |              |       |
+| gpt4o-mini       |                 |                |              |       |
+
+*Train time is based on using Kaggle's free T4 GPU
+
+The llama models were trained yesterday but today I fine-tuned and evaluated the Qwen, Gemma and Mistral models.
+
+It was a long day.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From e3e0c628dbab819e09d28bd48d629256c928d6a5 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 12 Oct 2024 23:57:55 +0900
Subject: [PATCH 461/638] Create 2024-10-12-day-285.md

---
 _posts/2024-10-12-day-285.md | 64 ++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 _posts/2024-10-12-day-285.md

diff --git a/_posts/2024-10-12-day-285.md b/_posts/2024-10-12-day-285.md
new file mode 100644
index 000000000..8b6a057d1
--- /dev/null
+++ b/_posts/2024-10-12-day-285.md
@@ -0,0 +1,64 @@
+---
+layout: post
+title: (Day 285) I was muted on stream :( + more LLM fine-tuning
+categories: [nlp,applying-knowledge,traditional-machine-learning,theory]
+---
+
+# Hello :) Today is Day 285!
+A quick summary of today:
+* streamed for ~2hrs
+* fine-tuned more LLMs for the movie review project
+
+Today I streamed for 2 hours. And the 1st 1hr and 20mins are without audio :( 
+
+I only realised it when I checked my streaming software that my mic was muted :( 
+
+And what's worse is the stream I did on Thursday for ~45mins, has no audio at all :( I have no idea when or why I muted the mic in Streamlabs.
+
+[Here](https://www.youtube.com/watch?v=gYJCuvjzdwk&t=1s) is the link to it.
+
+I read some scikit-learn docs - a bit about calibrating classification models. It's an area that I still need to read into to better understand.
+
+Also worked on fine-tuning models. Last night I finished the mistral-7b fine-tuning but the results are just ...
+
+I give it the normal input, and the output starts producing more reviews. The model I am fine-tuning is an instruct model (or at least says so), but it seems to just have learned to generate movie reviews. So that's 3.5hrs down the drain.
+
+I also tried to fine-tune some bigger models (>20B params) but Kaggle's free T4 GPU runs out of memory on those. 
+
+Afterwards, I started fine-tuning base models using a 2nd prompt (a modified version of the instruction task):
+
+```
+Your task is to summarize. You are a helpful assistant who will help me evaluate Korean reviews. Up to 50 reviews are provided for each movie. Analyze the reviews by each dimension: emotion, character, plot, visuals, and pacing, and return a dimension-specific rating (1-10) and a summary description. To do this, please perform the following steps:
+
+1. First, for each review, determine which dimension it is related to: emotion, character, plot, visuals, or pacing.
+
+2. Assign a dimension-specific rating (1-10) to each review.
+
+3. Aggregate the ratings of each individual review by dimension to generate a final dimension-specific rating and summary for the movie.
+
+4. Translate the final dimension-specific summary for each movie into Korean and return it with dimension-specific rating (1-10).
+```
+
+The results so far are:
+
+This is using the 1st version of the prompt:
+
+```
+Your task is to summarise. You are a helpful assistant that helps me evaluate Korean reviews. For each movie you are given 50 reviews. Analyze the reviews, and for the movie itself return a score(1 to 10) and explanation for each of the following criteria: Emotional, Characters, Plot, Visuals, Pacing. Return the review in Korean.
+```
+
+<img width="656" alt="image" src="https://github.com/user-attachments/assets/164517d6-1fdf-436b-a69d-d584593cd377">
+
+And 2nd version:
+
+<img width="663" alt="image" src="https://github.com/user-attachments/assets/06185be2-0274-4ec0-93c1-6880d6190173">
+
+[All of the models are on my huggingface account](https://huggingface.co/divakaivan)
+
+One of the things I will do tomorrow - use OpenAI's fine-tuning tool to fine-tune gpt4o-mini. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 964ad528e2c1fa8ad8eb3df4c0c17f49abff3095 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 13 Oct 2024 22:12:05 +0900
Subject: [PATCH 462/638] Create 2024-10-13-day-286.md

---
 _posts/2024-10-13-day-286.md | 106 +++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)
 create mode 100644 _posts/2024-10-13-day-286.md

diff --git a/_posts/2024-10-13-day-286.md b/_posts/2024-10-13-day-286.md
new file mode 100644
index 000000000..0dad0aee3
--- /dev/null
+++ b/_posts/2024-10-13-day-286.md
@@ -0,0 +1,106 @@
+---
+layout: post
+title: (Day 286) MLE - Model deployment from Andriy Burkov
+categories: [theory,mlops]
+---
+
+# Hello :) Today is Day 286!
+A quick summary of today:
+* read the 2nd to last chapter of Andriy Burkov's MLE book
+
+## Chapter 6 from Andriy Burkov's MLE book
+
+<img width="978" alt="image" src="https://github.com/user-attachments/assets/a3f8ff8c-6004-4459-8f22-1a6e3ea20097">
+
+Deploying a model means to make it available for accepting queries generated by the users of the production system. Once the production system accepts the query, the latter is transformed into a feature vector. The feature vector is then sent to the model as input for scoring. The result of the scoring then is returned to the user.
+
+A trained model can be deployed in various ways. It can be deployed on a server, or on a user’s device. It can be deployed for all users at once, or to a small fraction of users.
+
+A model can be deployed following several patterns:
+
+* statically, as a part of an installable software package
+* dynamically on the user’s device
+* dynamically on a server
+* via model streaming
+
+### Static deployment
+
+Static deployment involves packaging a machine learning model as part of traditional software deployment. The model and feature extractor are bundled into an installable binary, such as a dynamic-link library (DLL), shared object file, or serialized resource for virtual machine systems (e.g. Java, .NET). This approach offers fast execution, maintains user privacy, supports offline usage, and transfers model maintenance to the user. However, it complicates model upgrades, as updating the model often requires updating the entire application. Additionally, models with specific computational requirements, like GPU access, may limit deployment flexibility.
+
+### Dynamic deployment on user's device
+
+Dynamic deployment allows users to run part of a machine learning system on their device, but unlike static deployment, the model is not embedded in the application’s binary. This approach enables easier model updates without requiring a full application update. It can also select the appropriate model based on the device's compute resources. There are several methods for dynamic deployment:
+
+- deployment of model parameters: the model file only contains learned parameters, while the device runs the model in a pre-installed environment. Examples include TensorFlow Lite for mobile and Apple’s Core ML
+  
+- deployment of a serialized object: the model is deployed as a serialized object that is deserialized on the device, avoiding the need for a runtime environment but potentially leading to large updates
+
+- deployment to browser: using frameworks like TensorFlow.js, models can be run in a browser, leveraging JavaScript and possibly a GPU
+
+#### Advantages and Drawbacks
+
+The main advantage of dynamic deployment is faster model access for users and reduced server load for organizations. Browser-based deployments further lower infrastructure requirements by only needing to serve web pages with model parameters. However, drawbacks include higher bandwidth and startup costs due to repeated model downloads and challenges in maintaining model versions across users. Additionally, model security is at risk as it can be reverse-engineered by third parties.
+
+### Dynamic deployment on a server
+
+Dynamic deployment on a server is the most common approach, offering scalability and ease of model updates. Models are made available via REST API or gRPC services and deployed on virtual machines, containers, or serverless platforms.
+
+#### Deployment on a virtual Machine
+
+In a virtual machine (VM) deployment, a web service running on a VM processes incoming HTTP requests, applying the model to the input data and returning predictions in formats like JSON or XML. Multiple VMs can be used in parallel, with a load balancer routing requests based on resource availability. VMs can be scaled manually or automatically to handle varying traffic loads. The simplicity of this architecture makes it a common choice, but maintaining VMs adds operational complexity and costs. Additionally, network latency can slow down responses, and there’s some overhead from running separate operating systems for each VM.
+
+<img width="532" alt="image" src="https://github.com/user-attachments/assets/b61c88e6-2634-4c15-8534-83c2105b866f">
+
+#### Deployment in a Container
+
+Container-based deployment is a more modern approach where the model and necessary services are packaged within containers (like Docker). Containers share the host machine’s operating system, making them more lightweight and efficient compared to VMs. Containers can be managed using orchestration systems like Kubernetes, which can automatically scale the number of running containers based on demand. Containers can even scale to zero when idle, significantly reducing resource usage and costs. However, setting up and managing containerized deployments is more complex and requires specialized expertise.
+
+<img width="808" alt="image" src="https://github.com/user-attachments/assets/2e77175e-d427-44d6-9bd3-55b9f76fca41">
+
+#### Serverless deployment
+
+In serverless deployment, models are packaged into functions (i.e. AWS Lambda) that automatically scale based on requests. This is highly cost-efficient and easy to maintain but limited by file size, runtime memory, and the unavailability of GPUs for deep learning models.
+
+#### Model streaming
+
+Model streaming is an alternative to REST API-based deployment, designed for complex workflows involving multiple models. In streaming deployments, data flows continuously through a series of models and transformations managed by a stream-processing engine (i.e. Apache Flink or Apache Storm). Rather than handling individual requests and responses, streaming applications process data in real-time, providing faster insights for scenarios that require multiple intermediate predictions, such as natural language processing or recommendation systems. This approach reduces latency, improves resource efficiency, and is particularly suited for applications that follow a predictable sequence of operations. However, setting up a streaming infrastructure requires careful design and is more suitable for specific use cases where data is continuously processed.
+
+<img width="1020" alt="image" src="https://github.com/user-attachments/assets/94f729a6-7a52-44cd-a36a-aa01c5f75bfe">
+
+### Deployment strategies
+
+#### Single deployment
+
+This is the simplest and quickest strategy, where the new model directly replaces the old one. You serialize the new model, upload it (along with a new feature extractor if needed), and replace the previous version on the server or device. In cloud environments, this involves updating virtual machines or containers. Though easy to implement, it's the riskiest since all users are affected if the new model contains bugs.
+
+#### Silent deployment
+
+In this approach, both the old and new models run in parallel. The new model’s predictions are logged, but users continue seeing results from the old model. This allows engineers to detect issues with the new model without affecting users. However, running two models simultaneously increases resource consumption, and certain applications may require real-world feedback to properly evaluate the new model.
+
+#### Canary Deployment
+
+With canary deployment, the new model is rolled out to a small subset of users while the majority continue using the old model. This method enables validation of the model's real-world performance with minimal risk to the broader user base. However, rare bugs can be missed if the subset of users is too small to trigger those errors.
+
+#### Multi-Armed Bandits (MAB)
+
+This strategy involves deploying multiple versions of a model and using the MAB algorithm to optimize their use in real-time. Initially, the system explores all versions, but over time, it prioritizes the best-performing model, automatically routing users to it. MAB solves the challenge of online evaluation and deployment simultaneously, ensuring most users experience the optimal model after the system has enough performance data.
+
+### Best practices
+
+* algorithmic efficiency (using the Big O notation)
+* deployment of deep models - inference might require a GPU, so deploy the model on a GPU instance, and the rest of the app on a CPU instance (if that part of the app does not require a GPU which can be costly)
+* caching and memoization
+* use appropriate serialization tools for the model and code
+* start with a simple model
+* test on outsiders
+
+
+---
+
+Today I woke up at 6am and had to travel to and back from Seouk. While travelling, I read the chapter and took the notes above. I had a look at the last chapter on 'Model Serving, Monitoring, and Maintenance' which I should finish next week - maybe even on stream (will see). Also next week I will be able to stream every day 🥳
+
+Unfortunately, I could not do the gpt4o fine-tuning which I will do tomorrow when I get to the lab ^^
+
+That is all for today!
+
+See you tomorrow :)

From 90d4d3eebba2664c068c615590bbff128f56e358 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 14 Oct 2024 23:48:41 +0900
Subject: [PATCH 463/638] Create 2024-10-14-day-287.md

---
 _posts/2024-10-14-day-287.md | 211 +++++++++++++++++++++++++++++++++++
 1 file changed, 211 insertions(+)
 create mode 100644 _posts/2024-10-14-day-287.md

diff --git a/_posts/2024-10-14-day-287.md b/_posts/2024-10-14-day-287.md
new file mode 100644
index 000000000..04dcc865d
--- /dev/null
+++ b/_posts/2024-10-14-day-287.md
@@ -0,0 +1,211 @@
+---
+layout: post
+title: (Day 287) Calibrating classification models
+categories: [theory,mlops,traditional-machine-learning,applying-knowledge,nlp]
+---
+
+# Hello :) Today is Day 287!
+A quick summary of today:
+* finished MLE by Andriy Burkov
+* streamed for 2 hours (and learned a lot about model calibration)
+* finished the LLM fine-tuning (for now)
+
+## MLE by Andriy Burkov - Chapter 9 - Model serving, Monitoring, and Maintenance
+
+![image](https://github.com/user-attachments/assets/9a083d71-1f8c-452a-b632-2f3531303924)
+
+### Properties of the model serving runtime
+
+The model serving runtime is the environment in which the model is applied to the input data. The runtime properties are dictated by the model deployment pattern
+
+* security and correctness: the runtime is responsible for authenticating the user's identity, and authorising thei requests. Things to check:
+  * whether a specific user has authorised access to the models they want to run
+  * whether the names and the values of params passed correspond to the model's requirements
+  * whether those params and their values are currently available to the user
+
+* easy of deployment: the model runtime should allow easy updates without disrupting the entire application. For a web service on a physical server, updating the model involves replacing the model file and restarting the service. For virtual machines or containers, updating requires stopping old instances and starting new ones with the updated model image. In model streaming-based applications, the new model version is streamed, making the application stateful. Once updated, the application contains the new model and related components like feature extractors, with modern stream-processing engines supporting these stateful updates
+
+![image](https://github.com/user-attachments/assets/fb77f3e6-d0d9-45ab-a513-1d6b61b6033a)
+
+* guarantees of model validity: an effective runtime will automatically make sure the model its running is valid (alond with the feature extractor and any other components). The model should not be served in prod in either of the below two conditions:
+  * at least one of the end2end test examples was not scored correctly, or
+  * the value of the metric, calculated on the confidence test set examples, is not within the acceptable range
+ 
+* easy of recovery: the runtime allows easy fallback to previous versions in case of errors
+* avoidance of training/serving skew: avoid using a different codebase (i.e. one for train and one for testing). If using two different codebases is unavoidable, then we need to make sure to add logging of feature values generated in the prod env
+* avoidance of hidden feedback loops (using one model's output as the input for another)
+
+### Modes of model serving
+
+#### Serving in batch mode
+- batch mode is used when a model processes large amounts of input data, such as all users of a product or all incoming events
+- it is more resource-efficient compared to on-demand mode and tolerates some latency
+- a typical batch size ranges between 100 to 1,000 feature vectors, often using powers of two (ie. 32, 64, 128)
+- the output is usually saved in a database for future use
+- examples of batch mode applications:
+  - generating weekly music recommendations
+  - classifying incoming comments as spam or not
+  - extracting named entities from documents for search engines
+
+#### Serving on demand to a human
+
+- the six steps for serving a model on demand to a human:
+  1. validate the request
+  2. gather the context
+  3. transform context into model input
+  4. apply the model and get output
+  5. ensure the output makes sense
+  6. present the output to the user
+- context refers to the user's situation, which is either explicitly (like music recommendations) or implicitly (like auto-replies) provided
+- a good context provides enough data for feature extraction, logging, and debugging
+- examples of good context for various problems:
+  - device malfunctioning: vibration, noise levels, firmware, etc
+  - emergency room: age, vital signs, medical history, etc
+  - credit risk: salary, debts, past credit payments, etc
+  - advertisement display: webpage title, user position, browser info, etc
+- the feature extractor transforms the context into model input, often as a separate object in the pipeline
+- confidence scores are critical before serving the result, and low confidence can trigger prompts for user verification
+- prediction confidence can be combined with error cost estimates to guide decision-making and manage risk
+
+#### Serving on demand to a machine
+
+Machines are often served through streaming, as their data requirements are fixed and predetermined. A well-designed streaming application topology efficiently utilizes available resources. Demand can fluctuate, with high demand during the day and low at night. Autoscaling in the cloud helps manage this but isn't fast enough for sudden spikes.
+
+To handle these spikes, message brokers like RabbitMQ or Apache Kafka are used. Ondemand requests are placed in an input queue. The model reads input data in batches from the queue, processes them, and writes predictions to the output queue. A separate process reads the predictions from the output queue and delivers them to users. This setup helps manage demand spikes and improves resource efficiency
+
+![image](https://github.com/user-attachments/assets/59de8bf0-fe90-4446-8bed-f7cbdda865ea)
+
+### Model serving in the real world
+
+* be ready for errors
+* dealing with errors
+
+![image](https://github.com/user-attachments/assets/7e979b03-0ea2-40b8-93a2-a1f4b3980471)
+
+* be ready for, and dealing with, change
+* be ready for, and dealing with, human nature
+  * avoid confusion
+  * manage expectations
+  * gain trust
+  * manage user fatigue (and how the model affects the UX)
+  * beware of the creep factor (when the user perceives the model’s predictive capacity as too high; then user feels uncomfortable, especially when a prediction concerns their very private details. Make sure the system doesn’t feel like “big brother” and doesn’t take too much responsibility)
+
+### Model monitoring
+
+(just a reminder to myself, and anyone reading - [EvidentlyAI](https://www.evidentlyai.com/ml-observability-course) offers an amazing and 100% free course on AI monitoring)
+
+Monitoring makes sure that:
+
+* the model is serverd corrrectly
+* the performance of the model remains within acceptable limits
+
+#### What can go wrong?
+
+Monitoring models in production is crucial to catch potential issues early, such as:
+
+- degraded performance: new training data might worsen model accuracy
+- data drift: changes in live data without model updates can lead to outdated performance
+- feature extraction issues: updates to feature extraction code that aren’t reflected in the model can lead to unpredictable results
+- resource changes: loss or modification of resources needed for feature generation may impact model output
+- adversarial attacks: models, especially in e-commerce and media, can be targeted by malicious actors aiming to exploit weaknesses or manipulate behavior
+
+Additional training data can introduce biases or inconsistencies if not labeled correctly. Furthermore, dual-use concerns arise when models are repurposed for harmful applications, such as voice impersonation for fraud or surveillance in weapon systems.
+
+#### What and how to monitor?
+
+Effective monitoring is essential to ensure the model performs well against a regularly updated confidence test set, avoiding distribution shifts. Regular testing on end-to-end examples is also necessary. Key metrics to monitor include accuracy, precision, recall, and particularly prediction bias. A well-calibrated model should show a similar distribution of predicted and observed classes; deviations indicate potential issues that need investigation.
+
+Monitoring should also track changes in data sources, such as abandoned database columns or changes in definitions/formatting. Statistical tests like the Chi-square and Kolmogorov–Smirnov tests can identify significant shifts in feature value distributions. Alerts should be sent to stakeholders upon detection of these shifts.
+
+Also, monitoring should include:
+
+- numerical stability: alert if `nan`s or infinite values are encountered
+- computational performance: detect and warn about regressions and suspicious usage fluctuations, such as significant changes (ie. 30% hourly or 15% weekly) in model serving numbers
+- prediction statistics: monitor minimal and maximal prediction values, as well as the median, mean, standard deviation, latency, memory, and CPU usage
+
+For recommender systems, monitoring click-through rates (CTR) is important - a decline indicates the need for model updates.
+
+A balance is necessary between frequent alerts for minor metric changes and alert fatigue. Stakeholders can be allowed to set their own alert thresholds for non-critical cases.
+
+Logging monitoring events is crucial for traceability. Monitoring tools should visualize performance trends and allow metric computation on data slices, enabling targeted analysis of model degradation.
+
+Finally, logging additional data can assist in identifying problem sources and improving existing or future models, even if it cannot be analyzed in real-time.
+
+#### What to log?
+
+It is important to log enough information to reproduce any erratic system behavior during a future analysis. If the model is served to a front-end user, such as a website visitor or a mobile application user, it’s worth saving the user’s context at the moment of the model serving
+
+It is also useful to include the model input, that is, the features extracted from the context, and the time it took to generate those features.
+
+Also:
+
+* the model's output and time it took to generate it
+* the new context of the user (once they observed the model's output)
+* the user's reaction to the output (click, where, how long it took)
+
+#### Monitoring for abuse
+
+Monitoring for abuse is crucial as some users may exploit your model for their business needs, sending an excessive number of requests or attempting to reverse-engineer training data. Here are some strategies to prevent such abuse:
+
+- charge users: implement a pay-per-request model
+- introduce delays: create longer response times for requests
+- user blocking: block users who exhibit suspicious behavior
+
+Additionally, attackers might manipulate the model by submitting data that benefits them, potentially degrading overall model quality. To mitigate this risk:
+
+- data validation: only trust user data if corroborated by multiple users
+- reputation scoring: assign reputation scores to users and be cautious with low-reputation data
+- behavior classification: classify user behavior as normal or abnormal, and reject data from those exhibiting abnormal patterns
+
+To stay ahead of potential attacks, regularly update your models and incorporate new data and features to detect fraudulent activities effectively.
+
+### Model maintenance
+
+Most production models must be regularly updated. The rate depends on several factors:
+
+* how often it makes errors and how critical they are
+* how “fresh” the model should be, so as to be useful
+* how fast new training data becomes available
+* how much time it takes to retrain a model
+* how costly it is to deploy the model
+* how much a model update contributes to the product and the achievement of user goals
+
+#### When to update?
+
+When first deployed, a model may not be perfect and will require updates to correct critical errors. Over time, models can stabilize, needing fewer updates, but some models, like recommender systems, must always remain 'fresh' and updated frequently. The frequency of updates depends on business needs, user demands, and the speed of new data availability. In some cases, such as with machine translation models, updates can happen less frequently. But when new data streams in rapidly, labeling delays may affect update timing
+
+Complex models, especially those requiring hyperparameter tuning, can take significant time to retrain, potentially lasting days or weeks. Using parallelizable algorithms and GPUs can help speed up the process. In cases where updates are costly, such as in healthcare, less frequent updates might be necessary. Sometimes, even a small model improvement can provide substantial business outcomes, especially when user disturbance is low and deployment costs are manageable.
+
+#### How to update?
+
+Ideally, new model versions should be deployed without interrupting the system. This is often achieved in virtualized or containerized infrastructures by replacing the image of a virtual machine or container. The deployment process typically involves versioning data, code, and models, ensuring that when a new model is trained, it is tested before replacing the old model in production. Techniques like A/B testing or multi-armed bandit algorithms can be used to determine when to replace a model.
+
+Monitoring for distribution shifts is important, and when detected, human labelers may validate the changes. In streaming or on-demand model scenarios, updates are triggered when new data or state changes. A message broker architecture, involving a human in the loop for labeling, can facilitate model serving and updating. Continuous integration workflows are useful, ensuring that models are retrained as soon as new data becomes available. In these workflows, storing hyperparameters, labeler identities, and model versions is essential for traceability.
+
+Before deploying new models, ensure the system has adequate storage and RAM, as newer models may be larger and slower. Monitor whether the new model introduces more costly errors or disproportionately affects certain user groups. If a validation fails, avoid deploying the model and roll it back if issues arise post-deployment.
+
+Finally, in systems with model cascading, ensure that updating one model does not negatively affect others in the sequence. If models are interdependent, all models in the cascade must be updated accordingly.
+
+### This book is awesome!
+
+## Fine-tuning gpt4o-mini and submitting all the results to my professor
+
+<img width="1076" alt="image" src="https://github.com/user-attachments/assets/1a6965e8-20a7-468a-91e0-8280224e78df">
+
+[Here](https://docs.google.com/spreadsheets/d/1sOTsiy1OrxUYMaaLNOc3eZVN1huGlh8HBjEEuNilzBA/edit?gid=0#gid=0) is a link to the sheet
+
+## Streamed 
+
+[Part 1](https://www.youtube.com/watch?v=BhYCGVn63ls)
+
+Then my internet stopped for a minute so I had to reset my stream, and
+
+[Part 2](https://www.youtube.com/watch?v=UFS0PtjSJrw)
+
+On stream I also tested and learned more about calibration, and I ended up putting it all up in a [Kaggle notebook](https://www.kaggle.com/code/divakaivan12/classifier-calibration-example)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 88d8701d108b7387507730c903bd507c903c0db7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 15 Oct 2024 23:45:33 +0900
Subject: [PATCH 464/638] Create 2024-10-15-day-288.md

---
 _posts/2024-10-15-day-288.md | 438 +++++++++++++++++++++++++++++++++++
 1 file changed, 438 insertions(+)
 create mode 100644 _posts/2024-10-15-day-288.md

diff --git a/_posts/2024-10-15-day-288.md b/_posts/2024-10-15-day-288.md
new file mode 100644
index 000000000..6021cf057
--- /dev/null
+++ b/_posts/2024-10-15-day-288.md
@@ -0,0 +1,438 @@
+---
+layout: post
+title: (Day 288) New book + scikit-learn's inspection module
+categories: [traditional-machine-learning,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 288!
+A quick summary of today:
+* started reading [Financial Data Engineering](https://learning.oreilly.com/library/view/financial-data-engineering/9781098159986/) on O'Reilly
+* explored PDP and ICE plots (through scikit-learn) on stream
+
+## Financial Data Engineering Chapter 1 - Foundations
+
+* what is finance?
+
+![image](https://github.com/user-attachments/assets/0d33f3ff-0bbd-42e2-bb96-24278c06fba0)
+
+* what is data engineering? (a proper answer can be found in Joe Reis and Matt Housley's book haha)
+
+* what is _financial_ data engineering?
+
+![image](https://github.com/user-attachments/assets/8d5e6088-0865-48c6-888d-15d0dd87cfe9)
+
+The book's definition:
+
+_Financial data engineering is the domain-driven practice of designing, implementing, and maintaining data infrastructure to enable the collection, transformation, storage, consumption, monitoring, and management of financial data coming from mixed sources, with different frequencies, structures, delivery mechanisms, formats, identifiers, and entities, while following secure, compliant, and reliable standards._
+
+### Why financial data engineering?
+
+#### Volume, Variety, and Velocity of Financial Data
+
+**Volume**  
+
+Big data in finance is defined by its large size. Data volumes have increased due to socio-technological changes and advancements in digital technology, such as high-frequency trading and widespread adoption of mobile payments. This increase presents opportunities like better fraud detection and complex pattern recognition, but also technical challenges in data storage, querying, and aggregation.
+
+**Velocity**  
+
+Data velocity refers to the speed of data generation and ingestion. Financial markets, high-frequency trading, and social media have increased the velocity of data. This creates opportunities for quicker reactions and insights but introduces challenges in building systems capable of handling real-time data processing with high accuracy.
+
+**Variety**  
+
+Financial data comes in various types—structured, semi-structured, and unstructured. Recent years have seen growth in diverse data formats, such as regulatory filings, news, and social media posts. While this variety improves prediction and analytics, it also requires advanced infrastructure to manage, clean, and integrate different data types efficiently.
+
+### Finance-Specific Data Requirements and Problems
+
+* there is a lack of standardisation in some key areas:
+  * identification system for financial data
+  * classification system for financial assets and sectors
+  * financial information exchange
+ 
+* lack of established data standards for financial transaction processing
+* dispersed and diverse sources of financial data
+* adoption of multiple data formats by companies, data vendors, providers, and regulators
+* complexity in matching and identifying entities within financial datasets
+* lack of reliable methods to define, store, and manage financia lreference data
+* lack of relevant data for understanding and managing various financial problems due to poor data collection processes
+* the constant need to adapt data and tech infrastructure to meet new market and regulatory demands
+* the constant need to record, store, and share financial data for various regulatory and market purposes
+* absence of standardised practices for cleaning and ensuring the quality of financial data
+* difficulty in aggregating data across various silos and divisions within financial institutions
+* creating consolidated tapes, which integrate market data from multiple sources including trade and quote information across various venues, continues to pose technological and market challenges
+* balancing innovation and competitiveness with regulatory compliance
+* persisting concerns regarding security, privacy, and performance in cloud migration strategies
+* continued reliance on legacy technological systems due to organisational innertia and risk aversion
+
+### Financial ML
+
+* FinGPT
+* BloombergGPT
+* financial machine learning has proven to be very successful and is likely to be a major factor in shaping the future of financial markets, but it shouldn’t be ignored that it presents major challenges that need to be taken into consideration (like the problem of false discoveries)
+
+### FinTech
+
+![image](https://github.com/user-attachments/assets/4bb765e7-7b3b-4ae5-9b6b-075f4dabd8e8)
+
+### Regulatory requirements and compliance
+
+To avoid costly financial crises, the financial sector has been subjected to a large number of regulations, both national and international.
+
+A significant part of financial regulatory requirements concerns the way banks should collect, store, aggregate, and report data.
+
+To meet regulatory requirements, financial institutions must establish specialized financial data engineering and management teams to develop and maintain a reliable data infrastructure. This infrastructure should efficiently capture, process, and consolidate relevant data and metadata from various sources, while upholding stringent security standards and ensuring both operational and financial resilience. It must also provide risk and compliance officers with rapid, accurate access to the data necessary for proving regulatory compliance. Additionally, financial data engineers are responsible for establishing and upholding a financial data governance framework that ensures data quality and security, fostering trust among management, stakeholders, and regulators.
+
+### Responsibilities and activities of a financial data engineer
+
+The role of a financial data engineer varies based on the job, the hiring institution, and the firm's data maturity. Data maturity reflects an organization’s progress in managing and utilizing its data and is typically categorized into three stages:
+
+**Starting with Data**  
+
+In the early stages of data maturity, financial institutions may initiate digital transformation efforts. Data engineers handle broad responsibilities, including data engineering, software development, analytics, and infrastructure. The focus is on rapid feature expansion and speed over adherence to best practices.
+
+**Scaling with Data**  
+
+At this stage, the institution evaluates its processes to identify bottlenecks and scalability needs. Financial data engineers work on improving the scalability, security, reliability, and quality of the data infrastructure by adopting best practices, such as governance, security standards, DevOps, and system scalability
+
+**Leading with Data**  
+
+Once an institution becomes data-driven, its operations are fully automated, and scaling is seamless. Engineers specialize in areas like site reliability, platform engineering, MLOps, and FinOps. There is a continued focus on optimization and new integrations within the well-established data processes and infrastructure
+
+### Skills of a financial data engineer
+
+* financial domain knowledge (fin. instruments, markets, data generation mechanisms, company reports, financial variables, financial theory terms, compliance and privacy concepts, regulation and data protection laws)
+* technical data engineering skills (databases, cloud, data workflow and frameworks, infrastructure, programming, analytical skills)
+* soft and business skills (explaining complex concepts, identifying data requirements, staying updated on financial technology trends, establish data access policies, analyze high-value data needs, provide guidance on leveraging financial data for analysis and ML)
+
+## Chapter 2 - Financial data ecosystem
+
+### Sources of financial data
+
+The sources may differ depending on:
+
+* accessibility: whether the data is open access, commercially licensed, or proprietary
+* usability: whether the data source is easy to use and extract data from, and whether the data is provided in a raw format or in a clean structure that is ready to use
+* coverage: whether the data source offers complete and wide coverage of financial data
+* reliability: whether a source is trusted, secure, timely, and durable
+* quality: whether the data provided by the sources is of high quality
+* documentation: whether additional info is available on the data structures, fields, and content of the data source
+
+There are difference sources, but below 6 are identified.
+
+#### Public financial data
+
+Public financial data includes various types of freely accessible information provided through regulatory disclosures, institutional databases, public research, and free stock market APIs. Regulatory disclosures, such as those required by the SEC, include filings like 10-Ks, 10-Qs, and 8-Ks, which provide essential financial performance details of companies. The SEC’s EDGAR database is a primary source for these filings, handling thousands of documents daily. Additionally, international organizations such as the World Bank, European Central Bank, and the Federal Reserve offer vast collections of economic and financial data. Public research datasets, like the Ken French Data Library, offer specialized financial data for academic purposes. Moreover, free stock market APIs from platforms like Yahoo Finance and Google Finance provide access to real-time data such as stock prices, news, and financial reports, often supporting programmatic access through APIs. The main challenges with public financial data include dealing with raw, unstructured formats and incomplete disclosures designed to balance transparency with competitive protection.
+
+#### Security exchanges
+
+Market exchanges like LSE, NYSE gather and manage diverse datasets on the transactions they handle and provide up-to-date data crucial for trading and investment purposes.
+
+#### Commercial data vendors, providers, and distributors
+
+Commercial companies specialising in financial data and software providers collect and curate data from a large number of sources, including regulatory filings, news agencies, surveys, company websites, brokers, banks, rating agencies, company reports, bilateral agreements with companies, exchange venues, and many more.
+
+Vendors like Morningstar, Bloomberg, and FactSet offer standardized data, comprehensive documentation, and customer support, with options for customized solutions. Their services vary in data coverage, delivery mechanisms, and pricing, enabling financial institutions to select based on specific needs such as data frequency, quality, or budget. Key considerations when choosing a vendor include the required data types, delivery formats, data update frequency, and technical limitations.
+
+#### Survey data
+
+Survey data, collected through questionnaires, provides valuable insights by enhancing existing datasets. For example, banks use surveys to assess clients' financial knowledge and risk levels to personalize investment plans. Organizations like the Institute for Supply Management (ISM) use surveys to create key economic indicators like the Purchasing Managers' Index (PMI). While survey data is flexible and customizable, challenges include respondent bias, incomplete responses, and potential framing bias, making ethical survey design and bias mitigation crucial for reliable outcomes.
+
+#### Alternative data
+
+These sources are not inherently embedded in the financial system, unlike trading venues, payment systems, and commercial banking. Examples are:
+
+* satellites (i.e. collecting shipping images)
+* social media (tweets)
+* articles
+* blog posts
+* news channels
+* weather observers
+* online clicking patterns and browsing history
+* shipping and pipeline trackers
+* emails and chats
+* product and service reviews
+* security, social, and environmental scores and ratings
+
+#### Confidential and proprietary data
+
+Confidential and proprietary data in financial institutions includes three main types:
+
+* contextual, nontransactional data: this includes information about clients, employees, suppliers, products, pricing, and branch locations
+* transactional data: generated from activities such as payments, deposits, credit card transactions, and trading
+* analytical data: created by internal research teams and analysts, covering areas like sales forecasting, customer segmentation, credit scoring, and market analysis
+
+Additionally, financial institutions must report confidential data (i.e. trade details, large transactions, suspicious activity) to regulatory bodies for compliance with AML, KYC, tax, and risk management requirements.
+
+### Structures of financial data
+
+#### Time series data
+
+The time series is the most common data structure in financial markets, mainly due to the dynamic and transactional nature of financial activities that happen over time. Examples of financial activities that generate time-series data include trading, pricing, payment, investment, estimation, valuation, risk measures, volatility figures, corporate events, performance and accounting, and many more.
+
+#### Cross-sectional data
+
+A cross-section refers to a set of records for one or more variables observed among multiple entities at a specific point in time. In cross-sectional data, the time dimension is not relevant; the focus is instead on the variables themselves. The entities in a cross-section typically share similar characteristics, such as companies within the same industry, investments aligned with a particular strategy, or fund managers.
+
+Financial cross-sectional data arises from the interaction of various participants in financial markets, including financial institutions, brokers, investors, consumers, traders, and managers, as well as diverse entities like financial assets, investment strategies, and the behaviors and choices of consumers and firms.
+
+This type of data is useful for understanding and explaining significant differences among financial entities at a specific time, such as why different stocks yield varying returns, why companies exhibit different behaviors, or why performance levels vary among fund managers. Additionally, cross-sectional analysis serves as a vital tool for identifying correlations, causal relationships, or anomalies across different entities, which can be crucial for investment decisions, policy development, and comprehending market dynamics.
+
+#### Panel data
+
+Panel data combines both time series and cross-sectional structures, representing data for a set of variables across multiple entities at different points in time.
+
+![image](https://github.com/user-attachments/assets/5fff5c1b-6651-4e89-914a-0037dbe03aa5)
+
+#### Matrix data
+
+The use of matrix structures is quite common in finance. The best example is perhaps portfolio optimization theory, in particular Markowitz’s mean-variance optimization (MVO). According to MVO, when constructing a financial investment portfolio, three elements should be taken into consideration:
+
+* the expected return on the assets in the portfolio
+* the variance (risk) of the asset returns
+* the covariances between the asset returns
+
+If we have a portfolio with three assets, two matries can be constructed:
+
+![image](https://github.com/user-attachments/assets/eebe01bf-f104-4a32-8206-a3bee44f9989)
+
+Using these matrix representations, MVO relies on matrix algebra to optimize the portfolio’s expected return for a given level of risk appetite.
+
+#### Graph data
+
+Network analysis, or network science, is essential for studying these complex interactions. Traditional time-series or cross-sectional data are insufficient for analyzing financial networks, necessitating graph data, which comprises nodes (entities) and links (relationships).
+
+Graph data can be represented in several ways:
+
+* network visualization: a 2d diagram showing nodes and links, best for small networks
+* adjacency matrix: an N × N matrix indicating connections between nodes
+* adjacency list: an array where each node links to its neighbors
+* edge list: a list of edges defined as tuples of source and target nodes, possibly with attributes
+
+Financial graph data often stems from other data sources, such as transaction histories, executive datasets, or curated datasets from financial data vendors like S&P Global. Building networks often requires additional processing, such as calculating correlations between stock prices for similarity networks.
+
+It went in-depth on graph data and types of graphs which was a bit surprising because usually texts don't concentrate on graph data too much.
+
+#### Text data
+
+* it is the most common
+* have certain characteristics. For example, terms like 'risk', and 'liability' may have a negative connotation in certain contexts, but they are common in finance
+
+### Types of financial data
+
+#### Fundamental data
+
+Financial fundamental data, also called financial statements data, balance sheet data, or corporate finance data, conveys information about firms’ structures, operations, and performance.
+
+#### Market data
+
+Market data includes price and related information for financial instruments traded on market venues such as the New York Stock Exchange (NYSE) or in off-exchange OTC markets.
+
+#### Transaction data
+
+A financial transaction is a legal agreement between two or more parties to exchange financial securities, services, goods, risks, or commodities in return for money.
+
+Generally speaking, a financial transaction can be characterized by at least five elements: transaction specifications, transaction parties, initiation date, settlement date, and settlement method.
+
+#### Analytics data
+
+This type of data is often derived from other types of data such as fundamental, market, and transaction data. This data is typically computed using simple formulas, statistical models, machine learning techniques, and financial theories
+
+Examples:
+
+* news and market sentiment (novelty, score, relevance, impact)
+* financial risk measures (value at risk, option Greeks, bond duration, implied volatility)
+* market indexes (MSCI, Global Indexes),
+* ESG scores
+* others
+
+#### Alternative data
+
+Alternative data refers to nontraditional datasets used for financial analysis, investment, or forecasting, including sources like news, social media, satellite images, patents, and consumer behavior. Unlike conventional financial data, alternative data is often unstructured, lacking standard identifiers, and may be incomplete or biased.
+
+The key advantage of alternative data is its novelty and diversity, which can provide a competitive edge in portfolio analysis. However, challenges include the need for significant investment in data structuring and the potential for biases that are difficult to detect.
+
+#### Reference data
+
+Alternative data encompasses nontraditional datasets used for financial analysis, investment, or forecasting, which do not originate from conventional financial contexts like trading or corporate finance. Examples include news articles, social media posts, satellite imagery, patents, Google trends, and consumer behavior insights. These datasets can be sourced from data vendors or public repositories.
+
+The main advantage of alternative data lies in its novelty and diversity, offering financial institutions the potential for a competitive edge in portfolio analysis and decision-making. For example, satellite image data can predict financial performance by analyzing the number of cars in company parking lots.
+
+However, working with alternative data presents challenges, such as its often unstructured format, which requires substantial investment to organize and analyze. Additionally, alternative datasets frequently lack standardized identifiers, may be incomplete or imbalanced, and can contain biases that are difficult to identify. Thus, while alternative data can enhance financial analysis, it necessitates careful handling and expertise to unlock its full potential.
+
+#### Entity data
+
+Entity data provides information about corporate entities and their characteristics, often used in conjunction with reference data for deeper insights. Key elements of entity data include company name, identifiers, year of establishment, legal form, ownership structure, sector classification, associated individuals, credit ratings, ESG scores, risk exposures, and major corporate events.
+
+Commercial datasets, such as LSEG's legal entity data, Moody's Orbis database, and SwiftRef's entity data, are available for various applications, including corporate finance analysis, risk management (ie. supplier or credit risk), and compliance efforts such as anti-money laundering (AML), know your customer (KYC), and client due diligence (CDD).
+
+### Benchmark financial datasets
+
+Notable financial datasets:
+
+* CRSP: offers comprehensive stock data for over 32,000 securities, including price data and corporate actions
+* Compustat: standardized dataset for fundamentals of 80,000+ international public companies, useful for historical analysis
+* TAQ: high-frequency trade and quote data from major exchanges, with time precision from seconds to nanoseconds
+* I/B/E/S: earnings estimates database for over 60,000 companies, contributed by global analysts since 1976
+* IvyDB OptionMetrics: historical options data, including quotes and implied volatility for equity and index options
+* TRACE: bond transaction data from FINRA, providing real-time information on OTC transactions
+* Orbis: global database with information on 450 million companies, focusing on financial strength and ownership
+* SDC Platinum: source for corporate finance and market event data, including mergers and acquisitions
+* S&P DJI: provider of historical index data for various markets, including detailed information on popular indexes
+
+## Chapter 3 - Financial identification systems
+
+Financial identifiers are unique character sequences (i.e. alphanumeric codes) that accurately identify financial entities (like companies, transactions, and assets) across various datasets and systems.
+
+A financial identification system establishes principles for generating, interpreting, storing, and maintaining these identifiers, often referred to as symbology.
+
+### Importance of Financial Identifiers
+
+* transaction identification: identifiers are crucial for distinguishing financial instruments and entities involved in market transactions, facilitating efficient transactions and communication among participants
+
+* reporting compliance: regulators require accurate reporting from financial institutions, which necessitates the aggregation and consolidation of data through identifiers. Regulatory frameworks like MiFID II mandate the use of specific identifiers in reporting
+
+* exchange listing and trading: identifiers are essential for listing and trading securities, enabling easy tracking and analysis of financial instruments
+
+* data analysis: in financial analysis, identifiers help in cross-sectional studies by ensuring accurate data sampling, quality checks, and entity matching across datasets
+
+### Desired properties of financial identifiers
+
+* uniqueness: financial identification system must uniquely identify financial entities and never assign the same identifier to distinct entities
+* globality: a financial identification system must be able to accommodate the expanding ecosystem of financial activities and entities
+* scalability: a scalable financial identification system does not exhaust its pool of assignable identifiers
+* completeness: if an identification system is created to identify a set of financial entities, then each entity in the set must have an identifier
+* accessibility: a financial identifier data should be accessible, meaning it should not be restricted by license fees or usage limits, or monopolized by market players
+* timeliness: refers to the identifier's ability to process and generate identifiers quickly and efficiently
+* authenticity: an authentic financial identification system can confirm whether a specific identifier was generated by it
+* granularity: a financial identification system is granular if it can scale to provide different levels of details that reflect market hierarchies
+* permanence: a reliable financial identification system must ensure that identifiers and their associated data are permanent and durable
+* immutability: a robust financial identification system must ensure the immutability of its identifiers over time
+* security
+
+### Financial identification systems landscape
+
+The current landscape of financial identification systems is extensive, comprising open source and proprietary systems designed with different properties and for various purposes. Several standardization efforts have been initiated, yet a universally accepted standard has not been established.
+
+Financial identification systems in use globally:
+
+![image](https://github.com/user-attachments/assets/977f6d01-87be-479f-b765-75fd8bf475a5)
+
+## Chapter 4 - Financial entity system (FES)
+
+Vast majority of data in the world exsits in unstructured formats. To this end, many financial institutions develop systems to extract, recognize, identify, and match financial entities within financial datasets.
+
+The book defines a financial entity as:
+
+_A financial entity is a real-world object that may be recognized, identified, referenced, or mentioned as an essential part of financial market operations, activities, reports, events, or news. A financial entity may be human or not. It can be tangible (i.e. an ATM machine), intangible (i.e. common stock), fungible (i.e. one-dollar bills), or infungible (i.e. loans). A financial entity system is an organized set of technologies, procedures, and methods for extracting, identifying, linking, storing, and retrieving financial entities and related information from different sources of financial data and content._
+
+### Financial named entity recognition
+
+NER is the task of detecting and recognizing named entities in text, such as persons, companies, locations, events, symbols, time, etc.
+
+Example outcome of NER in finance:
+
+![image](https://github.com/user-attachments/assets/94a6b409-83ea-4a98-ae86-9f940bc97360)
+
+Making a knowledge base:
+
+![image](https://github.com/user-attachments/assets/16eaed9d-f995-41f3-a644-3ae304868fde)
+
+This chapter went in depth into different approaches for developing NER and it seems this is a good reference to learn about NER in general. 
+
+## Chapter 5 - Financial data governance
+
+As financial markets grow, the techniques and applications for collecting, storing, and utilizing financial data are also evolving. This has raised concerns among both the financial sector and regulatory bodies, which are advocating for robust data controls, quality assurance, privacy regulations, and enhanced security measures. Consequently, data governance frameworks have surfaced as an effective strategy for establishing and enforcing rules and principles that guide data practices within financial institutions.
+
+Data governance is critical to securing financial data, ensuring regulatory compliance, and fostering trust among stakeholders. By implementing robust DG, financial institutions can safeguard sensitive information, adhere to legal requirements, and maintain the integrity of their financial operations.
+
+DG definition by Google:
+
+_Data governance is everything you do to ensure data is secure, private, accurate, available, and usable. It includes the actions people must take, the processes they must follow, and the technology that supports them throughout the data life cycle._
+
+Financial data governance defined by the book:
+
+_Financial data governance is a technical and cultural framework that establishes a set of rules, roles, practices, controls, and implementation guidelines to ensure the quality, integrity, security, and privacy of financial data in compliance with both general and financial domain–specific internal and external policies, standards, requirements, and regulations._
+
+Implementing a financial data governance framework necessitates significant investment, involving the establishment and integration of the framework within a financial institution's data infrastructure and training employees to adhere to governance principles. The value of such a framework is especially pronounced in financial organizations due to its impact on performance and risk management.
+
+Effective data governance enhances data quality, a crucial factor in financial operations and decision-making. Poor data quality leads to operational inefficiencies and hinders institutions' abilities to gain insights, make informed decisions, and communicate accurately. Additionally, robust data governance alleviates the burden of constant data quality checks for employees and boosts confidence among developers and business teams regarding compliance and quality.
+
+Financial institutions face various risks, including cyberattacks, data breaches, biases in applications, erratic data affecting model outcomes, and privacy risks associated with third-party data sharing. Regulatory frameworks like the Sarbanes–Oxley Act, Bank Secrecy Act, and GDPR have emerged in response to these risks, compelling institutions to implement strong data governance frameworks to ensure compliance.
+
+Given the complexity of data governance, this chapter will present a practical framework focused on three key areas common to all financial institutions: data quality, data integrity, and data security and privacy.
+
+### Data quality
+
+Data quality in financial institutions refers to how well a dataset meets its intended use in various applications. Since financial data plays a crucial role in decision-making and product development, maintaining high-quality data is essential. Issues with data quality, known as "data downtime," can disrupt operations, damage customer trust, hinder research, and negatively affect decisions.
+
+To ensure data quality, financial institutions need a Data Quality Framework (DQF), which varies by institution but generally involves assessing key data quality dimensions (DQDs). These dimensions, such as errors, outliers, biases, granularity, and timeliness, help measure the overall quality of financial data. While the specific DQDs may differ depending on the institution's needs, they form the foundation of a robust data quality strategy.
+
+* dimension 1: data errors: records that have been recorded erroneously, and therefore reflect invalid or incorrect values (i.e. negative stock price, wrong decimal place)
+* dimension 2: data outliers
+* dimension 3: data biases
+* dimension 4: data granularity: the level of detail within a dataset
+* dimension 5: data duplicates
+* dimension 6: data availability and completeness
+* dimension 7: data timeliness: is the data available and accessible at the time it is expected to be; does the data reflect the most recent observations
+* dimension 8: data constraints: the degree to which data conforms to pre-defined technical and business rules and limitations
+* dimension 9: data relevance: the degree to which available data aligns with the specific problem or purpose it aims to address
+
+### Data integrity
+
+Data integrity refers to the principles and practices that ensure data remains consistent, accurate, traceable, and reliable throughout its lifecycle. As data undergoes various transformations, movements, and adjustments, maintaining its integrity is crucial. In financial institutions, ensuring data integrity involves adhering to key principles such as standards, backups, archiving, aggregation, data lineage, catalogs, ownership, contracts, and reconciliation. These principles help guarantee that data is trustworthy, usable, and compliant with business and regulatory requirements.
+
+* principle 1: data standards
+* principle 2: data backups (serve in data loss accidents)
+* principle 3: data archiving (server data retention purposes)
+* principle 4: data aggregation ([Basel's 13 principles](https://www.bis.org/publ/bcbs239.pdf))
+* principle 5: data lineage
+* principle 6: data catalogs (set of metadata combined with search and data management tools that allow a data producer or consumer to find and document a data asset within a financial institution)
+* principle 7: data ownership (data ownership involves designating an individual or team with the task of overseeing the collection, cleansing, maintenance, sharing, and management of a particular data asset within a financial institution)
+* principle 8: data contracts (_API-like agreements between Software Engineers who own services and Data Consumers that understand how the business works in order to generate well-modeled, high-quality, trusted, real-time data_)
+* principle 9: data reconciliation (the process of aligning different sets of records to ensure consistency and accuracy in financial transactions and balances across various systems)
+
+### Data security and privacy
+
+#### Data privacy
+
+Data privacy involves the governance of data to ensure it is collected, stored, processed, and shared according to legal regulations and the consent of the individual. It protects personally identifiable information (PII), such as names, social security numbers, and financial details. Laws like the EU’s GDPR give individuals rights over their data, including access, deletion, and portability. Other regulations, such as CCPA in the US and PIPEDA in Canada, provide similar protections. In system design, balancing data privacy with data sharing is crucial, as excessive sharing can lead to breaches, while overly restricting it may limit innovation. A privacy culture within organizations, backed by management, is key to ensuring data privacy.
+
+#### Data anonymisation
+
+It is a practice that enhances data security and privacy by transforming data to obscure its identifiable elements, making it unlinkable to specific individuals. Proper anonymization renders data unusable if breached. In financial institutions, it can be adopted as a best practice and is sometimes legally required. For instance, under GDPR, anonymized data is exempt from certain privacy restrictions.
+
+**Anonymization strategies**
+
+* identifiability spectrum: data ranges from fully identifiable (i.e. direct identifiers like names or social security numbers) to fully anonymized. Indirect identifiers (quasi-identifiers) can still reveal identity when combined with other data. The goal is to decide how much identifiability is acceptable based on reidentification risks
+* analytical integrity: anonymized data must remain useful for analysis, preserving key relationships between variables without distorting the dataset’s analytical value
+* reversibility: some anonymization processes may allow reidentification if necessary, such as when internal use data needs to be re-accessed (i.e. credit card numbers)
+* simplicity: methods range in complexity. Simple approaches are easier to implement and reverse, while complex methods offer more robust protection but at higher costs
+* static vs. dynamic: static anonymization alters data once for storage, while dynamic anonymization applies on-the-fly for query results, impacting speed and performance
+* deterministic vs. nondeterministic: deterministic methods yield the same anonymized output consistently, while nondeterministic methods may vary each time they are applied
+
+**Techniques for data anonymization**
+
+* generalization: replaces precise data (i.e revenues) with ranges
+* suppression: omits sensitive fields entirely
+* distortion: alters numerical data by adding noise
+* swapping: shuffles data between records to obscure patterns
+* masking: obfuscates sensitive data with modified characters
+
+#### Data encryption
+
+Data encryption is a key practice in information security, converting plain text into unreadable ciphertext using cryptographic algorithms. It ensures that unauthorized users cannot access the data without the decryption key. Data can be encrypted in three states: at rest, in transit, and in use. Encryption methods include symmetric encryption, which uses a single key (i.e. AES), and asymmetric encryption, which uses a public-private key pair (i.e. RSA). While symmetric encryption is faster, asymmetric encryption is more secure but computationally intensive. In the financial sector, encryption is essential for security and compliance, as exemplified by standards like ISO 9564 for PIN management.
+
+#### Access control
+
+Access control is a data security practice that manages who can access data and resources, defining access privileges and a strategy for handling anomalies. It consists of two main components: authentication, which verifies the identity of the user, and authorization, which determines what resources the authenticated user can access and their level of privilege.
+
+Best practices include the principle of least privileges, granting users only the minimum permissions necessary, and multifactor authentication for enhanced security. Additionally, access control audit logs monitor user activities to detect unauthorized access or anomalies. This practice is essential in financial institutions to prevent data breaches and misuse.
+
+## [Stream](https://www.youtube.com/watch?v=Yj5B2W_j5wo&t=26s)
+
+Today I discovered something new and cool - the inspection module in sklearn, and learned about partial dependence plots (PDP) and individual conditional expectation (ICE) plots which can provide some level of interpretability to models. I explored it on stream, but have some more to explore on tomorrow's stream.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 3b290e228eb2806057fe7c9626e5d7f4e6d27211 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 16 Oct 2024 21:33:04 +0900
Subject: [PATCH 465/638] Create 2024-10-16-day-289.md

---
 _posts/2024-10-16-day-289.md | 490 +++++++++++++++++++++++++++++++++++
 1 file changed, 490 insertions(+)
 create mode 100644 _posts/2024-10-16-day-289.md

diff --git a/_posts/2024-10-16-day-289.md b/_posts/2024-10-16-day-289.md
new file mode 100644
index 000000000..0bf6d60c0
--- /dev/null
+++ b/_posts/2024-10-16-day-289.md
@@ -0,0 +1,490 @@
+---
+layout: post
+title: (Day 289) Reading more of the Fin. DE book + stream
+categories: [traditional-machine-learning,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 289!
+A quick summary of today:
+* read more of the financial data engineering book
+* streamed and learned about permutation importance in sklearn
+
+## Financial Data Eng Ch. 6 - Overview of the FDE lifecycle
+
+![image](https://github.com/user-attachments/assets/a2108839-f86e-4d8d-8fb7-c203208b1622)
+
+Below are the criteria for building a financial data engineering stack
+
+### Criterion 1: open source vs. commercial software
+
+The main advantages of commercial software are:
+
+* vendor accountability
+* product integrations
+* enterprise-grade features
+* user-friendly experience
+
+The disadvantages of commercial software are:
+
+* cost
+* bulky products
+* vendor lock-in
+* lack of customisation
+
+The main advantages of open source are:
+
+* cost
+* customisation
+* community
+* transparency
+
+The main disadvantages of open source are:
+
+* support
+* documentation
+* complexity
+* compatibility
+* security
+* confidentiality
+
+### Criterion 2: ease of use vs. performance
+
+The passage discusses the tradeoff between ease of use and performance in software and hardware technology. Complex technologies, like low-level programming languages (like C) or specialized hardware (i.e. GPUs, FPGAs), provide significant performance advantages, but are harder to learn, implement, and manage. On the other hand, simpler, user-friendly technologies (i.e. Python, CPUs) are easier to use but may offer less optimal performance.
+
+In financial markets, especially in high-frequency trading, low latency is critical, as even millisecond delays can result in large financial losses. Trading firms invest in optimized systems, such as Direct Market Access (DMA) and colocation with exchanges, to reduce latency. FPGAs are often used to gain speed advantages in processing orders, but these setups can raise fairness concerns due to differential access to market data.
+
+### Criterion 3: cloud vs. on premises
+
+### Criterion 4: public vs. private vs. hybrid cloud
+
+### Criterion 5: single vs. multi-cloud
+
+### Criterion 6: monothonic vs modular codebase
+
+## Chapter 7 - Data ingestion layer
+
+### Data transmission and arrival processes
+
+Data ingestion is a process wherein data gets transmitted from a given source, travels over a network, and arrives at its destination. Understanding the details of such a process is critical for designing a reliable financial data infrastructure that meets different business and technical requirements.
+
+#### Data transmission protocols
+
+They enable communication between systems over a network. These protocols form essential layers in data infrastructure, particularly for industries like finance that rely heavily on data transfers.
+
+Protocols are organized in layered models, with the most popular being the seven-layer OSI model and the simpler four-layer TCP/IP model. The TCP/IP model includes:
+
+* Application Layer: contains protocols like HTTP, SMTP, and DNS for web, email, and file transfers
+* Transport Layer: responsible for reliable data transfer using TCP (connection-oriented) and UDP (faster but less reliable). It also includes TLS for secure transmission
+* Network Layer: handles data routing using IP addresses, with IPsec providing encryption
+* Network Access Layer: manages data transmission over physical hardware (ie. Ethernet, WiFi)
+
+Financial communication standards, like EBICS, leverage these protocols for secure and reliable transactions, utilizing TCP/IP and TLS encryption. Understanding these protocols is crucial for designing secure and efficient financial data infrastructures.
+
+![image](https://github.com/user-attachments/assets/438e7b31-1ae8-4264-826a-0fae2ffbfd3b)
+
+#### Data arrival processes (DAP)
+
+Data may arrive at the ingestion layer with different temporal and structural patterns. 
+
+* scheduled DAP
+* event-driven DAP
+* homogeneous DAP (the data has pre-determined consistent properties)
+* heterogeneous DAP (data may possess variable attributes such as extension, format, type, content, schema)
+* single-item DAP (data is ingested either on a record-at-a-time or file-at-a-time basis)
+* bulk DAP
+
+### Data ingestion formats
+
+* general-purpose formats (csv, tsv, txt, json, xml, etc)
+* big data formats (parquet, avro, orc)
+* in-memory formats (apache arrow, RDD by Spark)
+* standardised financial formats: the above can be used, but if different ones are used by market participants issues are inevitable. To that end, a standard format has been created (or at least an attempt) - FpML, FIX, IFX, MDDL, FEDI, OFX, XBRL, ISO 8583, ISO 15022, ISO 20022, SWIFT messages
+
+### Data ingestion technologies
+
+Below are 6 of the most common ingestion technologies used in financial markets
+
+* financial APIs
+* financial data feeds
+* secure file transfer
+* cloud access
+* web access
+* specialised financial software
+
+### Data ingestion best practices
+
+* meet business requirements
+* design for change
+* enforce data governance (data validation, logging and reporting, lineage and visibility, security)
+* perform benchmarking and stress testing
+
+## Chapter 8 - Data storage layer
+
+### Principles of data storage system (DSS) design 
+
+1. meet business requirements
+2. data modelling (conceptual, logical, and physical plan)
+3. transactional guarantee (ensure data consistency and reliability - ACID/BASE)
+4. consistency tradeoffs (CAP theoreom - consistency, availability, partition tolerance - involve balancing between ensuring all nodes have the latest data (consistency) and keeping the system operational during network issues (availability))
+5. scalability
+6. security
+
+### Data storage modelling 
+
+* SQL vs. noSQL
+* primary (secury and permanent repository for data storage) vs. secondary (may not maintain the same level of data consistency, but is generally intended for use cases where this discrepancy does not cause significant issues)
+* operational (OLTP) vs. analytical (OLAP)
+* native vs. non-native (i.e. storing graph data as a graph in neo4j vs in postgres)
+* multi-model vs. polyglot persistence (using neo4j for graphs, mongodb for content and posts, redis for caching frequently accessed data vs using postgres and its variations to store different types of data)
+
+### Data storage models
+
+#### The data lake model
+
+A data lake is a centralized repository that stores massive amounts of raw data, allowing flexible storage of both structured and unstructured data. It supports simple data ingestion, data variety, and integration while enabling cost-effective archiving and analysis. However, data lakes require proper governance, including security, privacy, and data cataloging to prevent turning into disorganized "data swamps." They are widely used in financial institutions for cloud migration, data aggregation, compliance, and handling unstructured data.
+
+#### The relational model
+
+The popular and highly trusted DSSs are built on the relational DSM, which organizes data in rows and columns, forming a table or relation
+
+![image](https://github.com/user-attachments/assets/7603205f-7cc4-4854-9835-0a7ede346613)
+
+By design, relational databases offer the strongest ACID transactional guarantee among all DSSs.
+
+* analytical querying: one of the strongest arguments in favor of using SQL databases is their advanced querying capabilities
+* schema enforcement: relational dbs require and enforce a data schema
+* data modelling is common with relational dbs
+* normalisation is the most popular SQL data modelling technique
+
+#### The document model
+
+json and xml are popular formats
+
+Document dbs offer:
+
+* schema flexibility
+* document modelling
+* horizontal scalability
+* ACID transactions
+* performance
+
+#### The time series model
+
+Time series data, consisting of sequential measurements over time, is widely used in financial markets for analyzing trends, risks, and predicting prices. To efficiently manage this growing data, specialized time series databases (TSDBs) have emerged, offering features like fast querying, time-based aggregations, and data immutability. Native TSDBs like kdb+ and InfluxDB are optimized for high performance, especially in time-critical applications like high-frequency trading, while general-purpose databases like PostgreSQL and MongoDB offer time series extensions to handle such data.
+
+#### The message broker model
+
+![image](https://github.com/user-attachments/assets/1e9880ad-37aa-4183-898e-ca5fef61e29a)
+
+Message brokers are crucial for event-driven, distributed, and streaming architectures, facilitating the exchange of messages between applications. They decouple data production and consumption, enabling independent scaling of producer and consumer applications based on load. With fault tolerance, message brokers ensure that failures in producers or consumers don’t impact operations. Key elements include topics, which organize messages similarly to tables in SQL, and message schemas, which define message structure to prevent complexity. Proper governance, including topic management and schema validation, ensures smooth communication and scalability in large systems.
+
+Technical implementations should:
+
+* cover business needs
+* ensure performance
+* have an appropriate delivery mode (at most once, at least once, and exactly once)
+* have message persistence
+* be scalable
+* may offer message prioritisation
+* offer message ordering
+
+#### The graph model
+
+Graphs can offer:
+
+* centrality analysis
+* search and pathfinding
+* community detection
+* contagion analysis (how shock impacting one or a set of nodes propages throughout the rest of the network)
+* link prediction
+
+Graph data modelling:
+
+![image](https://github.com/user-attachments/assets/1df63ca5-c957-49a1-be08-8d4df924c64d)
+
+#### The warehouse model
+
+_A data warehouse is a subject-oriented, integrated, nonvolatile, and time-variant collection of data in support of management’s decisions._
+
+Bill Inmon
+
+_A data warehouse is a copy of transaction data specifically structured for query and analysis._
+
+Ralph Kimball
+
+![image](https://github.com/user-attachments/assets/ccabed05-a3be-4652-a1a5-5b9fed031cb9)
+
+**Why data warehouses?**
+
+* consistent structure for data
+* advanced analytics
+* scalability
+* subject-oriented
+* integrated
+* nonvolatile
+* time-variant
+
+**Data modelling with data warehouses**
+
+* the subject modeling approach of Bill Inmon, vs.
+* the dimensional modeling approach of Ralph Kimball
+
+In the Inmon approach, data is sourced from various operational systems across the organization and integrated into a centralized data warehouse. Subsequently, based on the specific needs of individual departments or business units, dedicated data marts are derived from the data warehouse. In this model, the data warehouse and the data marts are physically separate entities, each with its own storage and characteristics. Data within the data marts is considered consistent and reliable, as it originates from the centralized source of truth—the data warehouse itself. Users from different departments typically employ specialized tools to access and analyze the data within their respective data marts
+
+![image](https://github.com/user-attachments/assets/71b11371-4953-4f82-ab94-8060cc8d5adf)
+
+The Kimball approach takes a different perspective. Instead of isolating the data warehouse from the data marts, Kimball suggested a user requirement-driven approach that defines the data marts in advance and integrates them within the data warehouse itself
+
+![image](https://github.com/user-attachments/assets/e5a0308d-6656-4fa5-a4af-22e87d950629)
+
+**Dimension modelling**
+
+Two main types
+
+![image](https://github.com/user-attachments/assets/d51c1e6e-7bc0-4533-8661-2f728c187d02)
+
+Examples are Google BigQuery, Amazon Redshift, Snowflake
+
+#### The blockchain model
+
+Blockchain is a data structure used to store data as a chain of linked blocks, offering immutability and tamper resistance. Its primary application is in creating digital ledgers, such as those for cryptocurrencies like Bitcoin and Ethereum. Blockchain ensures data integrity through cryptographic hashes and distributed ledger technology (DLT), where decentralized nodes validate and maintain the ledger. However, blockchain has limitations, such as slow data querying and performance issues with increasing nodes. Blockchain databases, like BigchainDB and Amazon QLDP, address these by combining traditional database performance with blockchain's immutability and security.
+
+![image](https://github.com/user-attachments/assets/02e4000c-2efc-47d1-adff-db38be6db1e5)
+
+## Chapter 9 - Data transformation and delivery layer
+
+### Data querying
+
+* time series queries
+
+```sql
+SELECT time_column, attribute_1, attribute_2 
+FROM financial_entity_table 
+WHERE entity_name = 'A' 
+AND date BETWEEN '2022-01-01' AND '2022-02-01'
+```
+
+* cross-section queries
+
+```sql
+SELECT entity_name, attribute_1, attribute_2 
+FROM financial_entity_table 
+WHERE entity_name in ('A', 'B', 'C', 'D') 
+AND date = '2022-02-01'
+```
+
+* panel queries
+
+```sql
+SELECT time_coluumn, entity_name, attribute_1, attribute_2 
+FROM financial_entity_table 
+WHERE entity_name in ('A', 'B', 'C', 'D') 
+AND date BETWEEN '2022-01-01' AND '2022-02-01'
+```
+
+* analytical queries
+
+```sql
+SELECT stock_symbol, date, MAX(price)
+FROM price_table
+GROUP BY stock_symbol, date
+```
+
+Recently, cloud data warehouse solutions such BigQuery and Snowflake have introduced features enabling users to create and interact with machine learning models using advanced analytical SQL queries
+
+```sql
+-- Create a linear regression model
+CREATE MODEL `mydataset.my_model`
+OPTIONS(MODEL_TYPE='LINEAR_REG') AS
+SELECT
+  feature_column1,
+  feature_column2,
+  label
+FROM
+  `mydataset.my_training_table_data`;
+```
+
+Calling the model would look like:
+
+```sql
+SELECT *
+FROM
+  ML.PREDICT(MODEL `mydataset.my_model`, (
+    SELECT
+      feature_column1,
+      feature_column2
+    FROM
+      `mydataset.my_test_table_data`
+  ));
+```
+
+#### Query optimisation
+
+* database-side query optimisations (indexing, clustering, partitioning)
+* user-side query optimisations (use indexes; for composite indexes, filter by the leftmost columns; avoid `SELECT *`; use anchored patterns in LIKE; break large queries into smaller, incremental batches to handle failures efficiently; filter data before joins to improve performance)
+
+![image](https://github.com/user-attachments/assets/d56424f0-7809-43f4-9d4a-380b754d8258)
+
+### Data transformation
+
+**Transformation operations**
+
+* format conversion
+* data cleaning
+* data adjustments
+* data standardisation
+* data filtering
+* feature engineering
+* advanced analytical computations
+
+**Transformation patterns**
+
+* batch vs. streaming transformations
+
+![image](https://github.com/user-attachments/assets/c0ac6682-3b3a-445c-b14a-1bb6e6747447)
+
+* memory-based vs. disk-based transformations
+
+![image](https://github.com/user-attachments/assets/d27a5acc-2339-4004-bc5c-b417693d1e0d)
+
+* full vs. incremental (CDC) data transformations 
+
+### Data delivery
+
+Financial data engineers must determine who the ultimate data consumers are and understand their specific needs, and then create the appropriate mechanisms to deliver the data.
+
+Data consumers include any user, team, or system that uses data from a company’s infrastructure. They may range from compliance officers and marketing teams, who specify data needs and rely on data engineers, to analysts and machine learning specialists more involved in data processes. Roles within data governance include data owners (e.g., finance directors managing financial data), data stewards (ensuring data quality), and data custodians (protecting data). Data contracts formalize expectations between producers and consumers, detailing requirements like data type, format, and SLAs.
+
+Data delivery mechanisms vary, including direct database/file access via interfaces, programmatic access through APIs or libraries, reports, dashboards, and email. Effective delivery design requires helping users search and find data, with practices like specifying data locations in contracts and using data catalogs as central search engines.
+
+## Chapter 10 - The monitoring layer
+
+Data monitoring is a continuous task requiring close collaboration between financial data engineers and business teams. It enables financial institutions to operate securely, compliantly, and efficiently in a rapidly changing environment. Monitoring is needed for:
+
+* operational continuity and efficiency
+* compliance
+* risk management
+
+When designing this layer, the 1st question to ask is what components of the financial data infrastructure do we need to monitor
+
+### Metrics, Events, Logs, and Traces
+
+* metrics: quantitative measurements that provide information about a particular aspect of a system, process, variable, or activity; usually metrics are observed over a specific time frame; and often enriched with metadata
+* events: structured data emitted by an application during runtime
+* logs: semi-structured data emitted by an application during runtime, providing greater flexibility in terms of content, structure, and triggering mechanisms compared to events (log types: debug, info, warning, error, critical)
+* traces: represent a more advanced form of system behavior records, capturing comprehensive details about each step taken by a process as it progresses through one or more system (types: business operation traces, application traces, security incident traces)
+
+### Data quality monitoring
+
+Data quality monitoring in financial institutions focuses on ensuring data consistently meets high standards, covering aspects like errors, duplicates, biases, completeness, and timeliness. The process begins with defining Data Quality Metrics (DQMs), tailored to client expectations, business needs, and technical standards. Common metrics include error, duplicate, and missing value ratios. Data profiling, which analyzes a dataset's structure and relationships, is a key monitoring method. Tools like Ataccama ONE, Great Expectations, and Talend Data Fabric help automate issue detection. Monitoring is a continuous process, adapting to evolving data types and business requirements.
+
+Example of a data profile report:
+
+![image](https://github.com/user-attachments/assets/0ab7370a-693b-46e0-8be9-4c16527e0160)
+
+### Performance monitoring
+
+Here are the key metrics for performance monitoring in financial data infrastructure:
+
+- **General Metrics:**
+  - RAM usage
+  - CPU usage
+  - Storage usage
+  - Execution time
+  - Requests per second (RPS)
+  - Ingress/egress bytes (bytes/sec)
+  - Uptime/downtime
+  - API response time
+  - Concurrent users
+
+- **Database-Specific Metrics:**
+  - Read/write operations
+  - Active connections
+  - Connection pool usage
+  - Query count
+  - Transaction runtime
+  - Replication lag
+  - Records read
+  - Blocks read
+  - Bytes scanned
+  - Number of output rows
+  - Scan time, sort time, aggregation time
+  - Peak memory usage
+  - Query plans
+
+- **Financial Market-Specific Metrics:**
+  - Trade execution latency
+  - Trade execution error rate
+  - Trade settlement time
+  - Market data latency
+  - Algorithmic trading performance
+  - Risk exposure calculation time
+  - Customer transaction processing time
+
+- **Incident Response Metrics:**
+  - Mean Time to Detect (MTTD)
+  - Mean Time Before Failure (MTBF)
+  - Mean Time to Recovery (MTTR)
+  - Mean Time to Failure (MTTF)
+  - Mean Time to Acknowledge (MTTA)
+
+### Cost monitoring
+
+Cost monitoring involves tracking financial data infrastructure expenses to ensure they remain within acceptable limits and to detect patterns of excessive usage. With cloud services' widespread use, managing costs is critical due to potential misinterpretations of pricing, particularly in serverless computing (like AWS Lambda). Misunderstandings about execution patterns, memory allocation, and data transfer fees can lead to unforeseen costs.
+
+Key practices include setting budgets with alerts, using tools like FinOps for financial accountability, and applying a FinOps Maturity Model (FMM) to improve cost management through collaboration between engineering, finance, and business teams.
+
+### Business and Analytical monitoring
+
+- advanced monitoring: observes not only raw data but also contextual and analytical insights for strategic decision-making
+- risk monitoring in financial institutions:
+  - market risk: losses from market price movements
+  - credit risk: losses from borrower defaults
+  - operational risk: losses from internal process failures or external events
+  - data engineering challenges arise in collecting, aggregating, and normalizing risk data for regulatory compliance (i.e. Basel III)
+- ML monitoring:
+  - concept drift: model no longer accurately reflects the original problem
+  - data drift: changes in input data distribution over time
+  - model risk: financial models may produce inconsistent results due to changing market conditions
+- fraud and illegal activity monitoring:
+  - money laundering, rerrorism financing, fraud, corruption: monitoring for illegal transactions
+  - market manipulation: activities such as pump and dump, spoofing, insider trading, and wash trading
+- investment monitoring:
+  - monitoring key performance indicators like Sharpe, Sortino, and Treynor ratios to evaluate investment portfolio performance
+ 
+### Data observability
+
+Observability elevates monitoring to a new level, allowing software and data engineering teams to handle issues proactively and gain deep insights into the internal state and behavior of a given software application or infrastructure
+
+A system is observable if through it we can:
+
+* understand the inner workings of our application
+* understand any system state the application may have gotten itself into, even new ones we have never seen before and couldn't have predicted
+* understand the inner workings and system state solely by observing and interrogating with external tools
+* understand the internal state without shipping any new custom code to handle it (because that implies you needed prior knowledge to explain it)
+
+A data observability system can bring:
+
+* higher data quality
+* operational efficiency (reduced TTD and TTR)
+* facilitated communication and trust between different team members
+* gains in client trust by being able to detect and understand the issues they might face
+* enabling complex data ingestion and transformation while maintaining reliability and visibility into the system
+* regulatory compliance
+
+## [Stream](https://www.youtube.com/watch?v=SGvLjNJ5y5Y&t=712s)
+
+Read two interesting articles(?) or examples (?) from the sklearn docs:
+
+* [Permutation Importance vs Random Forest Feature Importance (MDI)](https://scikit-learn.org/stable/auto_examples/inspection/plot_permutation_importance.html)
+* [Permutation Importance with Multicollinear or Correlated Features](https://scikit-learn.org/stable/auto_examples/inspection/plot_permutation_importance_multicollinear.html)
+
+Super interesting!
+
+And also found more for tomorrow. Again from the 'inspection' part of the sklearn docs
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 958a01b775003575dd2c855c784f35a6fe9a691e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 17 Oct 2024 22:42:38 +0900
Subject: [PATCH 466/638] Create 2024-10-17-day-290.md

---
 _posts/2024-10-17-day-290.md | 485 +++++++++++++++++++++++++++++++++++
 1 file changed, 485 insertions(+)
 create mode 100644 _posts/2024-10-17-day-290.md

diff --git a/_posts/2024-10-17-day-290.md b/_posts/2024-10-17-day-290.md
new file mode 100644
index 000000000..fbeca3ae4
--- /dev/null
+++ b/_posts/2024-10-17-day-290.md
@@ -0,0 +1,485 @@
+---
+layout: post
+title: (Day 290) Finishing the book - Financial Data Engineering
+categories: [traditional-machine-learning,theory,applying-knowledge,data-eng]
+---
+
+# Hello :) Today is Day 290!
+A quick summary of today:
+* finished the financial data engineering book
+* learned about a new feature selection method
+
+## Financial DE Chapter 11 - Financial data workflows
+
+### Workflow-oriented software architectures (WOSA)
+
+WOSA help manage the growing complexity of digital systems by organizing software transactions into structured workflows. Common in financial markets, WOSA structures processes like financial trades and credit card payments into defined steps. This enhances modularity, efficiency, and manageability in software systems, particularly for data workflows.
+
+### What is a data workflow?
+
+A data workflow is a repeatable process involving a structured sequence of steps that transform an initial dataset into a desired output. In data engineering, workflow abstraction helps generalize workflows by focusing on their conceptual framework, not technological specifics. The dataflow paradigm, often represented as a Directed Acyclic Graph (DAG), organizes computations into a directed graph, where each task depends on others. Tasks should be atomic, idempotent, and appropriately sized to ensure efficiency, traceability, and debuggability, while aligning with business logic
+
+![image](https://github.com/user-attachments/assets/c96f618c-0926-42a7-bd1f-19772d54e30a)
+
+### Workflow management systems (WMS)
+
+Once a workflow abstraction has been defined, the next step is to design a system that provides the infrastructure and tools necessary for building, orchestrating, and monitoring a WOSA
+
+When developing a WMS some properties need to be considered:
+
+* flexibility: a flexible WMS is capable of managing a diverse range of tasks and workflows
+* configurability: enables users to define workflow specifications according to their needs
+* dependency management: simple WMSs allow users only to be able to define static and linear workflows; more advanced WMSs may allow for parallel workflow/task execution, dynamic and conditional task creation, and information passing between tasks
+
+![image](https://github.com/user-attachments/assets/17eefb0c-a90a-419a-9af0-f0674f696ceb)
+
+* coordination patterns: two notable patterns are the Synchronous Blocking Pattern (SBP) and the Asynchronous Non-Blocking Pattern (ANBP)
+
+![image](https://github.com/user-attachments/assets/b6420d1f-f801-414e-a97a-18c4207614ed)
+
+* scalability: assessed based on the WMS' capacity to handle concurrent executions of workflows and tasks
+* integration with other tools
+
+### Types of financial data workflows
+
+#### ETL workflows
+
+![image](https://github.com/user-attachments/assets/02e6fd81-0f58-40ae-bf15-5501a9665ce8)
+
+#### Stream processing workflows
+
+![image](https://github.com/user-attachments/assets/14b4b034-740a-4f89-aac0-6299fff31e07)
+
+Lambda vs Kappa
+
+Lambda:
+
+![image](https://github.com/user-attachments/assets/a3f31fd9-3095-46f1-8f0f-51f64fd1c6c3)
+
+Kappa:
+
+![image](https://github.com/user-attachments/assets/80c01c12-0b42-4489-a80e-a20d9c352a81)
+
+#### Microservice workflows
+
+Microservice-oriented online store order processing system:
+
+![image](https://github.com/user-attachments/assets/778de463-f581-430f-b2fa-1e8d04096380)
+
+Microservice workflows coordinate multiple microservices, which are independently deployable, to follow specific business logic. Designing these workflows requires balancing technical factors like cohesion and low coupling with business logic formalization, often using Domain-Driven Design (DDD). Microservice workflows can be managed via the saga pattern, using either choreography (direct communication between microservices) or orchestration (centralized control via an orchestrator). This approach is widely adopted in FinTech and traditional financial institutions for greater agility, allowing rapid development and deployment of new services.
+
+There are two types of saga patterns:
+
+![image](https://github.com/user-attachments/assets/576a8c88-e462-4b8b-ba45-31c322857638)
+
+#### ML workflows
+
+![image](https://github.com/user-attachments/assets/5c2e9f62-e52b-43fc-bf4f-e62dc17995a5)
+
+The challenges involved in building reliable and high-performance ML models largely revolve around effective data management and ensuring data quality.
+
+## Chapter 12 - Hands-on projects
+
+All of the code from the book is [here](https://github.com/TamerKhraisha/FinancialDataEngineering.git)
+
+### Project 1: Designing a Bank Account Management System Database with PostgreSQL
+
+Account management is a core functionality for banks, allowing them to handle and oversee customer accounts, balances, and transactions. And this 1st project is for designing and implementing a relational db system for managing back accounts
+
+#### Conceptual model: understanding business requirements
+
+Operational and business models vary from one banking institution to another, and not all banks offer the same products and services. As such, a simple and generic bank account management system will be developed.
+
+We’ll assume that stakeholders have defined their requirements and that these have been formalized them into entities, relationships, and constraints.
+
+**Entities**
+
+Our bank account management system needs to store data for seven types of entities: accounts, customers, loans, transactions, branches, employees, and cards. Each of these needs to store detailed information for each case
+
+**Relationships**
+
+The business team has requested that the following relationships be established between entities:
+
+* each account should be linked to a customer
+* each loand should be linked to a customer
+* each loan payment should be linked to a transaction and an account
+* each employee should be affiliated with a branch
+* each card should be associated with both a customer and an account
+
+**Constraints**
+
+* account balances must never go below a customer-specific minimum amount
+* all entity records (i.e. accounts, loans, transactions) must be identified with a unique ID
+* data redundancy should be minimised
+* null values are not permitted for certain fields
+* specific fields, such as email addresses, must be unique across records
+
+#### Logical model: entity relationship diagram
+
+This stage focuses on selecting the data storage model most suitable for our system 
+
+After a thorough evaluation, the financial data engineering team has concluded that the relational model is the best fit, because this model effectively organises various entities into distinct tables and ensures data integrity through db constraints. Also the relational model supports the implementation of a normalised data structure which is essential for our system. 
+
+To implement this, we need to create the Entity Relationship Diagram (ERD) for our logical model. An ERD is a visual representation used to model the structure of a database.
+
+![image](https://github.com/user-attachments/assets/f8c491e5-5209-423b-83c0-749d478ffec1)
+
+#### Physical model: data definition and manipulation language
+
+It’s time to choose a relational database technology and write the queries to create and populate our tables. Let’s assume the financial data engineering team has selected PostgreSQL as the database management system due to its high reliability and strong adherence to SQL standards.
+
+#### Next is the coding part, I will share some of the code as I will follow along the book, but everything is in the github repo above. 
+
+Structure:
+
+![image](https://github.com/user-attachments/assets/79d9236e-9e1a-4dc9-aaa7-7c00d6c985c3)
+
+In the `docker-compose.yaml` there are two services: postgres and pgadmin
+
+The rest is just creating the tables from the created schema and inserting data into those tables.
+
+### Project 2: Designing a Financial Data ETL Workflow with Mage and Python
+
+(Mage getting another shout out in a new book; nice)
+
+#### Workflow definition
+
+**Data retrieval**
+
+1. fetch adjusted intraday time series history for the past month using an API endpoint
+2. store the raw data in the db, recording the ingestion timestamp
+3. data aggregation
+4. deduplication
+5. data export
+
+**Database design**
+
+Two tables will be needed: one for the raw intraday data retrieved from the API, and another for the transformed daily data product at the end of the workflow
+
+![image](https://github.com/user-attachments/assets/d0a839c3-7063-4df1-ac3a-0cacd56e440e)
+
+**Local practice**
+
+The `docker-compose.yaml` for this project contains: mage, postgres, and pgadmin
+
+After populating some data in the db, I created the pipeline in Mage:
+
+![image](https://github.com/user-attachments/assets/88d8b22c-0dbd-4baa-a87d-ef7abcb03c93)
+
+![image](https://github.com/user-attachments/assets/0ce59d91-2b76-43fc-9c05-2f9afadbe036)
+
+### Project 3: Designing a Microservice Workflow with Netflix Conductor, PostgreSQL, and Python
+
+**Workflow definition**
+
+The first step in designing a microservice workflow is to define its structure. Here are the microservices that will constitute the OMS system:
+
+* order acknowledgement service: handles the acknowledgment of customer orders. It receives a client order, persists it in the database, and returns an acknowledgment message to the customer
+* payment processing service: processes customer payment transactions and returns a message informing the customer about the status of their payment operation
+* stock and inventory service: manages the inventory and stock levels of products available for sale. It tracks and checks the quantity of each product in stock, books and updates inventory based on incoming orders, and returns a message to the customer about the stock booking
+* shipping service: manages the shipment of orders. This service coordinates with shipping carriers to book a delivery, generate a tracking number, and update the delivery status of orders. It returns a message informing the client about their upcoming delivery
+* notification service: sends notifications to customers at various stages of the order fulfillment process
+
+![image](https://github.com/user-attachments/assets/c13a71ac-1e55-4503-bb70-79ca179c6c51)
+
+**Database design**
+
+![image](https://github.com/user-attachments/assets/c1e7ced9-a6f0-40ec-a600-3195d2b962ef)
+
+**Local testing**
+
+The docker compose here consists of: conductor, pgadmin, postgres, and jupyterlab
+
+This project is a bit more detailed so I will share some of the code. And also because this is the 1st time I am trying out Netflix's conductor 
+
+First, create tables: `python3 database/create_tables.py` 
+
+Second, populate them: `python3 database/populate_tables.py`
+
+![image](https://github.com/user-attachments/assets/7755abe8-59b5-47c2-812c-34182bf35853)
+
+Third, create and register the microservice workflow. It is prepared for me and each service is in its own folder with a service and a worker.py
+
+![image](https://github.com/user-attachments/assets/556db4c1-32c3-4ee0-92a2-7223bac3717c)
+
+This is the `delivery_scheduling_service`
+
+```python
+# service.py
+
+import psycopg2
+import os
+import string
+import random
+from datetime import datetime, timedelta
+
+def generate_tracking_number():
+    return ''.join(random.choices(string.digits, k = 10))
+
+def generate_delivery_date():
+    # Get the current date
+    current_date = datetime.now()
+    # Calculate the date after 3 days (let's assume delivery takes places after 3 days)
+    date_after_10_days = current_date + timedelta(days=3)
+    # Format the date as YYYY-MM-DD
+    formatted_date = date_after_10_days.strftime('%Y-%m-%d')
+    return formatted_date
+
+
+def schedule_delivery(order_id: int):
+    # Connection parameters
+    connection_params = {
+        'host': os.environ.get('POSTGRES_HOST'),
+        'port': os.environ.get('POSTGRES_PORT'),
+        'database': os.environ.get('POSTGRES_DB'),
+        'user': os.environ.get('POSTGRES_USER'),
+        'password': os.environ.get('POSTGRES_PASSWORD')
+    }
+    try:
+        # Establish a connection to the PostgreSQL database
+        with psycopg2.connect(**connection_params) as connection:
+            # Create a cursor
+            with connection.cursor() as cursor:
+                # Schedule delivery
+                tracking_number = generate_tracking_number()
+                delivery_date = generate_delivery_date()
+
+                cursor.execute(
+                    "UPDATE DeliverySchedule SET delivery_service = 'DHL Express', expected_delivery_date=%s, tracking_number=%s WHERE order_id = %s;",
+                    (delivery_date, tracking_number, order_id)
+                )
+
+                # Commit the transaction
+                connection.commit()
+
+        print("Delivery added to DeliverySchedule table successfully!")
+        return {'success': True, 'order_id': order_id, 'notification': 'Your order has been shipped and is on its way to the delivery address.'}
+    except psycopg2.Error as e:
+        print("Error: Unable to add delivery to DeliverySchedule table:", e)
+        return {'success': False, 'order_id': order_id, 'notification': 'Your order could not be processed due to an unexpected error. Please contact customer support for assistance.'}
+```
+
+And `worker.py`
+
+```python
+from conductor.client.worker.worker_task import worker_task
+from delivery_scheduling_service.service import schedule_delivery as sd
+import uuid
+
+@worker_task(task_definition_name='schedule_delivery')
+def schedule_delivery(order_id: int) -> dict:
+    result = sd(order_id=order_id)
+    return result
+```
+
+Next, I need to register the order workflow `python3 workflow/register_order_workflow.py`
+
+Here is the `register_order_workflow.py`:
+
+```python
+from conductor.client.automator.task_handler import TaskHandler
+from conductor.client.configuration.configuration import Configuration
+from conductor.client.workflow.conductor_workflow import ConductorWorkflow
+from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor
+from order_workflow import order_workflow
+
+
+def register_workflow(workflow_executor: WorkflowExecutor) -> ConductorWorkflow:
+    workflow = order_workflow(workflow_executor=workflow_executor)
+    workflow.register(True)
+    return workflow
+
+
+def main():
+    # The app is connected to http://localhost:8080/api by default
+    try:
+        api_config = Configuration()
+    
+        workflow_executor = WorkflowExecutor(configuration=api_config)
+    
+        # Registering the workflow (Required only when the app is executed the first time)
+        workflow = register_workflow(workflow_executor)
+    
+        print(f'Successfully registered workflow {workflow.name} - version {workflow.version}')
+    except Exception as e:
+        print(e)
+
+if __name__ == '__main__':
+    main()
+```
+
+The executed workflow is this one which contains all the workers which call their respective service:
+
+```python
+from conductor.client.workflow.conductor_workflow import ConductorWorkflow
+from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor
+from order_acknowledgement_service.worker import acknowledge_order
+from notification_service.worker import notify_customer
+from payment_processing_service.worker import process_payment
+from stock_and_inventory_service.worker import update_inventory
+from delivery_scheduling_service.worker import schedule_delivery
+
+def order_workflow(workflow_executor: WorkflowExecutor) -> ConductorWorkflow:
+    workflow = ConductorWorkflow(
+        name='Order Workflow',
+        executor=workflow_executor,
+        version=2,
+        description='Executes the services required to process a customer order.'
+    )
+
+    # Acknowledge Order task
+    acknowledge_order_task = acknowledge_order(
+        task_ref_name='acknowledge_order_ref',
+        customer_id=workflow.input('customer_id'),
+        products=workflow.input('products'),
+        payment_amount=workflow.input('payment_amount'),
+        delivery_address=workflow.input('delivery_address')
+    )
+
+    # Notify the customer of the acknowledgment task
+    notify_acknowledgement_task = notify_customer(
+        task_ref_name='notify_acknowledgement_ref',
+        order_id=acknowledge_order_task.output('order_id'),
+        notification_text=acknowledge_order_task.output('notification')
+    )
+
+    # Process payment task
+    process_payment_task = process_payment(
+        task_ref_name='process_payment_ref',
+        order_id=acknowledge_order_task.output('order_id')
+    )
+
+    # Notify customer of payment task
+    notify_payment_task = notify_customer(
+        task_ref_name='notify_payment_ref',
+        order_id=acknowledge_order_task.output('order_id'),
+        notification_text=process_payment_task.output('notification')
+    )
+
+    # Update inventory
+    update_inventory_task = update_inventory(
+        task_ref_name='update_inventory_ref',
+        order_id=acknowledge_order_task.output('order_id')
+    )
+
+    # Notify customer of inventory booking
+    notify_inventory_task = notify_customer(
+        task_ref_name='notify_inventory_ref',
+        order_id=acknowledge_order_task.output('order_id'),
+        notification_text=update_inventory_task.output('notification')
+    )
+
+    # Schedule delivery
+    schedule_delivery_task = schedule_delivery(
+        task_ref_name='schedule_delivery_ref',
+        order_id=acknowledge_order_task.output('order_id'),
+    )
+
+    # Notify customer of delivery schedule
+    notify_delivery_task = notify_customer(
+        task_ref_name='notify_delivery_ref',
+        order_id=acknowledge_order_task.output('order_id'),
+        notification_text=schedule_delivery_task.output('notification')
+    )
+
+    # define the flow structure
+    workflow >> acknowledge_order_task >> \
+        notify_acknowledgement_task >> \
+        process_payment_task >> \
+        notify_payment_task >> \
+        update_inventory_task >> \
+        notify_inventory_task >> \
+        schedule_delivery_task >> \
+        notify_delivery_task
+
+    return workflow
+```
+
+Once I executed the bash command to register the workflow, I can see it in the UI:
+
+![image](https://github.com/user-attachments/assets/2eb623ff-dc7b-49bc-bccf-2f1d33706c3c)
+
+Clicking on it takes me to:
+
+![image](https://github.com/user-attachments/assets/1cebc1e5-8f41-41ae-a5ae-b4bf62702833)
+
+To execute the workflow with python (`python3 workflow/execute_order_workflow.py`), and the file looks like:
+
+```python
+from conductor.client.automator.task_handler import TaskHandler
+from conductor.client.configuration.configuration import Configuration
+from conductor.client.workflow.conductor_workflow import ConductorWorkflow
+from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor
+from order_workflow import order_workflow
+
+
+def main():
+    # The app is connected to http://localhost:8080/api by default
+    api_config = Configuration()
+
+    workflow_executor = WorkflowExecutor(configuration=api_config)
+
+    # Starting the worker polling mechanism
+    task_handler = TaskHandler(configuration=api_config)
+    task_handler.start_processes()
+
+    workflow_run = workflow_executor.execute(name='Order Workflow',
+                                             version=2,
+                                             workflow_input={
+                                                 'customer_id': 1,
+                                                 'products': {1: 1, 2: 3},
+                                                 'payment_amount': 2000,
+                                                 'delivery_address': '456 Broadway, New York, NY 10013, United States'},
+                                             )
+
+    task_handler.stop_processes()
+
+
+if __name__ == '__main__':
+    main()
+```
+
+The terminal shows these logs:
+
+![image](https://github.com/user-attachments/assets/22b3d0f2-a02a-4fd0-8f5f-cfbfd4dc605f)
+
+In conductor, I can see the run in the Executions tab:
+
+![image](https://github.com/user-attachments/assets/84947f64-47c7-452b-81e0-6be5250e87f2)
+
+And view all kinds of information for the run:
+
+![image](https://github.com/user-attachments/assets/f023f84b-ed3e-47f4-b7e5-2d7a7c42d3fc)
+
+![image](https://github.com/user-attachments/assets/41cb9964-fd96-4cde-894e-0833bbf65fa1)
+
+### Project 4: Designing a Financial Reference Data Store with OpenFIGI, PermID, and GLEIF APIs
+
+This project was just downloading financial identifiers from the above sources and saving them to the database.
+
+For instance - download and export BIC-to-LEI mapping data:
+
+```python
+api_url = "https://mapping.gleif.org/api/v2/bic-lei/165fb003-5254-498f-a3cd-6e7dfccb8075/download"
+bic_to_lei_dataframe = download_and_extract_zip(api_url)
+bic_to_lei_dataframe.head()
+```
+
+![image](https://github.com/user-attachments/assets/b41ceecd-c526-412a-ba21-989c7c90fe4e)
+
+And then insert this into postgres.
+
+### Overall
+
+Great book! Definitely in my 'hall of fame'
+
+## [Stream](https://youtu.be/rEYbEo_0ka0)
+
+* read the overall basics of [skrub](https://skrub-data.org/stable/index.html)
+* experimented with [shap_select](https://github.com/transferwise/shap-select)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From d4904c353baf2fb6d6eb2b92dc23df897f1bfe9a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 18 Oct 2024 22:42:37 +0900
Subject: [PATCH 467/638] Create 2024-10-18-day-291.md

---
 _posts/2024-10-18-day-291.md | 381 +++++++++++++++++++++++++++++++++++
 1 file changed, 381 insertions(+)
 create mode 100644 _posts/2024-10-18-day-291.md

diff --git a/_posts/2024-10-18-day-291.md b/_posts/2024-10-18-day-291.md
new file mode 100644
index 000000000..e68a256c9
--- /dev/null
+++ b/_posts/2024-10-18-day-291.md
@@ -0,0 +1,381 @@
+---
+layout: post
+title: (Day 291) Starting Machine Learning for Financial Risk Management with Python + some math for ML
+categories: [mathematics,traditional-machine-learning,theory]
+---
+
+# Hello :) Today is Day 291!
+A quick summary of today:
+* finally sat down and did the basic math exercises from Before ML Vol. 2 - Calculus
+* started Machine Learning for Financial Risk Management with Python on O'Reilly
+
+## Basic calculus exercises
+
+Today I finially got some time to sit down with my tablet and write down the exercises of the [Before ML Vol. 2 - Calculus](https://www.amazon.com/Before-Machine-Learning-fundamental-Intelligence/dp/B0CPW6WVLS). It is basic math, and as the title suggests is math needed *before* doing ML to better understand the common underlying mechanisms. I just wanted to check the exercise book because before each exercise there is also a short description/overview of the specific concept.
+
+Here are my notes:
+
+![Exercises P1](https://github.com/user-attachments/assets/f1651c4d-df2a-4f16-b154-741ad36f990b)
+![Exercises P2](https://github.com/user-attachments/assets/22ed25ab-1c16-475a-8b1f-b413370c55cf)
+![Exercises P3](https://github.com/user-attachments/assets/04474c76-7366-4476-b7e4-4ebecfe2e3f8)
+![Exercises P4](https://github.com/user-attachments/assets/9f12c89d-66ab-4092-821c-d35e3a214846)
+![Exercises P5](https://github.com/user-attachments/assets/fed065bf-5826-40d5-b1af-a7921c806f8f)
+![Exercises P6](https://github.com/user-attachments/assets/2a3cb379-869c-4ace-a560-f70d754ec225)
+![Exercises P7](https://github.com/user-attachments/assets/6c272f53-90a6-4597-9d2c-806e004e037a)
+![Exercises P8](https://github.com/user-attachments/assets/18a40f37-b346-472c-892b-fd721ad4b78f)
+![Exercises P9](https://github.com/user-attachments/assets/eea3889b-3196-4960-b6d1-f2c0dd01b4b2)
+![Exercises P10](https://github.com/user-attachments/assets/9d2d42b6-925d-4747-9db3-9b5750a04a79)
+
+## [ML for Financial Risk Management with Python](https://learning.oreilly.com/library/view/machine-learning-for/9781492085249/)
+
+Started this book today. I initially attemped to read it back in January (at the start of my ML journey) but I felt that this was not the book I needed back then, so here I am - it was high time I read it now. And also as an ex-risk analyst this is very exciting to me
+
+## Chapter 1 - Fundamentals of Risk Management
+
+### Risk
+
+Risk is challenging to assess due to its abstract nature and is often perceived as hazardous, whether expected or unexpected. Expected risks are typically priced, while unexpected risks can be more devastating. Financially, risk is often viewed as the potential for loss or uncertainty. Risk can be defined as any event that may hinder an organization's objectives or the likelihood of loss. Though risk is associated with costs, the relationship is not always direct—expected risks tend to have lower costs compared to unexpected ones.
+
+### Return
+
+Return refers to the profit gained from an investment over a specific period, representing the upside of risk. There is a trade-off between risk and return: higher assumed risk typically leads to greater realized return, making this relationship a contentious issue in finance. Markowitz (1952) provided clarity by defining risk using standard deviation, which allows for mathematical and statistical analysis in finance. The expected return is the probability-based return of interest, while portfolio variance involves weights, variance, and covariance. The portfolio expected return is calculated as a weighted average of individual returns.
+
+Risk and Return go hand-in-hand, but the magnitute of that relationship depends on the individual stocks and the financial market conditions
+
+![image](https://github.com/user-attachments/assets/542fc42a-eba0-48cc-b128-87eac545f4f0)
+
+### Risk Management
+
+Financial risk management is a process to deal with the uncertainties resulting from financial markets. It involves assessing the financial risks facing an organization and developing management strategies consistent with internal priorities and policies 
+
+Risk management is, therefore, not about mitigating risk at all costs. Mitigating risk may require sacrificing return, and it can be tolerable up to certain level as companies search for higher return as much as lower risk. Thus, to maximize profit while lowering the risk should be a delicate and well-defined task.
+
+There is a general framework for risk strategies:
+
+* ignore: companies accept all risks and their consequences and prefer to do nothing
+* transfer: involves transferring the risks to a third party by hedging or some other way
+* mitigate: companies develop a strategy to mitigate risk partly because its harmful effect might be considered too much to bear and/or surpass the benefit attached to it
+* accept: if companies embrace the strategy of accepting the risk, they properly identify risks and acknowledge the benefit of them. In other words, when assuming certain risks arising from some activities bring value to shareholders, this strategy can be chosen
+
+#### Main financial risks
+
+The book defines the below as the main types of risk (but there are others too)
+
+**Market risk**
+
+This arises from changes in financial market factors, such as interest rate fluctuations or exchange rate changes, which can adversely affect companies, especially those involved in international trade or holding short positions. Changes in commodity prices, influenced by various fundamentals, can also threaten financial sustainability.
+
+**Credit Risk** 
+
+Occurs when a counterparty fails to fulfill debt obligations, leading to potential losses for lenders. Deterioration in credit quality can also decrease the market value of securities held by an organization.
+
+**Liquidity Risk** 
+
+This type of risk gained attention after the 2007–2008 financial crisis. It encompasses two dimensions: trading liquidity risk, which is the ease of executing transactions, and funding liquidity risk, which refers to a company's ability to raise cash. Inability to convert assets to cash quickly can harm a company's financial health.
+
+**Operational Risk** 
+
+Involves challenges in managing risks associated with day-to-day operations. This includes risks from external events, internal failures, fraud, regulatory non-compliance, and insufficient training. Companies unprepared for these risks may face significant financial collapse, as historical events have shown.
+
+### Information Asymmetry in Financial Risk Management
+
+Information asymmetry and financial risk management go hand in hand as the cost of financing and firm valuation are deeply affected by information asymmetry. That is, uncertainty in valuation of a firm’s assets might raise the borrowing cost, posing a threat to a firm’s sustainability
+
+#### Adverse selection
+
+Adverse selection is a type of asymmetric information in which one party tries to exploit its informational advantage. This arises when sellers are better informed than buyers. 
+
+Adverse selection is common in the insurance industry. Suppose the consumer utility function (a tool used to represent consumer preferences for goods and services, and it is concave for risk-averse individuals) is:
+
+<img width="127" alt="image" src="https://github.com/user-attachments/assets/16ffaf7e-a741-4188-a37f-c73341da5f94">
+
+where x is income and gamma is a parameter, which takes on values between 0 and 1
+
+The ultimate aim of this example is to decide whether or not to buy an insurance based on consumer utility.
+
+For the sake of practice, assume that the income is US$2 and the cost of the accident is US$1.5.
+
+Now it is time to calculate the probability of loss, pi, which is exogenously given and uniformly distributed.
+
+As a last step, to find equilibrium, we have to define supply and demand for insurance coverage
+
+```python
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Use seaborn style for plots
+plt.style.use('seaborn')
+
+# Define a utility function
+gamma = 0.4  # Define gamma for utility function
+
+def utility(x):
+    return np.exp(x ** gamma)
+
+# Generate random probabilities and sort them
+pi = np.random.uniform(0, 1, 20)
+pi = np.sort(pi)
+
+# Print the three highest probabilities of losses
+print('The highest three probability of losses are {}'.format(pi[-3:]))
+
+# Define constants
+y = 2
+c = 1.5
+Q = 5
+D = 0.01
+
+# Supply function
+def supply(Q):
+    return np.mean(pi[-Q:]) * c
+
+# Demand function
+def demand(D):
+    return np.sum(utility(y - D) > pi * utility(y - c) + (1 - pi) * utility(y))
+
+# Plotting the demand and supply curves
+plt.figure()
+
+# Plot insurance demand
+plt.plot([demand(i) for i in np.arange(0, 1.9, 0.02)], np.arange(0, 1.9, 0.02), 
+         'r', label='insurance demand')
+
+# Plot insurance supply
+plt.plot(range(1, 21), [supply(j) for j in range(1, 21)], 
+         'g', label='insurance supply')
+
+# Labels and legend
+plt.ylabel("Average Cost")
+plt.xlabel("Number of People")
+plt.legend()
+plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/6d14eecf-4550-4ceb-aa08-0304d21f0661)
+
+both curves are downward sloping, implying that as more people demand contracts and more people are added on the contracts, the risk lowers, affecting the price of the contract
+
+The straight line presents the insurance supply and average cost of the contracts and the other line, showing a step-wise downward slope, denotes the demand for insurance contracts. As we start analysis with the risky customers, as you add more and more people to the contract, the level of riskiness diminishes in parallel with the average cost.
+
+#### Moral Hazard
+
+Market failures also result from asymmetric information. In a moral hazard situation, one party of the contract assumes more risk than the other party. Formally, moral hazard may be defined as a situation in which the more informed party takes advantages of the private information at their disposal to the detriment of others.
+
+## Chapter 2 - Introduction to Time Series Modeling
+
+Time series models are important for analyzing financial data, which often includes a time dimension. Common models include Moving Average (MA), Autoregressive (AR), and ARIMA. MA uses past error terms, AR relies on past values of the time series, and ARIMA combines both.
+
+The superiority of the time series approach comes from the idea that correlations of observations in time better explain the current value. Having data with a correlated structure in time implies a violation of the famous identically and independently distributed (IID) assumption, which is at the heart of many models.
+
+So, due to the correlation in time, the dynamics of a contemporaneous stock price can be better understood by its own historical values. How can we comprehend the dynamics of the data? This is a question that we can address by elaborating the components of time series.
+
+### Time Series Components
+
+Time series has four components: trend, seasonality, cyclicality, and residual.
+
+We can use statsmodels' seasonal_decompose:
+
+```python
+import yfinance as yf
+import numpy as np
+import pandas as pd
+import datetime
+import statsmodels.api as sm
+from statsmodels.tsa.stattools import adfuller
+from statsmodels.tsa.seasonal import seasonal_decompose
+import matplotlib.pyplot as plt
+
+ticker = '^GSPC'
+start = datetime.datetime(2015, 1, 1)
+end = datetime.datetime(2021, 1, 1)
+
+SP_prices = yf.download(ticker, start=start, end=end, interval='1mo').Close
+seasonal_decompose(SP_prices, period=12).plot()
+plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/e6261a30-b3c6-44cc-95c6-23ba4a3f41a8)
+
+In the 1st row, we see the plot of raw data, and in the 2nd row, trend can be observed showing upward movement. In the 3rd row, seasonality is exhibited, and finally residual is presented showing erratic fluctuations (this includes noise and the cyclical component)
+
+#### Trend
+
+Trend indicates a general tendency of an increase or decrease during a given time period
+
+In the above graph we can clearly see an upward trend. A line plot is not the only option for understanding trend. Rather, there are some strong tools for this. Like Autocorrelation and Partial Autocorrelation
+
+The autocorrelation function (ACF) is a statistical tool to analyze the relationship between the current value of a time series and its lagged values.
+
+This is the ACF plot of the S&P500
+
+![image](https://github.com/user-attachments/assets/5146312c-8e1b-494e-bc7d-768fe7d7e90e)
+
+The vertical lines represent the correlation coefficients; the first line denotes the correlation of the series with its 0 lag—that is, it is the correlation with itself. The second line indicates the correlation between series value at time t - 1 and t. In light of these, we can conclude that the S&P 500 shows a serial dependence. There appears to be a strong dependence between the current value and lagged values of S&P 500 data because the correlation coefficients, represented by lines in the ACF plot, decay in a slow fashion
+
+What are the likely causes of autocorrelation? Here are some potential causes:
+
+* the primary source of autocorrelation is “carryover,” meaning that the preceding observation has an impact on the current one
+* model misspecification
+* measurement error, which is basically the difference between observed and actual values
+* dropping a variable, which has an explanatory power
+
+Partial autocorrelation function (PACF) is another method of examining the relationship between current and previous values
+
+ACF is commonly considered as a useful tool in the MA(q) model simply because PACF does not decay fast but approaches toward 0. However, the pattern of ACF is more applicable to MA. PACF, on the other hand, works well with the AR(p) process.
+
+PACF provides information on the correlation between the current value of a time series and its lagged values, controlling for the other correlations. PACF measures the correlation between current values of series and lagged values in a way to isolate in-between effects.
+
+![image](https://github.com/user-attachments/assets/41d4ce8e-06d4-4489-a43a-c77105bf172d)
+
+In interpreting the PACF, we focus on the spikes outside the dark region representing confidence interval. The graph exhibits some spikes at different lags, but lag 10 is outside the confidence interval. So it may be wise to select a model with 10 lags to include all the lags up to lag 10.
+
+#### Seasonality
+
+Seasonality exists if there are regular fluctuations over a given period of time
+
+Here is a plot of energy capacity utilisation
+
+![image](https://github.com/user-attachments/assets/8cc30f4a-7693-4c37-a270-08eb3fd47012)
+
+The plot indicates periodic ups and downs over a nearly 10-year period with high-capacity utilization during the first months of every year and then going down toward the end of year, confirming that there is seasonality in energy-capacity utilization.
+
+We can also plot an ACF with lag 30 which confirms the above observation:
+
+![image](https://github.com/user-attachments/assets/ee29277f-a6e3-409e-b849-b08044a16ab9)
+
+#### Cyclicality
+
+Refers to variations that occur over long periods but without a fixed time interval, unlike seasonality.
+
+#### Residuals
+
+Residuals in time series analysis represent the "leftover" or unexplained variation after fitting a model to the data. They are calculated as the difference between the observed values and the model's predicted values. Residuals help assess the accuracy of the model—ideally, they should show no patterns and behave like random noise, meaning the model has captured all the underlying structure.
+
+If residuals display patterns, it indicates that the model has missed some aspects of the data, such as trends, seasonality, or cyclicality, suggesting the need for model adjustments or refinements.
+
+### Time Series Models
+
+Traditional time series models are univariate models, and they follow these phases:
+
+* identification: here, we explore the data using ACF and PACF, identifying patterns and conducting statistical tests
+* estimation: we estimate coefficients via the proper optimization technique
+* diagnostics: after estimation, we need to check if information criteria or ACF/PACF suggest that the model is valid. If so, we move on to the forecasting stage
+* forecast: this part is more about the performance of the model. In forecasting, we predict future values based on our estimation
+
+![image](https://github.com/user-attachments/assets/007c40bf-ac4f-469d-a29e-61b0ba59e657)
+
+### White Noise
+
+White noise is a time series with a mean of 0 and constant variance, where successive values are uncorrelated. It is stationary, meaning its properties don't change over time, and its plot shows random fluctuations around the mean. However, because the values are uncorrelated, white noise is not useful for forecasting future values, as we cannot predict one value based on previous ones.
+
+We need to identify the optimum number of lags before running the time series model. Deciding the optimal number of lags is a challenging task and the most widely used methods are ACF, PACF, and information criteria.
+
+**Information Criteria**
+
+A(Aikake)IC is introduced as an extension to the Maximum Likelihood Principle. Maximum likelihood is conventionally applied to estimate the parameters of a model once structure and dimension of the model have been formulated.
+
+AIC can be mathematically defined as:
+
+`AIC = -2ln(Maximum Likelihood) + 2d`
+
+where d is the total number of parameters. The last term, 2d, aims at reducing the risk of overfitting. It is also called a penalty term, by which the unnecessary redundancy in the model can be filtered out.
+
+The Bayesian information criterion (BIC) is the other information criterion used to select the best model. The penalty term in BIC is larger than that of AIC:
+
+`BIC = -2ln(Maximum Likelihood + ln(n)d`
+
+where n is the number of observations
+
+Hurvich and Tsai (1989):
+
+_If the true model is infinite dimensional, a case which seems most realistic in practice, AIC provides an asymptotically efficient selection of a finite dimensional approximating model. If the true model is finite dimensional, however, the asymptotically efficient methods, e.g., Akaike’s FPE (Akaike 1970), AIC, and Parzen’s CAT (Parzen 1977), do not provide consistent model order selections._
+
+#### Moving Average Model
+
+MA and residuals are closely related models. MA can be considered a smoothing model, as it tends to take into account the lag values of residuals
+
+To be consistent, all the below models will use the stock data of Microsoft and Apple.
+
+![image](https://github.com/user-attachments/assets/ff062ebb-e215-4d0c-af95-60596d30da97)
+
+We can see that there are significant spikes at some lags and, therefore, we’ll choose lag 9 for the short MA model and 22 for the long MA for Apple
+
+The data is preprocessed as follows:
+
+* drop na
+* take 1st difference
+* split into train and test
+
+MA model prediction results for Apple:
+
+![image](https://github.com/user-attachments/assets/d97489a8-ed6c-4a74-8147-ff88f9d3f560)
+
+MA model prediction results for Microsoft:
+
+![image](https://github.com/user-attachments/assets/4b89479f-43ac-4e4c-8a95-8a1412217328)
+
+#### Autoregressive model
+
+The dependence structure of successive terms is the most distinctive feature of the AR model, in the sense that current value is regressed over its own lag values in this model. So we basically forecast the current value of the time series X_t by using a linear combination of its past values
+
+The AR(p) model implies that past values up to order p have somewhat explanatory power on X_t. If the relationship has shorter memory, then it is likely to model X_t with a fewer number of lags.
+
+We have discussed one of the main properties of time series, stationarity; the other important property is invertibility. After introducing the AR model, it is time to show the invertibility of the MA process. It is said to be invertible if it can be converted to an infinite AR model.
+
+In the following analysis, we run the AR model to predict Apple and Microsoft stock prices. Unlike MA, partial ACF is a useful tool to find out the optimum order in the AR model. This is because, in AR, we aim to find out the relationship of a time series between two different times, say X_t and X_t-k, and to do that we need to filter out the effect of other lags in between
+
+![image](https://github.com/user-attachments/assets/4fb602b1-1b35-4d8e-99fa-54771e1087bb)
+
+![image](https://github.com/user-attachments/assets/bf847951-3329-403f-9f24-522d2b517903)
+
+In the 1st plot, obtained from the first differenced Apple stock price, we see a significant spike at lag 29, and in the 2nd plot, we have a similar spike at lag 26 for Microsoft. Thus, 29 and 26 are the lags that we are going to use in modeling AR for Apple and Microsoft, respectively
+
+AR model prediction results:
+
+![image](https://github.com/user-attachments/assets/bda005a3-48ef-4c14-a258-e491cb27093c)
+
+#### Autoregressive Integrated Moving Average Model
+
+The ARIMA is a function of past values of a time series and white noise. ARIMA has been proposed as a generalization of AR and MA, but they do not have an integration parameter, which helps us to feed the model with the raw data. In this respect, even if we include nonstationary data, ARIMA makes it stationary by properly defining the integration parameter.
+
+ARIMA has three parameters, namely p, d, and q. As should be familiar from previous time series models, p and q refer to the order of AR and MA, respectively. The d parameter controls for level difference. If d = 1, it amounts to first difference, and if it has a value of 0, that means that the model is ARIMA. It is possible to have a d greater than 1, but it’s not as common as having a d of 1.
+
+Pros:
+
+* ARIMA allows us to work with raw data without considering if it is stationary
+* it performs well with high-frequency data
+* it is less sensitive to the fluctuation in the data compared to other models
+
+Cons:
+
+* ARIMA might fail in capturing seasonality
+* it works better with long series and short-term (daily, hourly) data
+* as no automatic updating occurs in ARIMA, no structural break during the analysis period should be observed
+* having no adjustment in the ARIMA process leads to instability
+
+ARIMA prediction results:
+
+![image](https://github.com/user-attachments/assets/3e606d35-4147-4693-b60f-596ce0717f27)
+
+All the code from this chapter is [here](https://github.com/abdullahkarasan/mlfrm/blob/main/codes/chp_2.ipynb)
+
+## Chapter 3 - Deep Learning for Time Series Modeling
+
+This chapter was an explanation of what RNN and LSTM models are and their architecture. Then using tensorflow for predictions. The end prediction results are:
+
+**RNN prediction results**
+
+![image](https://github.com/user-attachments/assets/faad3730-0780-4a3a-a7fc-9a27fcf9bd91)
+
+**LSTM prediction results**
+![image](https://github.com/user-attachments/assets/edbd7bcf-7eac-4d8a-a5e0-03d409794a45)
+
+## Next in the book is **Part II. Machine Learning for Market, Credit, Liquidity, and Operational Risks** which should be very interesting
+
+<img width="253" alt="image" src="https://github.com/user-attachments/assets/2120fc71-ed41-496f-9279-32067770b472">
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From de38978574774c7af4f591111d1e9c1863d3a08f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 19 Oct 2024 23:15:11 +0900
Subject: [PATCH 468/638] Create 2024-10-19-day-292.md

---
 _posts/2024-10-19-day-292.md | 1098 ++++++++++++++++++++++++++++++++++
 1 file changed, 1098 insertions(+)
 create mode 100644 _posts/2024-10-19-day-292.md

diff --git a/_posts/2024-10-19-day-292.md b/_posts/2024-10-19-day-292.md
new file mode 100644
index 000000000..b891c8848
--- /dev/null
+++ b/_posts/2024-10-19-day-292.md
@@ -0,0 +1,1098 @@
+---
+layout: post
+title: (Day 292) Continuing with the ML for financial risk management book
+categories: [traditional-machine-learning,theory]
+---
+
+# Hello :) Today is Day 292!
+A quick summary of today:
+* read a few more chapters of the Financial Risk Management book
+* read a longer post on linear models and coefficient estimation from sklearn's docs
+
+## ML for Financial Risk Management Chapter 4 - Machine Learning-Based Volatility Prediction
+
+Volatility is the backbone of finance in the sense that it not only provides an information signal to investors, but it also is an input to various financial models. What makes volatility so important? The answer stresses the importance of uncertainty, which is the main characteristic of the financial model.
+
+Volatility used as a proxy of risk is among the most important variables in many fields, including asset pricing and risk management.
+
+Modeling volatility amounts to modeling uncertainty so that we better understand and approach uncertainty, enabling us to have good enough approximations of the real world. To gauge the extent to which proposed models account for the real-world situation, we need to calculate the return volatility, which is also known as realized volatility. Realized volatility is the square root of realized variance, which is the sum of squared return. Realized volatility is used to calculate the performance of the volatility prediction method. Here is the formula for return volatility:
+
+<img width="286" alt="image" src="https://github.com/user-attachments/assets/479ee5fb-5aa1-4c83-9851-2b0b93c2ad20">
+
+where r and mu are return and mean of return, and n is number of observations
+
+### ARCH model
+
+One of the early attempts to model volatility was proposed by Eagle (1982) and is known as the ARCH model. The ARCH model is a univariate model and based on historical asset returns. The ARCH(p) model has the following form:
+
+<img width="210" alt="image" src="https://github.com/user-attachments/assets/4c096b7a-f806-4f5d-8fc7-28e0be7372e1">
+
+where the mean model is:
+
+<img width="109" alt="image" src="https://github.com/user-attachments/assets/723ebc56-86f9-4143-8a61-38869104a08c">
+
+where Є_t is assumed to be normally distributed. In this parametric model, we need to satisfy some assumptions to have strictly positive variance. In this respect, the following conditions should hold:
+
+* w > 0, and
+* alpha_k >= 0
+
+All of these equations tell us that ARCH is a univariate and nonlinear model in which volatility is estimated with the square of past returns. One of the most distinctive features of ARCH is that it has the property of time-varying conditional variance1 so that ARCH is able to model the phenomenon known as volatility clustering—that is, large changes tend to be followed by large changes of either sign, and small changes tend to be followed by small changes. Hence, once an important announcement is made to the market, it might result in huge volatility.
+
+Here is S&P500's volatility clustering:
+
+![image](https://github.com/user-attachments/assets/dfcac18f-6e29-4e5b-8caa-8de31b5de587)
+
+Despite its appealing features, such as simplicity, nonlinearity, easiness, and adjustment for forecast, the ARCH model has certain drawbacks:
+
+* equal response to positive and negative shocks
+* strong assumptions such as restrictions on parameters
+* possible misprediction due to slow adjustments to large movements
+
+To run ARCH, we can use the arch library
+
+```python
+arch = arch_model(ret, mean='zero', vol='ARCH', p=1).fit(disp='off') # ret is the S&P500 return
+print(arch.summary())
+```
+
+```
+                        Zero Mean - ARCH Model Results                        
+==============================================================================
+Dep. Variable:              Adj Close   R-squared:                       0.000
+Mean Model:                 Zero Mean   Adj. R-squared:                  0.000
+Vol Model:                       ARCH   Log-Likelihood:               -4061.27
+Distribution:                  Normal   AIC:                           8126.54
+Method:            Maximum Likelihood   BIC:                           8138.50
+                                        No. Observations:                 2913
+Date:                Sat, Oct 19 2024   Df Residuals:                     2913
+Time:                        05:28:00   Df Model:                            0
+                            Volatility Model                            
+========================================================================
+                 coef    std err          t      P>|t|  95.0% Conf. Int.
+------------------------------------------------------------------------
+omega          0.7014  5.002e-02     14.023  1.129e-44 [  0.603,  0.799]
+alpha[1]       0.3909  7.011e-02      5.575  2.474e-08 [  0.253,  0.528]
+========================================================================
+
+Covariance estimator: robust
+```
+
+To find a better model we can compare a few with different lags. For that we will use Bayesian Informaiont Criteria (BIC) as the model selection method and to select lag. The reason BIC is used is that as long as we have large enough samples, BIC is a reliable tool for model selection
+
+```python
+bic_arch = []
+
+for p in range(1, 5):
+        arch = arch_model(ret, mean='zero', vol='ARCH', p=p)\
+                .fit(disp='off')
+        bic_arch.append(arch.bic)
+        if arch.bic == np.min(bic_arch):
+            best_param = p
+arch = arch_model(ret, mean='zero', vol='ARCH', p=best_param)\
+        .fit(disp='off')
+print(arch.summary())
+forecast = arch.forecast(start=split_date[0])
+forecast_arch = forecast
+```
+
+Result:
+
+```
+                        Zero Mean - ARCH Model Results                        
+==============================================================================
+Dep. Variable:              Adj Close   R-squared:                       0.000
+Mean Model:                 Zero Mean   Adj. R-squared:                  0.000
+Vol Model:                       ARCH   Log-Likelihood:               -3709.51
+Distribution:                  Normal   AIC:                           7429.03
+Method:            Maximum Likelihood   BIC:                           7458.91
+                                        No. Observations:                 2913
+Date:                Sat, Oct 19 2024   Df Residuals:                     2913
+Time:                        05:28:08   Df Model:                            0
+                             Volatility Model                             
+==========================================================================
+                 coef    std err          t      P>|t|    95.0% Conf. Int.
+--------------------------------------------------------------------------
+omega          0.2794  2.580e-02     10.829  2.498e-27   [  0.229,  0.330]
+alpha[1]       0.1519  3.458e-02      4.394  1.111e-05 [8.417e-02,  0.220]
+alpha[2]       0.2329  3.614e-02      6.444  1.164e-10   [  0.162,  0.304]
+alpha[3]       0.1917  3.702e-02      5.178  2.241e-07   [  0.119,  0.264]
+alpha[4]       0.1923  4.149e-02      4.634  3.586e-06   [  0.111,  0.274]
+==========================================================================
+
+Covariance estimator: robust
+```
+
+Get the RMSE:
+
+```python
+rmse_arch = np.sqrt(mse(realized_vol[-n:] / 100, 
+                        np.sqrt(forecast_arch\
+                        .variance.iloc[-len(split_date):] 
+                        / 100)))
+print('The RMSE value of ARCH model is {:.4f}'.format(rmse_arch))
+# The RMSE value of ARCH model is 0.0896
+```
+
+Then plot along with the predictions
+
+```python
+plt.figure(figsize=(10, 6))
+plt.plot(realized_vol / 100, label='Realized Volatility')
+plt.plot(forecast_arch.variance.iloc[-len(split_date):] / 100, 
+         label='Volatility Prediction-ARCH')
+plt.title('Volatility Prediction with ARCH', fontsize=12)
+plt.legend()
+plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/86c6c7d3-0737-40ba-a3ae-d12966c4f336)
+
+### GARCH Model
+
+To tackle the drawbacks of the ARCH model, researchers developed GARCH.
+
+The GARCH model is an extension of the ARCH model incorporating lagged conditional variance. So ARCH is improved by adding p number of delated conditional variance, which makes the GARCH model multivariate in the sense that it is an autoregressive moving average model for conditional variance with p number of lagged squared returns and q number of lagged conditional variance. GARCH(p, q) can be formulated as:
+
+<img width="332" alt="image" src="https://github.com/user-attachments/assets/28be82dd-514e-49f1-96f6-4f6b72954548">
+
+where w, beta, and alpha are parameters to be estimated and p and q are maximum lag in the model. To have consistent GARCH, the following conditions should hold:
+
+* w > 0
+* beta >= 0
+* alpha >= 0
+* beta + alpha < 1
+
+The ARCH model is unable to capture the influence of historical innovations. However, as a more parsimonious model, the GARCH model can account for the change in historical innovations because GARCH models can be expressed as an infinite-order ARCH
+
+To run GARCH we can again use the arch library:
+
+```python
+garch = arch_model(ret, mean='zero', vol='GARCH', p=1, o=0, q=1)\
+        .fit(disp='off')
+print(garch.summary())
+```
+
+```
+                       Zero Mean - GARCH Model Results                        
+==============================================================================
+Dep. Variable:              Adj Close   R-squared:                       0.000
+Mean Model:                 Zero Mean   Adj. R-squared:                  0.000
+Vol Model:                      GARCH   Log-Likelihood:               -3654.98
+Distribution:                  Normal   AIC:                           7315.97
+Method:            Maximum Likelihood   BIC:                           7333.90
+                                        No. Observations:                 2913
+Date:                Sat, Oct 19 2024   Df Residuals:                     2913
+Time:                        05:39:36   Df Model:                            0
+                              Volatility Model                              
+============================================================================
+                 coef    std err          t      P>|t|      95.0% Conf. Int.
+----------------------------------------------------------------------------
+omega          0.0393  8.429e-03      4.664  3.100e-06 [2.279e-02,5.583e-02]
+alpha[1]       0.1745  2.278e-02      7.661  1.845e-14     [  0.130,  0.219]
+beta[1]        0.7892  2.273e-02     34.729 2.853e-264     [  0.745,  0.834]
+============================================================================
+
+Covariance estimator: robust
+```
+
+We can again find more optimum params:
+
+```python
+bic_garch = []
+
+for p in range(1, 5):
+    for q in range(1, 5):
+        garch = arch_model(ret, mean='zero',vol='GARCH', p=p, o=0, q=q)\
+                .fit(disp='off')
+        bic_garch.append(garch.bic)
+        if garch.bic == np.min(bic_garch):
+            best_param = p, q
+garch = arch_model(ret, mean='zero', vol='GARCH',
+                   p=best_param[0], o=0, q=best_param[1])\
+        .fit(disp='off')
+print(garch.summary())
+forecast = garch.forecast(start=split_date[0])
+forecast_garch = forecast
+```
+
+```
+                       Zero Mean - GARCH Model Results                        
+==============================================================================
+Dep. Variable:              Adj Close   R-squared:                       0.000
+Mean Model:                 Zero Mean   Adj. R-squared:                  0.000
+Vol Model:                      GARCH   Log-Likelihood:               -3654.98
+Distribution:                  Normal   AIC:                           7315.97
+Method:            Maximum Likelihood   BIC:                           7333.90
+                                        No. Observations:                 2913
+Date:                Sat, Oct 19 2024   Df Residuals:                     2913
+Time:                        05:41:32   Df Model:                            0
+                              Volatility Model                              
+============================================================================
+                 coef    std err          t      P>|t|      95.0% Conf. Int.
+----------------------------------------------------------------------------
+omega          0.0393  8.429e-03      4.664  3.100e-06 [2.279e-02,5.583e-02]
+alpha[1]       0.1745  2.278e-02      7.661  1.845e-14     [  0.130,  0.219]
+beta[1]        0.7892  2.273e-02     34.729 2.853e-264     [  0.745,  0.834]
+============================================================================
+
+Covariance estimator: robust
+```
+
+```python
+rmse_garch = np.sqrt(mse(realized_vol[-n:] / 100,
+                         np.sqrt(forecast_garch\
+                         .variance.iloc[-len(split_date):] 
+                         / 100)))
+print('The RMSE value of GARCH model is {:.4f}'.format(rmse_garch))
+# 0.0878
+```
+
+```python
+plt.figure(figsize=(10,6))
+plt.plot(realized_vol / 100, label='Realized Volatility')
+plt.plot(forecast_garch.variance.iloc[-len(split_date):] / 100, 
+         label='Volatility Prediction-GARCH')
+plt.title('Volatility Prediction with GARCH', fontsize=12)
+plt.legend()
+plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/1239d3dd-884c-49da-8931-6724b7c7cb7a)
+
+The volatility of returns is well-fitted by the GARCH model partly because of its volatility clustering and partly because GARCH does not assume that the returns are independent, which allows it to account for the leptokurtic property of returns. However, despite these useful properties and its intuitiveness, GARCH is not able to model the asymmetric response of the shocks.
+
+To remedy this issue, GJR-GARCH was proposed by Glosten, Jagannathan, and Runkle (1993)
+
+### GJR-GARCH
+
+The GJR-GARCH model performs well in modeling the asymmetric effects of announcements in the way that bad news has a larger impact than good news. In other words, in the presence of asymmetry, the distribution of losses has a fatter tail than the distribution of gains
+
+<img width="493" alt="image" src="https://github.com/user-attachments/assets/1ea96443-ce5d-49db-8e83-c67a346b3152">
+
+The equation of this model includes gamma, which controls for the asymmetry of the announcements and if:
+
+* gamma = 0 -> the response to the past shock is the same
+* gamma > 0 -> the response to the past negative shock is stronger than a positive one
+* gamma < 0 -> the response to the past positive shock is stronger than a negative one
+
+Here is how to run it:
+
+```python
+bic_gjr_garch = []
+
+for p in range(1, 5):
+    for q in range(1, 5):
+        gjrgarch = arch_model(ret, mean='zero', p=p, o=1, q=q)\
+                   .fit(disp='off')
+        bic_gjr_garch.append(gjrgarch.bic)
+        if gjrgarch.bic == np.min(bic_gjr_garch):
+            best_param = p, q
+gjrgarch = arch_model(ret,mean='zero', p=best_param[0], o=1,
+                      q=best_param[1]).fit(disp='off')
+print(gjrgarch.summary())
+forecast = gjrgarch.forecast(start=split_date[0])
+forecast_gjrgarch = forecast
+```
+
+Model summary:
+
+```
+                     Zero Mean - GJR-GARCH Model Results                      
+==============================================================================
+Dep. Variable:              Adj Close   R-squared:                       0.000
+Mean Model:                 Zero Mean   Adj. R-squared:                  0.000
+Vol Model:                  GJR-GARCH   Log-Likelihood:               -3591.15
+Distribution:                  Normal   AIC:                           7190.30
+Method:            Maximum Likelihood   BIC:                           7214.21
+                                        No. Observations:                 2913
+Date:                Sat, Oct 19 2024   Df Residuals:                     2913
+Time:                        05:47:22   Df Model:                            0
+                               Volatility Model                              
+=============================================================================
+                 coef    std err          t      P>|t|       95.0% Conf. Int.
+-----------------------------------------------------------------------------
+omega          0.0430  7.754e-03      5.547  2.908e-08  [2.781e-02,5.821e-02]
+alpha[1]       0.0390  3.069e-02      1.270      0.204 [-2.116e-02,9.913e-02]
+gamma[1]       0.2798  4.799e-02      5.831  5.501e-09      [  0.186,  0.374]
+beta[1]        0.7907  2.700e-02     29.286 1.572e-188      [  0.738,  0.844]
+=============================================================================
+
+Covariance estimator: robust
+```
+
+_The RMSE value of GJR-GARCH models is 0.0882_
+
+Volatility prediction plot:
+
+![image](https://github.com/user-attachments/assets/0239a2c4-75c6-4939-8b88-3936516a07d6)
+
+### EGARCH
+
+Together with the GJR-GARCH model, the EGARCH model, proposed by Nelson (1991), is another tool for controlling for the effect of asymmetric announcements.
+
+The main difference in the EGARCH equation is that logarithm is taken of the variance on the left-hand side of the equation. This indicates the leverage effect, meaning that there exists a negative correlation between past asset returns and volatility.
+
+Here is a comparison table for the above 4 models:
+
+| Model       | RMSE   |
+|-------------|--------|
+| ARCH        | 0.0896 |
+| GARCH       | 0.0878 |
+| GJR-GARCH   | 0.0882 |
+| EGARCH      | 0.0904 |
+
+Above are all classical volatility models, but next are ML models.
+
+### Support Vector Regression: GARCH
+
+Support Vector Machine (SVM) is a supervised learning algorithm used for both classification and regression tasks. The primary goal of SVM is to identify the optimal hyperplane that separates two classes by maximizing the margin between the closest points (support vectors) of each class.
+
+In the context of regression, the aim is to find the hyperplane that minimizes error while maximizing margin, leading to a method known as Support Vector Regression (SVR). When applied to the GARCH model, this combination results in the SVR-GARCH model, leveraging the strengths of both approaches for improved predictions.
+
+There are 3 types of kernel functions used with SVMs - polynomial, gaussian, and exponential.
+
+The following code shows us the preparations before running the SVR-GARCH in Python. The most crucial step here is to obtain independent variables, which are realized volatility and square of historical returns:
+
+```python
+from sklearn.svm import SVR
+from scipy.stats import uniform as sp_rand
+from sklearn.model_selection import RandomizedSearchCV
+
+realized_vol = ret.rolling(5).std()
+realized_vol = pd.DataFrame(realized_vol)
+realized_vol.reset_index(drop=True, inplace=True)
+
+returns_svm = ret ** 2
+returns_svm = returns_svm.reset_index()
+del returns_svm['Date']
+
+X = pd.concat([realized_vol, returns_svm], axis=1, ignore_index=True)
+X = X[4:].copy()
+X = X.reset_index()
+X.drop('index', axis=1, inplace=True)
+
+realized_vol = realized_vol.dropna().reset_index()
+realized_vol.drop('index', axis=1, inplace=True)
+
+# create for each kind of kernel
+svr_poly = SVR(kernel='poly', degree=2)
+svr_lin = SVR(kernel='linear')
+svr_rbf = SVR(kernel='rbf')
+```
+
+#### SVR-GARCH-Linear
+
+```python
+para_grid = {'gamma': sp_rand(),
+             'C': sp_rand(),
+             'epsilon': sp_rand()}
+clf = RandomizedSearchCV(svr_lin, para_grid)
+clf.fit(X.iloc[:-n].values, 
+        realized_vol.iloc[1:-(n-1)].values.reshape(-1,))
+predict_svr_lin = clf.predict(X.iloc[-n:])
+
+predict_svr_lin = pd.DataFrame(predict_svr_lin)
+predict_svr_lin.index = ret.iloc[-n:].index
+
+rmse_svr = np.sqrt(mse(realized_vol.iloc[-n:] / 100,
+                       predict_svr_lin / 100))
+print('The RMSE value of SVR with Linear Kernel is {:.6f}'.format(rmse_svr))
+
+# The RMSE value of SVR with Linear Kernel is 0.000505
+
+realized_vol.index = ret.iloc[4:].index
+
+plt.figure(figsize=(10, 6))
+plt.plot(realized_vol / 100, label='Realized Volatility')
+plt.plot(predict_svr_lin / 100, label='Volatility Prediction-SVR-GARCH')
+plt.title('Volatility Prediction with SVR-GARCH (Linear)', fontsize=12)
+plt.legend()
+plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/f996df7c-012c-4472-bbdb-cdea899b6d4e)
+
+The linear kernel works quite well (if the features are linearly separable)
+
+Next are the radial basis function (RBF) and polynomial kernels. The former uses elliptical curves around the observations, and the latter, unlike the first two, focuses on the combinations of samples.
+
+#### SVR-GARCH RBF
+
+The RMSE value of SVR with RBF Kernel is  0.000455
+
+![image](https://github.com/user-attachments/assets/80d5caa0-c5b5-402f-81ea-0ae53ca2ca8c)
+
+#### SVR-GARCH Polynomial
+
+The RMSE value of SVR with Polynomial Kernel is 0.002386
+
+![image](https://github.com/user-attachments/assets/6bcec46d-4a0e-41d7-9e57-d36eeed050e9)
+
+### Neural Networks
+
+We can use sklearn's MLPRegressor
+
+```python
+from sklearn.neural_network import MLPRegressor
+NN_vol = MLPRegressor(learning_rate_init=0.001, random_state=1) 
+para_grid_NN = {'hidden_layer_sizes': [(100, 50), (50, 50), (10, 100)],
+               'max_iter': [500, 1000],
+               'alpha': [0.00005, 0.0005 ]}
+clf = RandomizedSearchCV(NN_vol, para_grid_NN)
+clf.fit(X.iloc[:-n].values, 
+        realized_vol.iloc[1:-(n-1)].values.reshape(-1, ))
+NN_predictions = clf.predict(X.iloc[-n:])
+
+NN_predictions = pd.DataFrame(NN_predictions)
+NN_predictions.index = ret.iloc[-n:].index
+
+rmse_NN = np.sqrt(mse(realized_vol.iloc[-n:] / 100,
+                      NN_predictions / 100))
+print('The RMSE value of NN is {:.6f}'.format(rmse_NN))
+
+# The RMSE value of NN is 0.000647
+
+plt.figure(figsize=(10, 6))
+plt.plot(realized_vol / 100, label='Realized Volatility')
+plt.plot(NN_predictions / 100, label='Volatility Prediction-NN')
+plt.title('Volatility Prediction with Neural Network', fontsize=12)
+plt.legend()
+plt.show()
+```
+
+Result:
+
+![image](https://github.com/user-attachments/assets/5e79adac-6dc6-4e5e-827b-6355a5028fd6)
+
+Despite its reasonable performance, we can play with the number of hidden neurons to generate a deep learning model.
+
+We can use TF:
+
+```python
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras import layers
+
+model = keras.Sequential(
+    [layers.Dense(256, activation="relu"),
+     layers.Dense(128, activation="relu"),
+     layers.Dense(1, activation="linear"),])
+
+model.compile(loss='mse', optimizer='rmsprop')
+
+epochs_trial = np.arange(100, 400, 4)
+batch_trial = np.arange(100, 400, 4)
+DL_pred = []
+DL_RMSE = []
+for i, j, k in zip(range(4), epochs_trial, batch_trial):
+    model.fit(X.iloc[:-n].values,
+              realized_vol.iloc[1:-(n-1)].values.reshape(-1,),
+              batch_size=k, epochs=j, verbose=False)
+    DL_predict = model.predict(np.asarray(X.iloc[-n:]))
+    DL_RMSE.append(np.sqrt(mse(realized_vol.iloc[-n:] / 100,
+                            DL_predict.flatten() / 100)))
+    DL_pred.append(DL_predict)
+    print('DL_RMSE_{}:{:.6f}'.format(i+1, DL_RMSE[i]))
+
+# DL_RMSE_1:0.000701
+# DL_RMSE_2:0.000611
+# DL_RMSE_3:0.000807
+# DL_RMSE_4:0.000655
+
+DL_predict = pd.DataFrame(DL_pred[DL_RMSE.index(min(DL_RMSE))])
+DL_predict.index = ret.iloc[-n:].index
+
+plt.figure(figsize=(10, 6))
+plt.plot(realized_vol / 100,label='Realized Volatility')
+plt.plot(DL_predict / 100,label='Volatility Prediction-DL')
+plt.title('Volatility Prediction with Deep Learning',  fontsize=12)
+plt.legend()
+plt.show()
+```
+
+![image](https://github.com/user-attachments/assets/d91548c9-7c84-4f0c-9c94-ff23ca64f4d8)
+
+_All the code from this chapter is [here](https://github.com/abdullahkarasan/mlfrm/blob/main/codes/chp_4.ipynb)_
+
+## Chapter 5. Modeling Market Risk
+
+_A measure of risk driven by historical data assumes the future will follow the pattern of the past. You need to understand the limitations of that assumption. More importantly, you need to model scenarios in which that pattern breaks down._
+
+Miles Kennedy
+
+Risk in finance is complex and varies by source, making it essential to differentiate risks and apply appropriate tools for each. The main types of risk are market, credit, liquidity, and operational. Market risk, for example, is tied to changes in financial indicators like inflation and interest rates, and tools like Value at Risk (VaR) and Expected Shortfall (ES) are used to manage it. 
+
+Traditional models, limited by assumptions and complexity, struggle with modern financial systems. Machine learning (ML) models, on the other hand, identify patterns and relationships in data without predefined theories, offering a fresh approach to both prediction and theory discovery. Below, first traditional statistical models are explored, and then more modern ML models.
+
+### Value at Risk (VaR)
+
+The VaR model emerged from a request made by a J.P. Morgan executive who wanted to have a summary report showing possible losses as well as risks that J.P. Morgan was exposed to on a given day. This report would inform executives about the risks assumed by the institution in an aggregated manner. The method by which market risk is computed is known as VaR. This report was the starting point of VaR, and now it has become so widespread that not only institutions prefer using VaR, but its adoption has become required by regulators.
+
+The adoption of VaR dates back to the 1990s, and despite numerous extensions to it and new proposed models, it is still in use
+
+What makes it so appealing? The answer comes from Kevin Dowd:
+
+_The VaR figure has two important characteristics. The first is that it provides a common consistent measure of risk across different positions and risk factors. It enables us to measure the risk associated with a fixed-income position, say, in a way that is comparable to and consistent with a measure of the risk associated with equity positions. VaR provides us with a common risk yardstick, and this yardstick makes it possible for institutions to manage their risks in new ways that were not possible before. The other characteristic of VaR is that it takes account of the correlations between different risk factors. If two risks offset each other, the VaR allows for this offset and tells us that the overall risk is fairly low._
+
+In fact, VaR addresses one of the most common questions an investor has: what is the maximum expected loss of my investment
+
+VaR provides a very intuitive and practical answer to this question. In this regard, it is used to measure the worst expected loss for a company over a given period and a pre-defined confidence interval. Suppose that a daily VaR of an investment is $1 million with 95% confidence interval. This would read as there being a 5% chance that an investor might incur a loss greater than $1 million in a day.
+
+In summary, there are some important points in VaR analysis that need to be highlighted:
+
+* VaR needs an estimation of the probability of loss
+* VaR concentrates on the potential losses. We are not talking about actual or realized losses; rather, VaR is a kind of loss projection
+
+VaR has three key ingredients:
+
+* standard deviation that defines the level of loss
+* fixed time horizon over which risk is assessed
+* Confidence interval
+
+VaR can be measured via three different approaches:
+
+* variance-covariance VaR
+* historical simulation VaR
+* Monte Carlo VaR
+
+#### Variance-Covariance Method
+
+The variance-covariance method is also known as the parametric method, because observations are assumed to be normally distributed. The variance-covariance method is commonplace in that returns are deemed to follow normal distribution. The parametric form assumption makes the application of variance-covariance method easy.
+
+As in all VaR approaches, we can either work with a single asset or a portfolio. However, working with a portfolio requires careful treatment in the sense that correlation structure and portfolio variance need to be estimated. At this point, correlation comes into the picture, and historical data is used to calculate correlation, mean, and standard deviation. When augmenting this with an ML-based approach, correlation structure will be the main focus.
+
+Suppose that we have a portfolio consisting of a single asset
+
+![image](https://github.com/user-attachments/assets/bae7e704-c266-4122-a157-919f75fa2ce2)
+
+It is shown that the return of this asset is zero and standard deviation is 1, and if the holding period is 1, the corresponding VaR value can be computed from the value of the asset by the corresponding Z-value and standard deviation. Hence, the normality assumption makes things easier, but it is a strong assumption, as there is no guarantee that asset returns are normally distributed; rather, most asset returns do not follow a normal distribution. Moreover, due to the normality assumption, potential risk in tail might not be captured. Therefore the normality assumption comes with a cost. 
+
+Here is how to apply the variance-covariance VaR in python.
+
+<img width="350" alt="image" src="https://github.com/user-attachments/assets/8c2fe1d6-c31d-4540-8a14-20db95f145a5">
+
+```python
+stocks_returns = (np.log(stocks) - np.log(stocks.shift(1))).dropna()
+
+stocks_returns_mean = stocks_returns.mean()
+weights  = np.random.random(len(stocks_returns.columns))
+weights /= np.sum(weights)
+cov_var = stocks_returns.cov()
+port_std = np.sqrt(weights.T.dot(cov_var).dot(weights))
+
+initial_investment = 1e6
+conf_level = 0.95
+
+def VaR_parametric(initial_investment, conf_level):
+    alpha = norm.ppf(1 - conf_level, stocks_returns_mean, port_std)
+    for i, j in zip(stocks.columns, range(len(stocks.columns))):
+        VaR_param = (initial_investment - initial_investment * 
+                     (1 + alpha))[j]
+        print("Parametric VaR result for {} is {} "
+              .format(i, VaR_param))
+    VaR_param = (initial_investment - initial_investment * (1 + alpha))
+    print('--' * 25)
+    return VaR_param
+
+VaR_param = VaR_parametric(initial_investment, conf_level)
+
+# Parametric VaR result for IBM is 42606.16125893139
+# Parametric VaR result for MSFT is 41024.50194348814
+# Parametric VaR result for INTC is 43109.25240851776
+
+var_horizon = []
+time_horizon = 30
+for j in range(len(stocks_returns.columns)):
+    for i in range(1, time_horizon + 1):
+        var_horizon.append(VaR_param[j] * np.sqrt(i))
+plt.plot(var_horizon[:time_horizon], "o",
+         c='blue', marker='*', label='IBM')
+plt.plot(var_horizon[time_horizon:time_horizon + 30], "o",
+         c='green', marker='o', label='MSFT')
+plt.plot(var_horizon[time_horizon + 30:time_horizon + 60], "o",
+         c='red', marker='v', label='INTC')
+plt.xlabel("Days")
+plt.ylabel("USD")
+plt.title("VaR over 30-day period")
+plt.legend()
+plt.show()
+```
+
+VaR changes depending on the time horizon in the sense that holding assets for a longer period makes an investor more susceptible to risk
+
+![image](https://github.com/user-attachments/assets/a0858b71-772a-4ee6-9538-98f25cb0d860)
+
+The pros and cons of the variance-covariance method are as follows:
+
+Pros
+
+* easy to calculate
+* does not require a large number of samples
+
+Cons
+
+* observations are normally distributed
+* does not work well with nonlinear structures
+* requires the computation of the covariance matrix
+
+So, even though assuming normality sounds appealing, it may not be the best way to estimate VaR, especially in the case where the asset returns do not have a normal distribution. Luckily, there is another method that does not have a normality assumption, namely the historical simulation VaR model.
+
+#### The Historical Simulation Method
+
+This is an empirical method: instead of using a parametric approach, we find the percentile, which is the Z-table equivalent of variance-covariance method. Suppose that the confidence interval is 95%; 5% will be used in lieu of the Z-table values, and all we need to do is to multiply this percentile by the initial investment.
+
+The following are the steps taken in the historical simulation VaR:
+
+1. Obtain the asset returns of the portfolio (or individual asset)
+2. Find the corresponding return percentile based on confidence interval
+3. Multiply this percentile by initial investment
+
+In code:
+
+```python
+def VaR_historical(initial_investment, conf_level):
+    Hist_percentile95 = []
+    for i, j in zip(stocks_returns.columns,
+                    range(len(stocks_returns.columns))):
+        Hist_percentile95.append(np.percentile(stocks_returns.loc[:, i],
+                                               5))
+        print("Based on historical values 95% of {}'s return is {:.4f}"
+              .format(i, Hist_percentile95[j]))
+        VaR_historical = (initial_investment - initial_investment *
+                          (1 + Hist_percentile95[j]))
+        print("Historical VaR result for {} is {:.2f} "
+              .format(i, VaR_historical))
+        print('--' * 35)
+
+VaR_historical(initial_investment,conf_level)
+
+Based on historical values 95% of IBM's return is -0.0371
+Historical VaR result for IBM is 37081.53
+----------------------------------------------------------------------
+Based on historical values 95% of MSFT's return is -0.0426
+Historical VaR result for MSFT is 42583.68
+----------------------------------------------------------------------
+Based on historical values 95% of INTC's return is -0.0425
+Historical VaR result for INTC is 42485.39
+----------------------------------------------------------------------
+```
+
+
+The historical simulation VaR method implicitly assumes that historical price changes have a similar pattern, i.e., that there is no structural break. The pros and cons of this method are as follows:
+
+Pros
+
+* no distributional assumption
+* works well with nonlinear structures
+* easy to calculate
+
+Cons
+
+* requires a large sample
+* needs high computing power
+
+#### The Monte Carlo Simulation VaR
+
+Monte Carlo is a computerized mathematical method used to make an estimation in cases where there is no closed-form solution, so it is a highly efficient tool for numerical approximation. Monte Carlo relies on repeated random samples from a given distribution.
+
+From the application standpoint, Monte Carlo is very similar to the historical simulation VaR, but it does not use historical observations. Rather, it generates random samples from a given distribution. Monte Carlo helps decision makers by providing links between possible outcomes and probabilities, which makes it an efficient and applicable tool in finance.
+
+So in a nutshell, a Monte Carlo simulation is doing nothing but generating random samples and calculating their mean. Computationally, it follows these steps:
+
+1. Define the domain
+2. Generate random numbers
+3. Iterate and aggregate the result
+
+### Denoising
+
+The financial market is filled with both noise (random information) and signal (valuable information). Noise traders act on random behavior, while informed traders use insider information or signals to make rational decisions. The key challenge is distinguishing between noise and signal, as failing to do so can lead to poor profit gains and risk assessment. Investors must be cautious, as only the signal provides useful insights for making informed decisions. The problem lies in how to differentiate and use this information effectively.
+
+The Marchenko–Pastur theorem helps have homogenous covariance matrices. The Marchenko–Pastur theorem allows us to extract signal from noise using eigenvalues of covariance matrices.
+
+Eigenvalue and eigenvector have special meanings in a financial context. Eigenvectors represent the variance in covariance matrix, while an eigenvalue shows the magnitude of an eigenvector. Specifically, the largest eigenvector corresponds to largest variance, and the magnitude of this is equal to the corresponding eigenvalue. Due to noise in the data, some eigenvalues can be thought of as random, and it makes sense to detect and filter out these eigenvalues to retain only signals.
+
+Below is a PDF of a Marchenko–Pastur distribution which is fit to the data
+
+![image](https://github.com/user-attachments/assets/2e499671-b547-4d92-846a-980b3378f9e0)
+
+The distribution fits the data well and we are able to differentiate the noise and signal. we can now refer to data for which the noise has filtered as denoised.
+
+We can plug it into the VaR model, which is called the denoised VaR estimation. Denoising the covariance matrix is nothing but taking unnecessary information (noise) out of the data. So we can then make use of the signal from the market, focusing our attention on the important events only.
+
+Denoising the covariance matrix includes the following stages:
+
+1. Calculate the eigenvalues and eigenvectors based on correlation matrix
+2. Use kernel density estimation, find the eigenvector for a specific eigenvalue
+3. Fit the Marchenko–Pastur distribution to the kernel density estimation
+4. Find the maximum theoretical eigenvalue using the Marchenko–Pastur distribution
+5. Calculate the average of eigenvalues greater than the theoretical value
+6. Use these new eigenvalues and eigenvectors to calculate the denoised correlation matrix
+7. Calculate the denoised covariance matrix by the new correlation matrix
+
+The difference between the traditionally applied VaR and the denoised VaR is even more pronounced in a crisis period. During a crisis period, correlation among assets becomes higher, which is sometimes referred to as correlation breakdown
+
+Despite its appeal and ease of use, VaR is not a coherent risk measure, which requires satisfying certain conditions or axioms
+
+Here are the four axioms of a coherent risk measure:
+
+1. **Monotonicity**:  
+   If one portfolio always produces a higher return than another, its risk should be lower. If Portfolio A consistently performs better than Portfolio B, the risk of Portfolio A should be lower than or equal to that of Portfolio B.
+
+2. **Sub-additivity**:  
+   The risk of a combined portfolio should not exceed the sum of the risks of the individual portfolios. Diversification should reduce or at least not increase overall risk.
+
+3. **Positive Homogeneity**:  
+   Scaling a portfolio by a positive constant scales the risk proportionally. For example, doubling a portfolio's size will double its risk.
+
+4. **Translation Invariance**:  
+   Adding a risk-free asset to a portfolio reduces the risk by the amount of that asset. If you add a certain amount of risk-free cash, the portfolio’s risk decreases by that same amount.
+
+VaR is not, but Expected Shortfall is a coherent risk measure.
+
+### Expected Shortfall
+
+Unlike VaR, ES focuses on the tail of the distribution. More specifically, ES enables us to take into account unexpected risks in the market. However, this doesn’t mean that ES and VaR are two entirely different concepts. Rather, they are related—that is, it is possible to express ES using VaR.
+
+ES can also be computed based on the historical observations. Like the historical simulation VaR method, parametric assumption can be relaxed. To do that, the first return (or loss) corresponding to the 95% is found, and then the mean of the observations greater than the 95% gives us the result.
+
+### Liquidity-Augmented Expected Shortfall
+
+ES provides us with a coherent risk measure to gauge market risk. However, though we differentiate financial risks as market, credit, liquidity, and operational risks, that does not necessarily mean that these risks are entirely unrelated to one another. Rather, they are, to some extent, correlated. That is, once a financial crisis hit the market, market risk surges along with the drawdown on lines of credit, which in turn increases liquidity risk.
+
+Ignoring the liqudity dimension of risk may result in underestimating the market risk. Therefore, augmenting ES with liquidity risk may make a more accurate and reliable estimation. Sounds good, but what is a good proxy for liquidity risk?
+
+Bid-ask spread measures are commonly used for modeling liquidity. Shortly, bid-ask spread is the difference between the highest available price (bid price) that a buyer is willing to pay and the lowest price (ask price) that a seller is willing to get. So bid-ask spread gives a tool to measure the transaction cost.
+
+To the extent that bid-ask spread is a good indicator of transaction cost, it is also a good proxy of liquidity in the sense that transaction cost is one of the components of liquidity. Spreads can be defined various ways depending on their focus. Here are the bid-ask spreads usedto incorporate liquidity risk into the ES model:
+
+<img width="734" alt="image" src="https://github.com/user-attachments/assets/726e2311-481a-49d9-8195-c4fd6ed2d44f">
+
+### Effective Cost
+
+A buyer-initiated trade occurs when a trade is executed at a price above the quoted mid price. Similarly, a seller-initiated trade occurs when a trade is executed at a price below the quoted mid price. We can then describe the effective cost as follows:
+
+<img width="477" alt="image" src="https://github.com/user-attachments/assets/3f94595b-0365-48c1-93f1-fc9b24f7ddd4">
+
+Now we need to find a way to incorporate these bid-ask spreads into the ES model so that we are able to account for the liquidity risk as well as market risk. We will employ two different methods to accomplish this task:
+
+* take the cross-sectional mean of the bid-ask spread (a row-wise averaging of the bid-ask spread); using this method, we are able to generate a measure for market-wide liquidity
+* use PCA
+
+In PCA, the number of components matter and we need to determine on which side of the n_components and information lost do we stand. Here are some possible cutoffs our case:
+
+* greater than 80% explained variance
+* more than one eigenvalue
+* the point at which the scree plot gets flatter
+
+_All detailed code from this chapter is [here](https://github.com/abdullahkarasan/mlfrm/blob/main/codes/chp_5.ipynb)_
+
+## Chapter 6 - Credit Risk Estimation
+
+_Although market risk is much better researched, the larger part of banks’ economic capital is generally used for credit risk. The sophistication of traditional standard methods of measurement, analysis, and management of credit risk might, therefore, not be in line with its significance._
+
+Uwe Wehrspohn (2002)
+
+Credit risk and its goal can be defined in a more formal way (BCBS and BIS 2000):
+
+Credit risk is most simply defined as the potential that a bank borrower or counterparty will fail to meet its obligations in accordance with agreed terms. The goal of credit risk management is to maximise a bank’s risk-adjusted rate of return by maintaining credit risk exposure within acceptable parameters.
+
+### Estimating the Credit Risk
+
+Aside from the probability of default, credit risk has three defining characteristics:
+
+* exposure: refers to a party that may possibly default or suffer an adverse change in its ability to perform
+* likelihood: the likelihood that this party will default on its obligations
+* recovery rate: how much can be retrieved if a default takes place
+
+The BCBS put forth the global financial credit management standards, which are known as the Basel Accord. There are currently three Basel Accords. The most distinctive rule set by Basel I in 1988 was the requirement to hold capital equating to at least 8% of risk-weighted assets.
+
+Basel I includes the very first capital measurement system, which was created following the onset of the Latin American debt crisis. In Basel I, assets are classified as follows:
+
+* 0% for risk-free assets
+
+* 20% for loans to other banks
+
+* 50% for residential mortgages
+
+* 100% for corporate debt
+
+In 1999, Basel II issued a revision to Basel I based on three main pillars:
+
+* minimum capital requirements, which sought to develop and expand the standardized rules set out in the 1988 Accord
+* supervisory review of an institution’s capital adequacy and internal assessment process
+* effective use of disclosure as a lever to strengthen market discipline and encourage sound banking practices
+
+The last accord, Basel III in 2010, was inevitable. as the 2007–2008 mortgage crisis heightened. It introduced a new set of measures to further strengthened liquidity and poor governance practices. For instance, equity requirements were introduced to prevent a serial failure in the financial system, known as domino effect, during times of financial turbulence and crises. Accordingly, Basel III requires the financial ratios for banks to be:
+
+<img width="722" alt="image" src="https://github.com/user-attachments/assets/eacd4066-6dd3-4e07-a658-a6f2217b2019">
+
+Basel II suggests banks implement either a standardized approach or an internal ratings–based (IRB) approach to estimate the credit risk.
+
+The key params of the IRB approach are:
+
+Expected Loss = EAD x LGD x PD
+
+The most important and challenging part of estimating credit risk is to model the probability of default which is adressed in this chapter. 
+
+### Risk Bucketing
+
+Risk bucketing is nothing but grouping borrowers with similar creditworthiness. The behind-the-scenes story of risk bucketing is to obtain homogenous groups or clusters so that we can better estimate the credit risk. Treating different risky borrowers equally may result in poor predictions because the model cannot capture entirely different characteristics of the data at once. Thus, by dividing the borrowers into different groups based on riskiness, risk bucketing enables us to make accurate predictions.
+
+A common way to do this is through K-means clustering. And there are multiple methods to determine the right k:
+
+* the elbow method
+* the Silhouette score
+* Calinski-Harabasz (CH)
+
+### Probability of Default Estimation with Logistic Regression
+
+For this, a German Credit Dataset is used from [Kaggle](https://www.kaggle.com/datasets/kabure/german-credit-data-with-risk). 
+
+```python
+from sklearn.preprocessing import StandardScaler
+from sklearn.cluster import KMeans
+import numpy as np
+
+scaler = StandardScaler()
+scaled_credit = scaler.fit_transform(numerical_credit)
+
+kmeans = KMeans(n_clusters=2) # 2 found after using the above mentioned methods
+clusters = kmeans.fit_predict(scaled_credit)
+```
+
+We can see the 2 clusters plotted for different features
+
+![image](https://github.com/user-attachments/assets/f57ee2d9-f55d-4059-b8dc-cdfe312278c3)
+
+Next, the dataset is prepared by adding a cluster target (0 or 1) to the dataset:
+
+```python
+clusters, counts = np.unique(kmeans.labels_, return_counts=True)
+
+cluster_dict = {}
+for i in range(len(clusters)):
+    cluster_dict[i] = scaled_credit[np.where(kmeans.labels_==i)]
+
+credit['clusters'] = pd.DataFrame(kmeans.labels_)
+
+df_scaled = pd.DataFrame(scaled_credit)
+df_scaled['clusters'] = credit['clusters']
+
+df_scaled['Risk'] = credit['Risk']
+df_scaled.columns = ['Age', 'Job', 'Credit amount',
+                     'Duration', 'Clusters', 'Risk']
+
+df_scaled[df_scaled.Clusters == 0]['Risk'].value_counts()
+
+# Risk	count
+# good	139
+# bad	  110
+
+df_scaled[df_scaled.Clusters == 1]['Risk'].value_counts()
+
+# Risk	count
+# good	561
+# bad	  190
+```
+
+<img width="581" alt="image" src="https://github.com/user-attachments/assets/8abfa8a0-8d9c-4aca-b8e5-b3ce1e4885fb">
+
+there is an imbalance distribution across risk level in the first cluster, whereas the frequency of good and bad risk levels are more balanced
+
+Before dealing with the imbalance, split the data:
+
+```python
+from sklearn.model_selection import train_test_split
+
+df_scaled['Risk'] = df_scaled['Risk'].replace({'good': 1, 'bad': 0})
+
+X = df_scaled.drop('Risk', axis=1)
+y = df_scaled.loc[:, ['Risk', 'Clusters']]
+
+X_train, X_test, y_train, y_test = train_test_split(X, y,
+                                                    test_size=0.2,
+                                                    random_state=42)
+
+first_cluster_train = X_train[X_train.Clusters == 0].iloc[:, :-1]
+second_cluster_train = X_train[X_train.Clusters == 1].iloc[:, :-1]
+```
+
+To deal with the imbalance a combined method between SMOTE and ENN is used (SMOTEENN)
+
+```python
+import statsmodels.api as sm
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import roc_auc_score, roc_curve
+from imblearn.combine import SMOTEENN
+import warnings
+warnings.filterwarnings('ignore')
+
+X_train1 = first_cluster_train
+y_train1 = y_train[y_train.Clusters == 0]['Risk']
+smote = SMOTEENN(random_state = 2)
+X_train1, y_train1 = smote.fit_resample(X_train1, y_train1.ravel())
+```
+
+First, a model for the 1st cluster:
+
+```
+logit = sm.Logit(y_train1, X_train1)
+logit_fit1 = logit.fit()
+print(logit_fit1.summary())
+```
+
+Result from `logit_fit1`
+
+```
+Optimization terminated successfully.
+         Current function value: 0.646649
+         Iterations 5
+                           Logit Regression Results                           
+==============================================================================
+Dep. Variable:                      y   No. Observations:                   61
+Model:                          Logit   Df Residuals:                       57
+Method:                           MLE   Df Model:                            3
+Date:                Sat, 19 Oct 2024   Pseudo R-squ.:                 0.06545
+Time:                        09:54:50   Log-Likelihood:                -39.446
+converged:                       True   LL-Null:                       -42.208
+Covariance Type:            nonrobust   LLR p-value:                    0.1371
+=================================================================================
+                    coef    std err          z      P>|z|      [0.025      0.975]
+---------------------------------------------------------------------------------
+Age               0.5274      0.338      1.561      0.118      -0.135       1.190
+Job              -0.1123      0.376     -0.299      0.765      -0.849       0.624
+Credit amount    -0.2877      0.247     -1.165      0.244      -0.772       0.196
+Duration          0.5352      0.306      1.751      0.080      -0.064       1.134
+=================================================================================
+```
+
+The AUC-ROC for the 1st cluster model is:
+
+![image](https://github.com/user-attachments/assets/4305d46e-f72e-4828-b2f4-235d2dd4f5ec)
+
+The 2nd model summary from `logit_fit2`:
+
+```
+Optimization terminated successfully.
+         Current function value: 0.591220
+         Iterations 5
+                           Logit Regression Results                           
+==============================================================================
+Dep. Variable:                   Risk   No. Observations:                  590
+Model:                          Logit   Df Residuals:                      586
+Method:                           MLE   Df Model:                            3
+Date:                Sat, 19 Oct 2024   Pseudo R-squ.:                -0.04282
+Time:                        09:56:08   Log-Likelihood:                -348.82
+converged:                       True   LL-Null:                       -334.50
+Covariance Type:            nonrobust   LLR p-value:                     1.000
+=================================================================================
+                    coef    std err          z      P>|z|      [0.025      0.975]
+---------------------------------------------------------------------------------
+Age               0.4019      0.101      3.992      0.000       0.205       0.599
+Job              -0.0639      0.098     -0.652      0.515      -0.256       0.128
+Credit amount    -0.6215      0.187     -3.322      0.001      -0.988      -0.255
+Duration         -0.9486      0.165     -5.760      0.000      -1.271      -0.626
+=================================================================================
+```
+
+And its AUC-ROC:
+
+![image](https://github.com/user-attachments/assets/e08875ff-6771-4e06-9d78-a2c2f37495d4)
+
+These particular models for both clusters are not that good as they go alone the diagonal of their respective plots.
+
+Below are different methods to compare with the above one.
+
+#### Probability of Default Estimation with Support Vector Machines
+
+```python
+from sklearn.svm import SVC
+from sklearn.experimental import enable_halving_search_cv
+from sklearn.model_selection import HalvingRandomSearchCV
+import time
+
+param_svc = {'gamma': [1e-6, 1e-2],
+             'C':[0.001,.09,1,5,10],
+             'kernel':('linear','rbf')}
+
+svc = SVC(class_weight='balanced')
+halve_SVC = HalvingRandomSearchCV(svc, param_svc, 
+                                  scoring = 'roc_auc', n_jobs=-1)
+halve_SVC.fit(X_train1, y_train1)
+print('Best hyperparameters for first cluster in SVC {} with {}'.
+      format(halve_SVC.best_score_, halve_SVC.best_params_))
+
+# Best hyperparameters for first cluster in SVC 0.6233333333333333 with {'kernel': 'linear', 'gamma': 1e-06, 'C': 10}
+
+y_pred_SVC1 = halve_SVC.predict(X_test1)
+print('The ROC AUC score of SVC for first cluster is {:.4f}'.
+      format(roc_auc_score(y_test1, y_pred_SVC1)))
+
+# The ROC AUC score of SVC for first cluster is 0.4079
+
+halve_SVC.fit(X_train2, y_train2)
+print('Best hyperparameters for second cluster in SVC {} with {}'.
+      format(halve_SVC.best_score_, halve_SVC.best_params_))
+
+# Best hyperparameters for second cluster in SVC 0.6388292598206392 with {'kernel': 'rbf', 'gamma': 0.01, 'C': 0.001}
+
+y_pred_SVC2 = halve_SVC.predict(X_test2)
+print('The ROC AUC score of SVC for first cluster is {:.4f}'.
+      format(roc_auc_score(y_test2, y_pred_SVC2)))
+
+# The ROC AUC score of SVC for first cluster is 0.5000
+```
+
+The AUC performance criteria indicates that the predictive performance of SVC is slightly below that of logistic regression
+
+#### Probability of Default Estimation with Random Forest
+
+```python
+from sklearn.ensemble import RandomForestClassifier
+
+rfc = RandomForestClassifier(random_state=42)
+
+param_rfc = {'n_estimators': [100, 300],
+    'criterion' :['gini', 'entropy'],
+    'max_features': ['auto', 'sqrt', 'log2'],
+    'max_depth' : [3, 4, 5, 6],
+    'min_samples_split':[5, 10]}
+
+halve_RF = HalvingRandomSearchCV(rfc, param_rfc,
+                                 scoring = 'roc_auc', n_jobs=-1)
+halve_RF.fit(X_train1, y_train1)
+print('Best hyperparameters for first cluster in RF {} with {}'.
+      format(halve_RF.best_score_, halve_RF.best_params_))
+
+# Best hyperparameters for first cluster in RF 0.9133333333333333 with {'n_estimators': 100, 'min_samples_split': 5, 'max_features': 'sqrt', 'max_depth': 3, 'criterion': 'entropy'}
+
+y_pred_RF1 = halve_RF.predict(X_test1)
+print('The ROC AUC score of RF for first cluster is {:.4f}'.
+      format(roc_auc_score(y_test1, y_pred_RF1)))
+
+# The ROC AUC score of RF for first cluster is 0.5829
+
+halve_RF.fit(X_train2, y_train2)
+print('Best hyperparameters for second cluster in RF {} with {}'.
+      format(halve_RF.best_score_, halve_RF.best_params_))
+
+# Best hyperparameters for second cluster in RF 0.6535338960049029 with {'n_estimators': 100, 'min_samples_split': 10, 'max_features': 'sqrt', 'max_depth': 4, 'criterion': 'entropy'}
+
+y_pred_RF2 = halve_RF.predict(X_test2)
+print('The ROC AUC score of RF for first cluster is {:.4f}'.
+      format(roc_auc_score(y_test2, y_pred_RF2)))
+
+# The ROC AUC score of RF for first cluster is 0.5125
+```
+
+It does not go into getting a model with good performance. Just an overview of the above + a neural net and a Keras model. [Here](https://github.com/abdullahkarasan/mlfrm/blob/main/codes/chp_6.ipynb) is the full code
+
+## [Common pitfalls in the interpretation of coefficients of linear models](https://scikit-learn.org/stable/auto_examples/inspection/plot_linear_model_coefficient_interpretation.html#)
+
+I was saving this to read on stream, but I wanted to read it casually on my own today. Definitely a resource that is awesome for beginners and something I wish I read earlier in my learning journey haha
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f796ef59e3e208edaa1c907851fc90b8b3f49de2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 20 Oct 2024 01:02:18 +0900
Subject: [PATCH 469/638] update 292

---
 _posts/2024-10-19-day-292.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-19-day-292.md b/_posts/2024-10-19-day-292.md
index b891c8848..853c252cf 100644
--- a/_posts/2024-10-19-day-292.md
+++ b/_posts/2024-10-19-day-292.md
@@ -1087,7 +1087,7 @@ print('The ROC AUC score of RF for first cluster is {:.4f}'.
 
 It does not go into getting a model with good performance. Just an overview of the above + a neural net and a Keras model. [Here](https://github.com/abdullahkarasan/mlfrm/blob/main/codes/chp_6.ipynb) is the full code
 
-## [Common pitfalls in the interpretation of coefficients of linear models](https://scikit-learn.org/stable/auto_examples/inspection/plot_linear_model_coefficient_interpretation.html#)
+## [Common pitfalls in the interpretation of coefficients of linear models](https://scikit-learn.org/stable/auto_examples/inspection/plot_linear_model_coefficient_interpretation.html#) and [Failure of Machine Learning to infer causal effects](https://scikit-learn.org/stable/auto_examples/inspection/plot_causal_interpretation.html#)
 
 I was saving this to read on stream, but I wanted to read it casually on my own today. Definitely a resource that is awesome for beginners and something I wish I read earlier in my learning journey haha
 

From 5a9577d9a602969ac4164e3bf9a3b4a14ff5373e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 21 Oct 2024 00:34:41 +0900
Subject: [PATCH 470/638] Create 2024-10-20-day-293.md

---
 _posts/2024-10-20-day-293.md | 308 +++++++++++++++++++++++++++++++++++
 1 file changed, 308 insertions(+)
 create mode 100644 _posts/2024-10-20-day-293.md

diff --git a/_posts/2024-10-20-day-293.md b/_posts/2024-10-20-day-293.md
new file mode 100644
index 000000000..d0904da8b
--- /dev/null
+++ b/_posts/2024-10-20-day-293.md
@@ -0,0 +1,308 @@
+---
+layout: post
+title: (Day 293) Finishing the book + reading sklearn docs for fun
+categories: [traditional-machine-learning,theory]
+---
+
+# Hello :) Today is Day 293!
+A quick summary of today:
+* finished ML for Financial Risk management
+* read some more sklearn docs for fun
+
+## ML for fin. risk management Chapter 7 - Liquidity Modeling
+
+Liquidity risk has long been neglected in financial modeling, despite its crucial role in systemic financial risk, as demonstrated during the 2007–2008 mortgage crisis. Liquidity risk arises from incomplete markets and asymmetric information, leading to issues like moral hazard and adverse selection. Including liquidity in financial models enhances accuracy, as it impacts both asset returns and uncertainty levels. Liquidity is a multifaceted concept, with four key dimensions: 
+
+* tightness (transaction cost)
+* immediacy (speed of trading)
+* depth (presence of buyers and sellers)
+* resiliency (ability to recover from price imbalances)
+
+Modeling liquidity is challenging due to its multidimensionality, and different liquidity measures are used to capture these aspects.
+
+### Liquidity Measures
+
+#### Volume-Based Liquidity Measures
+
+Large orders are fulfilled when the market has depth, meaning a deep market can accommodate substantial orders. This provides insight into the market's health, as a lack of depth leads to order imbalances and discontinuities. Volume-based liquidity measures can help differentiate between liquid and illiquid assets based on market depth. Additionally, these measures are closely related to the bid-ask spread: a wide spread indicates low volume, while a narrow spread suggests higher volume.
+
+To properly represent the depth dimension of liquidity, the following volume-based measures are used:
+
+**Liquidity ratio**
+
+Measures the extent to which volume is required to induce a price change of 1%. Again, it measures how much trading volume is needed to cause a 1% change in the price of an asset. A higher ratio indicates greater liquidity, meaning that a large volume of trades can occur with only a small change in price. On the other hand, if a small volume of trades causes a price change, the asset is considered illiquid. This ratio primarily focuses on the relationship between trading volume and price changes, rather than the time it takes to execute trades or the costs involved in doing so.
+
+**Hui-Heubel ratio**
+
+The Hui-Heubel ratio is a measure of market depth, specifically designed to assess the liquidity of individual stocks rather than entire portfolios. It compares the percentage change in a stock’s price (based on its maximum and minimum prices over a set period) to the volume of trades relative to the stock's market capitalization. This ratio helps evaluate how much trading activity is needed to move the stock’s price. One key feature of the Hui-Heubel ratio is its focus on individual stocks, making it versatile for liquidity analysis. Although it can use the bid-ask spread instead of price changes, low volatility in the bid-ask spread may lead to an underestimation of liquidity.
+
+**Turnover ratio**
+
+The turnover ratio is a commonly used proxy for liquidity, reflecting how frequently a stock is traded. It measures the ratio of trading volume (the number of shares traded) to the number of shares outstanding over a specific period. A high turnover ratio suggests a high level of liquidity, as it indicates frequent trading of the stock. The inclusion of shares outstanding in the calculation provides a more refined measure of liquidity, as it accounts for the stock's availability in the market. Essentially, the turnover ratio captures both the trading activity and the stock's market presence.
+
+#### Transaction Cost–Based Liquidity Measures
+
+These measures focus on the costs involved in executing trades, which include both explicit costs (like taxes and brokerage fees) and implicit costs (such as the bid-ask spread and execution timing). These costs are crucial in determining market tightness (the ability to trade with low costs) and immediacy (the speed of executing trades). High transaction costs discourage trading, leading to a fragmented and shallow market, while low costs promote a more active, centralized market. The bid-ask spread is a commonly used proxy for transaction costs and liquidity, indicating how easily an asset can be converted to cash. Methods like quoted, effective, and realized spreads are used to calculate this spread, though actual trade conditions may sometimes differ from observed spreads.
+
+**Percentage quoted and effective bid-ask spreads**
+
+Percentage quoted and effective bid-ask spreads are key measures of transaction costs in trading. 
+
+- Percentage quoted spread: this measures the cost of completing a trade by calculating the difference between the ask and bid prices of a stock, scaled as a percentage. It shows how much more a buyer must pay (ask price) than what a seller would receive (bid price), indicating the cost of executing a trade.
+  
+- Percentage effective spread: this spread measures the difference between the actual trading price and the midpoint of the bid-ask prices (considered the true value of the stock). It accounts for trades that occur inside or outside the quoted prices, offering a more accurate reflection of trading costs. The percentage effective half spread is often used to calculate this spread on a percentage basis.
+
+Both spreads provide insight into trading costs, with the effective spread giving a more realistic measure when trades deviate from quoted prices.
+
+**Roll’s spread estimate**
+
+This estimate is a method to infer the bid-ask spread using only price data. It assumes that price changes caused by the spread create negative autocorrelation, as prices move between the bid and ask. By calculating the covariance of consecutive price changes, Roll’s model estimates the spread. The larger the negative covariance, the wider the spread. While useful, it doesn’t account for information-driven price changes and works best in markets where transaction costs dominate price movements.
+
+**The Corwin-Schultz spread**
+
+This is a method for estimating the bid-ask spread using only the daily high and low prices of a stock. The core idea is that the highest price on a given day is typically driven by buyers, while the lowest price is driven by sellers, so the price difference reflects both the stock’s volatility and the bid-ask spread.
+
+The approach works by comparing price ranges over two consecutive days. The sum of the daily price ranges for two days reflects both the spread and the volatility over that period, while the price range for the two-day period captures only the volatility and one bid-ask spread. This method offers an easy and intuitive way to estimate the spread without needing more detailed trade data.
+
+#### Price Impact–Based Liquidity Measures
+
+Such measures assess how sensitive asset prices are to changes in trading volume and turnover ratios. Resiliency, in this context, refers to the market's ability to respond to new orders and correct imbalances. If a market effectively adjusts its prices in response to changes in volume or turnover, it is considered resilient. Therefore, a high price adjustment in relation to changes in volume or turnover indicates greater market resiliency.
+
+**Amihud illiquidity**
+
+It measures the sensitivity of the return to trading volume. More concretely, it gives us a sense about a change in absolute return as trading volume changes by $1. This measure is well known by researchers and practitioners. 
+
+The Amihud measure has two advantages over many other liquidity measures. First, the Amihud measure has a simple construction that uses the absolute value of the daily return-to-volume ratio to capture price impact. Second, the measure has a strong positive relation with expected stock return
+
+**The price impact ratio**
+
+The price impact ratio, known as Return-to-Turnover (RtoTR), was developed by Florackis, Andros, and Alexandros to address shortcomings in the Amihud illiquidity ratio. The authors identified two main drawbacks of Amihud’s measure: it is not comparable across stocks with varying market capitalizations, and it does not consider the investor's holding horizon.
+
+To overcome these limitations, the RtoTR replaces the volume ratio in Amihud's model with the turnover ratio, which captures trading frequency. This adjustment allows the new measure to better reflect the liquidity of stocks by considering how actively they are traded. The RtoTR retains the other components of Amihud’s illiquidity measure, providing a more comprehensive view of liquidity dynamics.
+
+**Coefficient of elasticity of trading**
+
+CET is a liquidity measure proposed to remedy the shortcomings of time-related liquidity measures such as number of trades and orders per unit of time. These measures are adopted to assess the extent to which market immediacy affects the liquidity level.
+
+Market immediacy and CET go hand in hand as it measures the price elasticity of trading volume and if price is responsive (i.e. elastic) to the trading volume, that amounts to a greater level of market immediacy.
+
+#### Market Impact-Based Liquidity Measures
+
+Price movements from the market as a whole don't provide the same information as those from individual stocks, making it necessary to capture price changes accurately.
+
+To achieve this, the Capital Asset Pricing Model (CAPM) is used to separate systematic risk from unsystematic risk. Systematic risk is represented by the slope coefficient in CAPM, while unsystematic risk is linked to individual stocks when market risk is eliminated.
+
+The daily return of a stock is modeled, and the idiosyncratic (unsystematic) risk is calculated as a residual. These residuals are then regressed against the stock's volatility, with the result indicating the stock's liquidity level or idiosyncratic risk. The formula involves the squared residuals, daily percentage change in trading volume (volatility), and a residual term.
+
+### Gaussian Mixture Model
+
+Liquidity in a high-volatility period cannot be modeled in the same way as a low-volatility period. In a similar vein, given the depth of the market, we need to focus on these different aspects of liquidity. GMM provides us with a tool to address this problem.
+
+Long story short, if a market is experiencing a boom period, which coincides with high volatility, volume and transaction cost–based measures would be good choices, and if a market ends up with price discontinuity, price-based measures would be the optimal choice.
+
+After applying GMM, we can use PCA for dimensionality reduction. In the context of GMM-PCA, loading analysis serves as a critical tool for interpreting the underlying structure of the data. GMM-PCA combines the benefits of both GMM, which captures the probabilistic distribution of the data, and PCA, which reduces dimensionality while retaining variance. Through loading analysis, you can identify which liquidity measures, such as turnover ratio, percent quoted bid ask, and others, are most influential in shaping the principal components derived from the GMM-PCA. This understanding is essential for distinguishing between different liquidity states, allowing for more informed decision-making and deeper insights into how various liquidity ratios interact within different market conditions. Ultimately, loading analysis enhances the interpretability of complex financial data, facilitating targeted strategies in liquidity management and risk assessment.
+
+### Gaussian Mixture Copula Model
+
+Given the complexity and sophistication of financial markets, it is not possible to suggest one-size-fits-all risk models. Thus, financial institutions develop their own models for credit, liquidity, market, and operational risks so that they can manage the risks they face more efficiently and realistically. However, one of the biggest challenges that these financial institutions come across is the correlation, also known as joint distribution, of the risk.
+
+The task of modeling 'extreme market phases' leads us to the concept of joint distribution by which we are allowed to model multiple risks with a single distribution. A model disregarding the interaction of risks is destined for failure. In this respect, an intuitive yet simple approach is proposed: copulas.
+
+Copula is a function that maps marginal distribution of individual risks to multivariate distribution, resulting in a joint distribution of many standard uniform random variables. If we are working with a known distribution, such as normal distribution, it is easy to model joint distribution of variables, known as bivariate normal. However, the challenge here is to define the correlation structure between these two variables, and this is the point at which copulas come in.
+
+## Chapter 8 - Modelling Operational Risk
+
+Op. risk is ambiguous. This arises from varied sources and can lead to significant financial losses for institutions. Operational risk includes both direct and indirect losses caused by failures in internal processes, people, systems, or external events.
+
+Direct losses can result from legal liabilities, theft, compliance issues, or business interruptions, while indirect losses are linked to opportunity costs due to poor decision-making. Financial institutions often allocate funds to cover such losses, but determining the appropriate amount is challenging—overfunding leads to idle resources, while underfunding creates liquidity risks.
+
+This chapter will focus on fraud risk, one of the most pervasive forms of operational risk. Fraud is intentional deception for gain, and it can be either internal (committed by insiders) or external (by third parties). Several factors increase the likelihood of fraud, including globalization (which complicates operations), poor risk management (which fosters misleading reporting and unauthorized trading), and economic pressures (which tempt individuals and institutions into illegal activities).
+
+Fraud can cause financial losses and damage to an institution's reputation, as demonstrated by high-profile cases like Barings Bank and Enron. The chapter concludes by introducing a machine learning (ML) model to detect fraud, employing both supervised and unsupervised learning techniques to improve detection and prevention efforts.
+
+(The dataset used here is the dataset I used in the [KB fraud transaction project](https://github.com/divakaivan/kb_project))
+
+### Supervised Learning Modeling for Fraud Examination
+
+It shows example models with XGB, RFC, LogReg, and Decision tree. [Here](https://github.com/abdullahkarasan/mlfrm/blob/main/codes/chp_8.ipynb) is the full code
+
+#### Cost-Based Fraud Examination
+
+In fraud, a false negative is significantly more costly than a false positive. So we can come up with a cost matrix like:
+
+|                      | Positive Prediction | Negative Prediction |
+|----------------------|---------------------|---------------------|
+| **Actual Positive**  | 2   | Transaction Amount |
+| **Actual Negative**  | 2  | 0    |
+
+Using this we can calculate the cost of each model (and this would be called a cost-sensitive classifier since we take cost into account)
+
+Below is a table made from the confusion matrices of the 4 created models. 
+
+| Model                | Total Cost | F1 Score |
+|----------------------|------------|----------|
+| Logistic Regression   | 5995       | 0.79     |
+| Decision Tree         | 5351       | 0.96     |
+| Random Forest         | 5413       | 0.87     |
+| XGBoost              | 5371       | 0.97     |
+
+#### Saving Score
+
+There are different metrics that can be used in cost improvement, and saving score is one of them.
+
+The cost formula is:
+
+<img width="689" alt="image" src="https://github.com/user-attachments/assets/5bba104e-ceba-4c25-9e81-20f0d90ccc3b">
+
+The saving formula is:
+
+<img width="717" alt="image" src="https://github.com/user-attachments/assets/139b52a5-8ad3-4172-8ca7-15c48220f657">
+
+| Model                | Saving Score | F1 Score |
+|----------------------|--------------|----------|
+| Logistic Regression   | -0.5602      | 0.0000   |
+| Decision Tree         | 0.6557       | 0.7383   |
+| Random Forest         | 0.4789       | 0.7068   |
+
+In this case we can see that the Decision Tree model is the best
+
+#### Cost-Sensitive Modelling
+
+This type of modelling focuses on how varying costs of misclassification impact fraud detection, particularly through saving scores. Traditional fraud models often overlook the variability in these costs, assuming uniform costs for all classifications, which is not accurate in real-world scenarios
+
+| Model                | Saving Score | F1 Score |
+|----------------------|--------------|----------|
+| Logistic Regression   | -0.5906      | 0.0000   |
+| Decision Tree         | 0.8414       | 0.3281   |
+| Random Forest         | 0.8913       | 0.4012   |
+
+The best and the worst saving scores are obtained in random forest and logistic regression, respectively. This confirms two important facts: first, it implies that random forest has a low number of inaccurate observations, and second, that those inaccurate observations are less costly. To be precise, modeling fraud with random forest generates a very low number of false negatives, which is the denominator of the saving score formula
+
+#### Bayesian Minimum Risk
+
+Bayesian decision can also be used to model fraud taking into account the cost sensitivity. The Bayesian minimum risk method rests on a decision process using different costs (or loss) and probabilities. Mathematically, if the transaction is predicted to be fraud, the overall risk is defined as follows:
+
+<img width="415" alt="image" src="https://github.com/user-attachments/assets/b3d7066e-0501-4837-b9a3-332710908cbf">
+
+On the other hand, if a transaction is predicted to be legitimate, then the overall risk turns out to be:
+
+<img width="404" alt="image" src="https://github.com/user-attachments/assets/f7bc8eb4-0d87-4630-8adc-370e56caa50d">
+
+Alternatively, the Bayesian minimum risk formula can be interpreted as:
+
+<img width="373" alt="image" src="https://github.com/user-attachments/assets/25c4707b-9e8d-417e-9c65-ea1c059fef8e">
+
+with admin is administrative cost and amt is the transaction amount. With that being said, the transaction is labeled as fraud if:
+
+<img width="423" alt="image" src="https://github.com/user-attachments/assets/d8f878af-3cff-441f-b7b7-6413547027e7">
+
+This is the results table when the Bayesian Minimum Risk model is applied:
+
+| Model                | Saving Score | F1 Score |
+|----------------------|--------------|----------|
+| Logistic Regression   | 0.8064       | 0.1709   |
+| Decision Tree         | 0.7343       | 0.6381   |
+| Random Forest         | 0.9624       | 0.4367   |
+
+Here is a plot comparing the base model, cost-sensitive method, and Bayesian min. risk method:
+
+![image](https://github.com/user-attachments/assets/b03e5611-3613-4ba9-af16-acd3566a27af)
+
+### Unsupervised Learning Modeling for Fraud Examination
+
+The most prominent advantage of this method over the supervised model is that there is no need to apply a sampling procedure to fix the imbalanced-data problem. Unsupervised models, by their nature, do not require any prior knowledge about the data
+
+#### Self-Organizing Map (SOM)
+
+A Self-Organizing Map (SOM) is an unsupervised learning model that excels at clustering and visualizing high-dimensional data. Unlike supervised models, SOMs do not require labeled data, making them particularly useful for fraud examination where labeled examples may be scarce. SOMs project data onto a lower-dimensional grid while preserving the topological relationships, allowing for easier visualization of patterns. This ability to cluster similar data points helps identify anomalies that may indicate fraud. Additionally, SOMs can adapt to new data without retraining, making them effective for continuous monitoring of transactions. SOMs are valuable for detecting hidden patterns in transaction data without the challenges of imbalanced datasets.
+
+#### Autoencoders
+
+Autoencoders are unsupervised deep learning models designed to transform inputs into outputs through a hidden layer, comprising two main components: an encoder and a decoder. The encoder extracts features from the input data, while the decoder reconstructs the original input. The primary goal of an autoencoder is not merely to copy the input but to learn a lower-dimensional representation that captures the essential characteristics of the data.
+
+Types of Autoencoders:
+
+1. undercomplete: this basic type has a hidden layer with fewer neurons than the input data, aiming to capture latent attributes by minimizing the reconstruction loss. They face a bias-variance trade-off, balancing accurate reconstruction with a low-dimensional representation
+
+2. sparse: these address the bias-variance trade-off by imposing sparsity on the reconstruction error. Regularization techniques, such as L_1 regularization or Kullback-Leibler (KL) divergence, are used to encourage the model to represent the data in a sparse manner, effectively focusing on the most relevant features
+
+3. denoising: instead of minimizing the standard reconstruction error, denoising autoencoders add noise to the input data and learn to reconstruct the original data from the corrupted input. This approach helps the model generalize better by focusing on the underlying structure of the data, making it more robust to noise
+
+## Chapter 9 - A Corporate Governance Risk Measure: Stock Price Crash
+
+The link between corporate governance and risk measure has been established via stock price crash risk, which is referred to as the risk of a large negative individual stock return
+
+Detecting the determinants of stock price crashes is crucial for identifying the root causes of corporate governance issues, enabling companies to address problematic managerial areas and enhance performance and reputation. Improved governance reduces the risk of stock price crashes and can increase total revenue. Stock price crashes signal the strengths and weaknesses of corporate governance, defined by fairness, transparency, and accountability. 
+
+Fairness ensures equal treatment of shareholders, transparency involves timely communication about company events, and accountability relates to establishing a code of conduct that presents an accurate company position to shareholders. Agency costs arise from conflicts of interest between shareholders and management, where managers may prioritize their own power and wealth over shareholder value. 
+
+When managers withhold bad news, stock price crashes can occur once accumulated negative information is disclosed, leading to sudden declines in stock prices, known as crash risk. Opaque financial reporting exacerbates this issue by preventing timely corrective actions. Overall, a strong link exists between corporate governance and stock price crashes, highlighting the importance of effective governance practices. The chapter will first explore stock price measures and then discuss how these measures can be applied to detect crashes.
+
+### Stock Price Crash Measures
+
+Key measures discussed include Down-to-Up Volatility (DUVOL), Negative Coefficient of Skewness (NCSKEW), and the CRASH measure
+
+- DUVOL: this calculates crash risk based on the standard deviation of firm-specific weekly returns, distinguishing between "down" weeks (returns below the annual mean) and "up" weeks (returns above the annual mean). Higher DUVOL values indicate a greater risk of a crash
+
+- NCSKEW: this is derived from the negative third moment of daily returns, normalized by the standard deviation raised to the third power. Similar to DUVOL, higher NCSKEW values suggest an increased crash risk
+
+- CRASH: This measure is based on the distance of firm-specific weekly returns from the mean, assigning a value of 1 if returns fall more than 3.09 (or 3.2) standard deviations below the mean, indicating a potential crash
+
+These measures provide different perspectives on crash risk, paving the way for the introduction of machine learning-based approaches to enhance stock price crash detection
+
+### Minimum Covariance Determinant
+
+MCD is a method proposed to detect anomalies in a distribution with elliptically symmetric and unimodal datasets. Anomalies in stock returns are detected using the MCD estimator, and this becomes the dependent variable in the logistic panel regression by which we explore the root causes of crash risk.
+
+The MCD estimator provides a robust and consistent method in detecting outliers. This is important because outliers may have a huge effect on the multivariate analysis.
+
+The strength of the MCD comes in the form of its explainability, adjustability, low computational time requirement, and robustness:
+
+* explainability is the extent to which the algorithm behind a model can be explained. MCD assumes the data to be elliptically distributed and the outliers are computed by the Mahalanobis distance metric
+* adjustability stresses the importance of having a data-dependent model that is allowed to calibrate itself on a consistent basis so that structural change can be captured
+* low computational time refers to fast calculation of covariance matrix and avoids using an entire sample. Instead, MCD uses a half sample in which no outliers are included so that outlying observations do not skew MCD location or shape
+* robustness: using a half sample in MCD also ensures robustness, because it implies that the model is consistent under contamination
+
+### Variables used in analysis
+
+**Independent variables used in the analysis of stock price crash**
+
+| Variable                           | Explanation                                                                                                    |
+|------------------------------------|----------------------------------------------------------------------------------------------------------------|
+| Size (log_size)                   | Logarithm of total assets owned by the company.                                                              |
+| Receivables (rect)                | Accounts receivable/debtors.                                                                                  |
+| Property, plant, and equipment (ppegt) | Total property, plant, and equipment.                                                                          |
+| Average turnover (dturn)          | The average monthly turnover ratio in year t minus the average monthly turnover ratio in year t - 1. The turnover ratio is the monthly trading volume divided by the total number of shares outstanding. |
+| NCSKEW (ncskew)                   | The negative coefficient of skewness of firm-specific weekly returns in a year, calculated as the negative of the third moment of firm-specific weekly returns divided by the cubed standard deviation. |
+| Firm-specific return (residuals)  | The average of firm-specific weekly returns in a year.                                                        |
+| Return on asset (RoA)             | The returns on assets in a year, defined as the ratio of net income to total assets.                          |
+| Standard deviation (annual_std)    | The standard deviation of firm-specific weekly returns in a year.                                             |
+
+**Variables used for firm-specific sentiment**
+
+| Variable                              | Explanation                                                                                   |
+|---------------------------------------|-----------------------------------------------------------------------------------------------|
+| Price-to-earnings ratio (p/e)       | Market value per share divided by earnings per share.                                       |
+| Turnover rate (turnover_rate)        | Total number of shares traded divided by the average number of shares outstanding.           |
+| Equity share (equity_share)          | Represents common stock.                                                                      |
+| Closed-end fund discount (cefd)      | Refers to assets that raise a fixed amount of capital through an initial public offering.   |
+| Leverage (leverage)                  | The ratio of the sum of long-term debt and current liabilities to total assets.              |
+| Buying and selling volume (bsi)      | Buying volume is the number of shares associated with buying trades, and selling volume is for selling trades. |
+
+## Reading some more scikit-learn docs on my own (no stream)
+
+* [Cross decomposition](https://scikit-learn.org/stable/modules/cross_decomposition.html#)
+* [Compare cross decomposition methods](https://scikit-learn.org/stable/auto_examples/cross_decomposition/plot_compare_cross_decomposition.html#)
+* [PCR vs PLS](https://scikit-learn.org/stable/auto_examples/cross_decomposition/plot_pcr_vs_pls.html#)
+* [Gaussian processes](https://scikit-learn.org/stable/modules/gaussian_process.html#)
+* [Elliptic Envelope](https://scikit-learn.org/stable/modules/generated/sklearn.covariance.EllipticEnvelope.html)
+
+There are so many little gems in the sklearn docs. 
+
+I have some other docs that I will continue reading on tomorrow's stream.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 437d9c1c63fe0ce2465c602dad54887e67738d98 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 21 Oct 2024 23:43:24 +0900
Subject: [PATCH 471/638] Create 2024-10-21-day-294.md

---
 _posts/2024-10-21-day-294.md | 56 ++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)
 create mode 100644 _posts/2024-10-21-day-294.md

diff --git a/_posts/2024-10-21-day-294.md b/_posts/2024-10-21-day-294.md
new file mode 100644
index 000000000..9267f3c9a
--- /dev/null
+++ b/_posts/2024-10-21-day-294.md
@@ -0,0 +1,56 @@
+---
+layout: post
+title: (Day 294) Reading more of sklearn's docs
+categories: [traditional-machine-learning,theory]
+---
+
+# Hello :) Today is Day 294!
+A quick summary of today:
+* read the XGBoost and LightGMB papers
+* discovered more cool little modules from sklearn's docs
+
+## Read two papers
+
+* [XGBoost: A Scalable Tree Boosting System](https://arxiv.org/pdf/1603.02754)
+* [LightGBM: A Highly Efficient Gradient Boosting Decision Tree](https://papers.nips.cc/paper_files/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf)
+
+I decided to read the above papers as they were referenced [here (Ensembles: Gradient boosting, random forests, bagging, voting, stacking)](https://scikit-learn.org/stable/modules/ensemble.html#) in the scikit-learn docs post which introduces various ensemble methods. The XGBoost one I read just because of XGBoost's popularity, and LightGBM was used as motivation for creating sklearn's HistGradBoostClassifier(Regressor) model. 
+
+## [Computing with sklearn](https://scikit-learn.org/stable/computing.html)
+
+I will give a short summary of this as the ideas of more efficient computation in sklearn are something I have not explored before.
+
+Several factors affecting prediction latency are explored:
+
+* number of features: more features lead to increased memory consumption and processing time, resulting in slower prediction times
+* input data representation and sparsity: sparse data representation can significantly speed up predictions for sparse datasets by reducing the number of operations. However, dense representation can benefit from optimized BLAS operations, making it faster for datasets with low sparsity. The document recommends using sparse formats if the sparsity ratio is greater than 90%
+* model complexity: increasing complexity often leads to increased latency. For linear models, the impact on latency is minimal, but for non-linear models like SVM and ensemble methods, complexity significantly affects latency
+* feature extraction: in real-world applications, feature extraction can be much more time-consuming than the prediction itself
+
+Here are some tips and tricks for performance optimization
+
+* using optimized linear algebra libraries (BLAS/LAPACK): optimized libraries like Atlas, OpenBLAS, MKL, and Apple Accelerate can lead to significant speedups for models relying heavily on linear algebra operations
+* limiting working memory: by limiting temporary memory usage, you can avoid memory exhaustion during calculations
+* model compression: controlling model sparsity can reduce memory usage and latency, especially when combined with sparse input data
+* model reshaping: selecting only the relevant features used by the model can reduce memory overhead and processing time
+
+## [Isotonic Regression](https://scikit-learn.org/stable/auto_examples/miscellaneous/plot_isotonic_regression.html#isotonic-regression)
+
+I feel like I should provide *some* info on the read paper sometimes because I feel like just pasting the link is kind of lazy. Especially if most of what I have studied during the day is something I am already familiar with, have written about it multiple times in my blog and the original post explains it very well in a concise way.
+
+_The isotonic regression algorithm finds a non-decreasing approximation of a function while minimizing the mean squared error on the training data. The benefit of such a non-parametric model is that it does not assume any shape for the target function besides monotonicity._
+
+<img width="320" alt="image" src="https://github.com/user-attachments/assets/6338c8bc-4ce3-48cd-91ac-5ac23e373e83">
+
+## [Stream](https://youtu.be/CrkHhWS5wPk)
+
+Read through the below guides. Particularly more in-depth on the plethora of feature selection techniques offered by scikit-learn. 
+
+* [Feature selection](https://scikit-learn.org/stable/modules/feature_selection.html)
+* [Classification of text documents using sparse features](https://scikit-learn.org/stable/auto_examples/text/plot_document_classification_20newsgroups.html#)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 681196643562e6cd2074700bcd3d12767dbf935f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 22 Oct 2024 23:23:52 +0900
Subject: [PATCH 472/638] Create 2024-10-22-day-295.md

---
 _posts/2024-10-22-day-295.md | 270 +++++++++++++++++++++++++++++++++++
 1 file changed, 270 insertions(+)
 create mode 100644 _posts/2024-10-22-day-295.md

diff --git a/_posts/2024-10-22-day-295.md b/_posts/2024-10-22-day-295.md
new file mode 100644
index 000000000..b878eea90
--- /dev/null
+++ b/_posts/2024-10-22-day-295.md
@@ -0,0 +1,270 @@
+---
+layout: post
+title: (Day 295) Starting a Credit Risk Modelling course on Udemy
+categories: [traditional-machine-learning,theory,math,statistics,applying-knowledge]
+---
+
+# Hello :) Today is Day 295!
+A quick summary of today:
+* read about optimisation algorithms
+* read about financial calculus
+* started a course on udemy
+* streamed
+
+## [Optimization Algorithms](https://learning.oreilly.com/library/view/optimization-algorithms/9781633438835/) Chapter 1 - Introduction to search and optimization
+
+I found this book randomly and decided to check the 1st few chapters which cover introductory knowledge to optimisation. 
+
+### Why care about search and optimisation?
+
+_Search is the systematic examination of states, starting from an initial state and ending (hopefully) at the goal state. Optimization techniques are in reality search methods, where the goal is to find an optimal or a near-optimal state within the feasible search space. This feasible search space is a subset of the optimization problem space where all the problem’s constraints are satisfied._
+
+### Going from toy problems to the real world
+
+![image](https://github.com/user-attachments/assets/8a6e861e-14e5-43b1-9892-5ead7cbdbd3b)
+
+Examples of classic and real-world optimisation problems:
+
+![image](https://github.com/user-attachments/assets/81eed854-b4ec-401a-ab04-231b2bfa46bc)
+
+* design problems: situations when time is not as important as the quality of the solution and users are willing to wait to get optimal solutions
+* planning problems: neeed to be solved in a time span from a few seconds to a few miutes
+* control probmes: need to be solved repetitively and quickly, in a time span from a few milliseconds to a few seconds (usually optimality is traded in for speed gains)
+
+### Basic ingredients of optimisation problems 
+
+Optimisation refers to the practice of finding the 'best' solution to a given problem, where 'best' usually means satisfactory or acceptable, possibly subject to a given set of constraints. The solutions can be classified into feasible, optimal, and near-optimal:
+
+* feasible are solution that satisfy all the given constraints
+* optimal are both feasible and provide the best objective function value
+* near-optimal are feasible solutions that provide a superior objective function value but are not necessarily the best
+
+Assuming we have a minimisation problem, where the goal is to find the values of adecision variable that minimise a certain objective function, a search space may combine multiple global minima, strong local minima, and weak local minima:
+
+![image](https://github.com/user-attachments/assets/70807c12-7af2-40e1-9b20-2b3f93764778)
+
+Feasible solutions fall within the constraints of the problem. 
+
+* a global optimum is the best of a set of candidate solutions. Mathematically, if f(x) is the objective function, a point x* is the global min if for all x in the domain of f, f(x*) <= f(x)
+* a strong local min is a point where the function's value is <= the values of the function in a neighbourhood around that point but is higher than the global min. Mathematically, a point x* is a strong local min if there is a neighbourhood N of x* such that f(x*) < f(x) for all x in N with x =/= x*
+* a weak local min is a point where the function's value is <= to the function's values at neighbouring points, but there are sequences of points convering to this point for which the function's values strictly decrease. Mathematically, a point x* is a weak local min if there is a neighbourhood N of x* such that f(x*) <= f(x) for all x in N
+
+Optimization problems can generally be stated as follows. Find X which optimizes f, subject to a possible set of equality and inequality constraints:
+
+![image](https://github.com/user-attachments/assets/d87d2ed1-7eec-40d7-bbdd-c52d5159af79)
+
+There are 3 main components of optimisation problems:
+
+* decision variables: a set of unknowns or variables that affect the objective function’s value. These are the variables that define the possible solutions to an optimization problem. If X represents the unknowns, also referred to as the independent variables, then f(X) quantifies the quality of the candidate solution or feasible solution.
+
+* objective functions (also known as criterion, merit function, utility function, cost function): stands for the quantity to be optimised. Simple math operations like addition, substraction, multiplication, and division do not change the value of the optimal point. Without a loss of generality, optimization can be interpreted as the minimization of a value, since the maximization of a primal function f(x) can be just the minimization of a dual problem generated after applying mathematical operations on f(x). This means that if the primal function is a minimization problem, then the dual problem is a maximization problem (and vice versa). According to this duality aspect of optimization problems, a solution x*, which is the minimum for the primal minimization problem, is also, at the same time, the maximum for the dual maximization problem
+
+![image](https://github.com/user-attachments/assets/8798178b-240a-4be0-a414-fa2572430f14)
+
+If we have a single objective function to be optimised, the problem is called a _mono-objective optimisation problem_. The opposite is multi-objective optimisation. 
+
+* constraints
+
+Constrained optimisation problems have a set of equality and/or inequality constraints that restrict the values assigned to the decision variables. In addition, most problems have a set of boundary constraints, which define the domain of values for each variable. Also, constraints can be hard (must be satisfied) or soft (desirable to satisfy). Soft constraints can be modelled by incorporating a reward/penalty function as part of the obj. function. The function can reward solutions that satisfy the soft constraints and penalise if not. 
+
+Soft constrains can also be used to make the search algorithm more adaptive. For example, the severity of the penalty can be dynamically changed as the algorithm progresses, imposing less strict penalties at first to encourage exploration, but imposing more severe penalties near the end to generate a result largely bound by the constraint.
+
+### Well-structured problems (WSP) vs ill-structured problems (ISP)
+
+We can classify optimization problems based on their structure and the procedure that exists (or doesn’t exist) for solving them.
+
+**WSP**
+
+There are 6 key characteristics of WSPs. These include the presence of a clear criterion for testing proposed solutions, the existence of a problem space capable of representing the initial problem state and potential solutions, and the representation of attainable and considerable state changes within the problem space. Moreover, any knowledge acquired by the problem solver can be represented within these spaces, and if the problem involves interacting with the external world, the state changes reflect the laws governing the real world.
+
+**ISP**
+
+These are complex discrete or continuous problems without algorithmic solutions or general problem solvers. ISPs are characterised by one or more of the following characteristics:
+
+* a problem space with different views of the problems, unclear goals, multimodality, and a dynamic nature
+* a lack of exact mathematical models or a lack of well-proven algorithmic solutions
+* solutions that are contradictory, consequences that are difficult to predict, and risk that is difficult or impossible to calculate, resulting in a lack of clear evaluation criteria
+* considerable data imperfection in terms of uncertainty, partial observability, vagueness, incomplete information, ambiguity, or unpredictability that makes monitoring the execution of the solutions difficult and sometimes impossible
+* computational intractability
+
+**WSP, but ISP in practice**
+
+The traveling salesman problem (TSP) is an example of a problem that may be well-structured in principle, but in practice becomes ill-structured. This is because of the impractical amount of computational power required to solve the problem in real time.
+
+The book largely focuses on ISPs, and on WSPs that are ISPs in practice since WSPs tend to have a well-known solving algorithms that often provide trivial, step-by-step procedures, and most problems in the real world are ISPs. 
+
+Most of the algorithms explored in the book are derivative-free and stochastic; they use randomness in their parameters and decision processes. These algorithms are often well suited to solving ISPs, as the randomness of their initial states and operators allows the algorithms to escape local minima and find optimal or near-optimal solutions. In contrast, deterministic algorithms use well-defined and procedural paths to reach solutions and generally are not well suited for ISPs, as they either cannot work in unknown search spaces or are unable to return solutions in a reasonable amount of time. Moreover, most of the algorithms covered in this book are black-box solvers that deal with the optimization problem as a black box. This black box provides, for certain decision variable values, the corresponding values of the objective functions and constraint functions. Importantly, this approach eliminates the need to consider various properties of the objective and constraint functions, such as nonlinearity, differentiability, nonconvexity, monotonicity, discontinuities, or even stochastic noise.
+
+### Search algorithms and the search dilemma
+
+The feasible search space, we may find a few reasonably good neighboring solutions, and the question is whether we should exploit this region or keep exploring, looking for better solutions in other regions of the feasible search space.
+
+**Exploration** (or diversification) is the process of investigating new regions in the feasible search space with the hope of finding other promising solutions. On the other hand, **exploitation** (or intensification) is the process of directing the search agent to focus on an attractive region of the search space where good solutions have already been found.
+
+![image](https://github.com/user-attachments/assets/8c24750e-2f8d-4b4a-8458-c071afaea3e5)
+
+## Chapter 2 - A deeper look at search and optimization
+
+### Classifying optimisation problems
+
+The process of optimization involves selecting decision variables from a given feasible search space in such a way as to optimize (minimize or maximize) a given objective function or, in some cases, multiple objective functions.
+
+Optimization problems are characterized by three main components: decision variables or design vectors, objective functions or criteria to be optimized, and a set of hard and soft constraints to be satisfied. The nature of these three components, the permissible time allowed for solving the problem, and the expected quality of the solutions lead to different types of optimization problems
+
+![image](https://github.com/user-attachments/assets/59e4dd68-1965-4c84-b691-b403d1b5d18a)
+
+#### Number and type of decision variables
+
+Based on the number of decision variables, optimisation problems can be broadly grouped into univariate and multivariate problems. 
+
+Problem classification also varies according to the type of decision variables. A continuous problem involves continuous-valued variables, where xj ∈ R. In contrast, if xj ∈ Z, the problem is an integer or discrete optimization problem. A mixed-integer problem has both continuous-valued and integer-valued variables. For example, optimizing elevator speed and acceleration (continuous variables) and the sequence of picking up passengers (a discrete variable) is a mixed-integer problem. Problems where the solutions are sets, combinations, or permutations of integer-valued variables are referred to as combinatorial optimization problems.
+
+Types of problems:
+
+![image](https://github.com/user-attachments/assets/b885b142-c970-4388-9c03-db5a780743fd)
+
+A complexity class P represents all decision problems that can be solved in polynomial time by deterministic algorithms (i.e., algorithms that do not guess at a solution). The NP or nondeterministic polynomial problems are those whose solutions are hard to find but easy to verify and are solved by a nondeterministic algorithm in polynomial time. NP-complete problems are those that are both NP-hard and verifiable in polynomial time. Finally, a problem is NP-hard if it is at least as hard as the hardest problem in NP-complete. NP-hard problems are usually solved by approximation or heuristic solvers, as it is hard to find efficient exact algorithms to solve such problems.
+
+#### Landscape and number of objective functions
+
+An obj. function's landscape represents the distribution of the function's values in the feasible search space. In this landscape, we will fund the optimal solution or the global minima in the lowest valley, asuuming we are dealing with a minimisation problem, or at the highest peak in the case of a maximisation problem. 
+
+According to the landscape, if there is only one clear global optimal solution, the problem is unimodal (e.g. convex and concave functions). In a multimodal problem, more than one optimum exists. 
+
+The obj. function is called deceptive when the global min lies in a very narrow valley and there is also a strong local minimum with a wide basin of attraction, such that the value of this obj. function is close to the value of an obj. function at the global minimum. 
+
+![image](https://github.com/user-attachments/assets/f348f522-9ea0-406c-b7c2-0b926cccd76c)
+
+As mentioned before, there are mono- and multi-objective functions depending on the # of obj. functions. Also, if a problem does not have an explicit obj. function, it's called a constraint-satisfaction problem (CSP). The goal is such cases is to just find a solution that satisfies a given set of constraints.
+
+#### Constraints
+
+Constrained problems have hard or soft constraints for equality, inequality, or both. Hard constraints must be satisfied, while soft constraints are nice to satisfy (but are not mandatory). If there are no constraints to be considered, aside from the boundary constraints, the problem is an unconstrained optimization problem.
+
+If we have a bivariate mono-objective constrained optimisation problem. we can convert it to an unconstrained optimisation using the Lagrange multiplier.  The idea is to convert the constrained optimization problem defined by the objective function f(x,y) with an equality constraint g(x,y) into an unconstrained optimization problem using the Lagrangian function L(x,y,λ) = f(x,y) + λg(x,y). This function combines an objective function and constraints, enabling constrained optimization problems to be formulated as unconstrained problems through the use of Lagrange multipliers. To do so, we take the partial derivatives of the objective functions and the constraints, with respect to the decision variables x and y, to form the unconstrained optimization equations to be used by the SymPy solver
+
+![image](https://github.com/user-attachments/assets/0d42f177-3cbd-4cb4-94a7-efd19ddf68b3)
+
+#### Linearity of objective functions and constraints
+
+If all the obj. functions and associated constraint conditions are linear, the optimisation problem is categorised as a linear optimisation problem or linear programming problem (LPP or LP), where the goal is to find the optimal value of a linear function s.t. linear constraints. Blending problems are a typical application of mixed integer linear programming (MILP), where a number of ingredients are to be blended or mixed to obtain a product with certain characteristics or properties. 
+
+If one of the objective functions, or at least one of the constraints, is nonlinear, the problem is considered a nonlinear optimization problem or nonlinear programming problem (NLP), and it’s harder to solve than a linear problem.
+
+A special case of NLP, when the objective function is quadratic, is called quadratic programming (QP).
+
+#### Expected quality and permissible time for the solution
+
+Optimization problems can also be categorized according to the expected quality of the solutions and the time allowed to find the solutions
+
+![image](https://github.com/user-attachments/assets/7ba8ce7b-3a3f-4063-8c31-362b1de53f1b)
+
+### Classifying search and optimisation algorithms
+
+When searching, we try to examine different states to find a path from the start (initial) state to the goal state. Often, an optimization algorithm searches for an optimum solution by iteratively transforming a current state or a candidate solution into a new, hopefully better, solution. Search algorithms can be classified based on the way the search space is explored:
+
+* local search: uses only local information about the search space surrounding the current solution to produce new solutions. Since only local info is used, local search algorithms (also known as local optimisers) locate local optima (which may or may not be the global optima)
+* global search: uses more info about the search space to local a global optima
+
+Another classification distinguishes between deterministic and stochastic algorithms:
+
+* deterministic algorithms: they follow a rigorous procedure in their path, and both the values of their design variables and their functions are repeatable. From the same starting point, they will follow the same path, whether you run the program today or tomorrow. Examples include, but are not limited to, graphical methods, gradient and Hessian-based methods, penalty methods, gradient projection methods, and graph search methods. Graph search methods can be further subdivided into blind search methods (e.g. depth-first, breadth-first, or Dijkstra) and informed search methods (e.g. hill climbing, beam search, best-first, A*, or contraction hierarchies).
+
+* stochastic algorithms: explicitly use randomness in their params or decision-making process or both. or example, genetic algorithms use some random or pseudo-random numbers, resulting in individual paths that are not exactly repeatable. With stochastic algorithms, the time taken to obtain an optimal solution cannot be accurately estimated. Solutions do not always get better, and stochastic algorithms sometimes miss the opportunity to find optimal solutions. This behavior can be advantageous, however, because it can prevent them from becoming trapped in local optima. Examples of stochastic algorithms include tabu search, simulated annealing, genetic algorithms, differential evolution algorithms, particle swarm optimization, ant colony optimization, artificial bee colony, firefly algorithm, etc. Most statistical machine learning algorithms are stochastic because they make use of randomness during the learning stage and they make predictions during the inference stage with a certain level of uncertainty. Moreover, some machine learning models are, like people, unpredictable. Models trained using human behavior-based data as independent variables are more likely to be unpredictable than those trained using independent variables that strictly follow physical laws. Due to the uncertainty associated with machine learning predictions, machine learning–based algorithms used to solve optimization problems can be considered stochastic methods
+
+![image](https://github.com/user-attachments/assets/c4c7c4ed-b3ae-4fba-9878-245542745417)
+
+### Heuristics and metaheuristics
+
+Heuristics (also known as mental shortcuts or rules of thumb) are solution strategies, seeking methods, or rules that can facilitate finding acceptable (optimal or near-optimal) solutions to a complex problem in a practical time. Despite seeking near-optimal solutions at a reasonable computational cost, they cannot guarantee either feasibility or degree of optimality.
+
+Metaheuristic algorithms are often global optimizers that can be applied to different linear and nonlinear optimization problems with relatively few modifications for specific problems. These algorithms are often robust and can handle different problem sizes, problem instances, and random variables.
+
+The goal of metaheuristics is to efficiently explore the search space in order to find optimal or near-optimal solutions. Metaheuristics may incorporate mechanisms to achieve a trade-off between exploration (diversification) and exploitation (intensification) of the search space to avoid getting trapped in confined areas of the search space while also finding optimal or near-optimal solutions in a reasonable amount of time. 
+
+Different heuristic search strategies can be used to generate candidate solutions. These strategies include, but are not limited to, search by repeated solution construction (e.g. graph search and ant colony optimization), search by repeated solution modification (e.g. tabu search, simulated annealing, genetic algorithm, and particle swarm optimization), and search by repeated solution recombination (e.g. genetic algorithm and differential evolution).
+
+### Nature-inspired algorithms
+
+These algorithms mimic natural phenomena to solve complex, dynamic, and multi-objective problems. These algorithms draw inspiration from molecular dynamics, biological evolution, ethology, and neural networks. Key types include:
+
+- simulated annealing (from thermal processes)
+- evolutionary computing (e.g. genetic algorithms, evolutionary programming)
+- swarm Intelligence (e.g. particle swarm, ant colony optimization)
+- neural networks (mimicking biological neurons)
+
+Other algorithms include bacterial foraging, water cycle, and quantum-inspired methods. The book is organized into five parts, covering various algorithms such as graph search, optimization, and machine learning models.
+
+## [Credit Risk Modelling in Python](https://www.udemy.com/course/credit-risk-modeling-in-python/)
+
+This course on udemy was recommended to me once I finished the ML for financial risk management book (thank you cookies 😆) and it seems its the most popular on the topic on udemy. I did many udemy courses back in 2020 when I was learning full-stack dev and its a bit nostalgic going back now. 
+
+The course is very practical and according to the teach will show how to implement some models that follow the Basel regulations:
+
+<img width="997" alt="image" src="https://github.com/user-attachments/assets/47809205-0dc5-488c-b7c1-d83069ded1ae">
+
+It is common for PD to be done using logistic regression, and LGD and EAD using beta regressions
+
+I will share little bits, maybe the code parts, because I am not sure how much I can share from udemy's video lectures.
+
+I will learn to build models like the below ones following regulators' guidelines and how to maintain and deploy such models
+
+<img width="997" alt="image" src="https://github.com/user-attachments/assets/3e1efc65-217a-49a6-b09e-7add6e56a4e8">
+
+Under Basel II, companies can calculate the components of credit risk as such:
+
+|                   | Standardised Approach                   | Foundation-IRB               | Advanced-IRB               |
+|-------------------|-----------------------|---------------------|---------------------|
+| **PD** (Probability of Default)   | Externally provided   | Internally estimated | Internally estimated |
+| **LGD** (Loss Given Default)      | Externally provided   | Externally provided  | Internally estimated |
+| **EAD** (Exposure at Default)     | Externally provided   | Externally provided  | Internally estimated |
+
+The SA approach is very conservative as it comes from regulators. As such, its capital requirements are very high. For that reasons banks may start with the SA, but then move onto F-IRB and A-IRB. 
+
+IRB approachs:
+
+* allow banks to establish their own credit ratings
+* precise calculations about the held capital for each individual exposure
+* allows banks to better allocate resources to cover losses (and be more profitable)
+
+### There are application and behaviour models for loans
+
+The former is used to estimate the credit rating at the moment of application, the latter are used for calculated PD and EL once a loan is granted, and can be used to decide whether additional loans should be granted to existing customers
+
+If a credit card holder applies for a mortgage, the bank uses a behavioral model to estimate their credit rating by considering both application and behavioral characteristics. For ease of use by non-statistical users, like credit agents, these models are presented as scorecards, which are simplified representations of probability of default models. Scorecards are based on model coefficients and can either be application or behavioral scorecards, depending on the input characteristics. A feature of the PD model (according to the teacher) is that its best for all its features to be categorical so that created scorecard is based on categories and can be easily used. 
+
+This was section 1:
+
+<img width="406" alt="image" src="https://github.com/user-attachments/assets/ec6fea67-f890-43e4-ae6c-3a2935506cb4">
+
+Section 2 was setting up python ~ 
+
+Section 3 and 4 were doing some basic preprocessing on the dataset (Lending Club data from 2007 to 2014 from Kaggle) and this part is aimed at python and pandas beginners as it goes into the different cleaning/preprocessing functions used. 
+
+Section 5 is the longest one - Data prep for the PD model 
+
+<img width="411" alt="image" src="https://github.com/user-attachments/assets/ed8107bd-6e9b-4c6c-b567-d1755cff414c">
+
+I will cover that tomorrow and share whatever I can (maybe just the code)
+
+## Financial calculus by Baxter and Rennie
+
+Read the 1st two chapters and took basic notes of the above book
+
+![p0](https://github.com/user-attachments/assets/0086eae9-e7b8-4f75-be93-98363885ee1e)
+![Chapter 2 Discrete processes P1](https://github.com/user-attachments/assets/5a7c76f1-1968-4120-8e80-8fe0a0a11ae5)
+![Chapter 2 Discrete processes P2](https://github.com/user-attachments/assets/674e24b9-89e0-40c3-9dbd-c60b1c7c5203)
+
+
+## [Stream](https://www.youtube.com/watch?v=PFWQ2-UvFoY&t=9s)
+
+I read 2 pages from the sklearn docs on stream today but they were not very insightful 
+
+* [Common pitfalls and recommended practices](https://scikit-learn.org/stable/common_pitfalls.html#controlling-randomness)
+* [Decomposing signals in components (matrix factorization problems)](https://scikit-learn.org/stable/modules/decomposition.html#)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From e6fc22015298c5ddb4c1bd73e25b853b026ff626 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 23 Oct 2024 22:48:06 +0900
Subject: [PATCH 473/638] Create 2024-10-23-day-296.md

---
 _posts/2024-10-23-day-296.md | 71 ++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 _posts/2024-10-23-day-296.md

diff --git a/_posts/2024-10-23-day-296.md b/_posts/2024-10-23-day-296.md
new file mode 100644
index 000000000..c88e52e15
--- /dev/null
+++ b/_posts/2024-10-23-day-296.md
@@ -0,0 +1,71 @@
+---
+layout: post
+title: (Day 296) Neo4j & LLM Fundamentals
+categories: [traditional-machine-learning,theory,statistics,applying-knowledge]
+---
+
+# Hello :) Today is Day 296!
+A quick summary of today:
+* covered creating a PD model from the udemy course
+* did a neo4j & langchain course on neo4j's website (on stream)
+
+## Continuing with the credit risk modelling course on udemy
+
+Overall, this course seems such a great introduction to credit risk modelling. It introduces popular concepts and measures, and how to derive them in python. The code is not even on github, so I am a bit concious of sharing it here as it is not public at all. Therefore I will just share some key insights from each part. 
+
+### Section 5, 6: PD model data prep + model creating
+
+Interestingly, it introduced using Weight of Evidence (WoE) and Information Value (IV) for grouping metrics into bins. Unintentionally, I used this method in one of me own projects - [Insurance fraud detection](https://github.com/divakaivan/insurance-fraud-mlops-pipeline) so I am aware of how it works but still am excited to see it being applied here as well.
+
+**WoE** - to what extent an independent variable would predict a dependente var. It shows the extent to which each of the different categories of an independent variable explains the dependent one
+
+WoE for a category i is equal to the log of the ratio between the % of class 1 belonging to i divided by the % of class 0 depending to i. The further away from 0 the better
+
+<img width="978" alt="image" src="https://github.com/user-attachments/assets/9889fe58-4da9-4585-8899-c5d0b7af3b1a">
+
+**Information Value** - how much information the original independent variable brings w.r.t. explaining the dependent variable. IV can be used for preselection
+
+### Section 7 - evaluation
+
+Three metrics were introduced
+
+* AUC (ROC-AUC)
+* Gini (coming from economics) - plots the cumulative proportion of defaulted borrowers as a function of the cumulative proportion of all borrowers
+* Kolmogorov-Smirnov coefficient - shows to what extent the model separates the actual good from the actual bad borrowers; formally, it is the maximum difference between the cumulative distributions of good and bad borrowers w.r.t. predicted probabilities. The greater the difference, the better the model
+
+### Section 8 - decision-making
+
+We can use scorecards or PDs (both can be derived from each other). Furthermore we can set cut-offs to accept applicants based on a PD threshold (which is usually better than setting a (FICO) score cut-off)
+
+### Section 9 - monitoring
+
+We can use the Population Stability Index (PSI) to evaluate whether we need to re-train our model because there has been a change in the population w.r.t. a certain feature using categories of the feature.
+
+We can use the table below for guidance. 
+
+| Values of PSI (0 - 1)   | Population Difference                |
+|-------------------------|--------------------------------------|
+| PSI = 0                 | No difference                       |
+| PSI < 0.1               | Little to no difference              |
+| 0.1 < PSI < 0.25        | Little difference (No action is taken)|
+| PSI > 0.25              | Big difference (Action is taken)     |
+| PSI = 1                 | Absolute difference                  |
+
+This is what's left for tomorrow:
+
+<img width="401" alt="image" src="https://github.com/user-attachments/assets/7f8c3ab6-495e-4439-86b3-a776f1332840">
+
+## Completed [Neo4j & LLM Fundamentals](https://graphacademy.neo4j.com/courses/llm-fundamentals/) on stream
+
+* [part 1](https://youtu.be/CYZbXMXYAC8)
+* [part 2](https://youtu.be/32SDuJON80A)
+
+Had to cut in the middle because of problems with Streamlabs. Overall, a very nice course. It says it is 4 hours but I did it in under 2 hours. Definitely a good course ^^
+
+<img width="1292" alt="Screenshot at Oct 23 19-44-42" src="https://github.com/user-attachments/assets/d3670908-a708-45c5-948b-ba237a509e0d">
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 8e565d48d0db5151e0b6a5fd38e1964cdfe3f77d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 25 Oct 2024 00:28:43 +0900
Subject: [PATCH 474/638] Create 2024-10-25-day-297.md

---
 _posts/2024-10-25-day-297.md | 113 +++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 _posts/2024-10-25-day-297.md

diff --git a/_posts/2024-10-25-day-297.md b/_posts/2024-10-25-day-297.md
new file mode 100644
index 000000000..ec6964b10
--- /dev/null
+++ b/_posts/2024-10-25-day-297.md
@@ -0,0 +1,113 @@
+---
+layout: post
+title: (Day 297) More about LLMs, and credit risk modelling
+categories: [traditional-machine-learning,theory,applying-knowledge,nlp,data-eng]
+---
+
+# Hello :) Today is Day 297!
+A quick summary of today:
+* company review LLM
+* finished the credit risk modelling course
+* streamed
+* fixed my pyspark setup for some cool new tasks
+
+## Company review LLM
+
+Previously, when I talked about the movie reviewer project and data - well it was related to **this**. The data is here, and today I started working on it. 
+
+Company review data is [here](https://huggingface.co/datasets/divakaivan/anon_company_reviews), after I preprocessed it. The data came is an arbitrary format from the professor so I had to do some basic preprocessing and formating to get it to the format I need. 
+
+We will employ different PEFT strategies, but today I started with a simple LoRA fine-tuning of llama-3.2-3b-instruct. The results... bad. I did not even need to run the results through the evaluation metrics because just by looking at them - the text does not match the desired output, and is completely unralated to the desired output.
+
+Next, I tried to run the llama-3.1-8b models in colab's free T4, but the training said it would take 6-7 hours and towards the end I got a message I rand out of free hours... And the more confusing and frustrating part is that Kaggle is giving me some weird torch errors that I believe are coming from unsloth and kaggle. I might need to ask for resources from the professor - either credits for colab GPUs or using his 2nd GPU PC.
+
+I also fine-tuned a gpt4o-mini and the results for that seem by eye-balling it, however I need to run the results through the eval metrics code to get proper results for my report. 
+
+## Finishing the [Credit Risk Modelling in Python](https://www.udemy.com/course/credit-risk-modeling-in-python/) course on Udemy
+
+### LGD and EAD models
+
+For these models we need data for accounts that have been written-off and a sufficient amount of time has passed so there can be some recoveries. 
+
+**Recovery rate**
+
+LGD - if a borrower defaults, it is the proportion of the total exposure that can be recovered by the lender once a default has occured. An established approach in practice is to model the proportion of the total exposure that can be recovered by the lender once a default has occured. This proportion is called **recovery rate** (recoveries / funded amount). From there, the proportion that can't be recovered (LGD) can be calculated easily because it equals 1 - recovery_rate for each exposure.
+
+Sometimes due to calculation and database specifications, we might end up with values < 0 and/or > 1. In such cases, replace the ones below 0 with 0, and the ones above 1 with 1.
+
+**Credit conversion factor (CCF)**
+
+EAD is the total value that a lender is exposed to when a borrower defaults. Therefore, it is the max that a bank may lose when a borrower defaults on a loan. On many occasions, the bank has decided to grant an amount of money but the lender has not disbursed the whole amount. Also, the borrower might have repaid a signigicant amount of the loan already at the time of default. Moreover, for revolving facilities such as credit cards, but also for some consumer laons, the borrower may be able to repay and spend what they had already repaid again, up to a certain limit called credit limit. So the borrower may only have defaulted on a proportion of the original funded amount. That proportion is going to be the dependent variable for the EAD model. Often, this proportion is called credit conversion factor. And EAD = total funded amount X credit conversion factor. Higher CCF -> higher risk, and vice versa. 
+
+Both of the above are in the range [0, 1] and both of them are described with a Beta distribution so a beta regression is used to model them. But because no major python package offers this type of regression natively, we will can use some other approach. 
+
+For recovery rate and estimating LGD, we can use a 2-stage approach:
+
+1. is recovery rate equal to 0 or greater than 0? (binary question - logistic regression)
+2. if recovery rate is greater than 0, how much exactly is it? (use linear regression - make sure the values are between 0 and 1; we can use correlation coef to evaluate the model; and also look that the distribution of the residuals follows a normal distribution)
+
+<img width="843" alt="image" src="https://github.com/user-attachments/assets/dcae829d-5612-4023-b3be-0288a4e14997">
+
+For CCF, the course suggests just to use a multi-linear regression model (here as well, we can use correlation coef as an evaluation metric; and also that the residuals follow a normal distribution)
+
+After we have all three, we can calculate Expected Loss = PG x LGD x EAD for our customer base or on a portfolio level.
+
+We can then calculate the ratio of EL divided by total funded amount and that number can add to our decision-making in whether to employ a more aggressive or conservative lending strategy.
+
+### Overall
+
+Wow, such a great course and intro to the topic of credit risk modelling. I am not the exact audience for this course as I am already familiar with majority of the concepts, but it was a nice refresher and reminded me of my 1st year at Lloyds Banking Group (which I cherish deeply)
+
+![image](https://github.com/user-attachments/assets/5d36f0ba-b6f8-4387-bb76-476c4f4ac7f5)
+
+## [Stream](https://youtu.be/LFh-4WPGgl0)
+
+Completed this Noe4j course on stream - [Build a Neo4j-backed Chatbot using Python](https://graphacademy.neo4j.com/courses/llm-chatbot-python/)
+
+It was not bad. After the stream ended I was looking around and found this - [GraphRAG Python Package: Accelerating GenAI With Knowledge Graphs](https://neo4j.com/blog/graphrag-python-package/?utm_source=GPMax&utm_medium=PaidSearch&utm_campaign=Evergreen&utm_content=EMEA-Search-SEMCE-DSA-None-SEM-SEM-NonABM&utm_term=&utm_adgroup=pmax&gad_source=1&gclid=Cj0KCQjw4Oe4BhCcARIsADQ0csnjz9fW4fKpJhTHZj-snczIwJyVCG63YvozGGs5aEfN6XvpR7XwBEAaAhCdEALw_wcB)
+
+This is neo4j's native GraphRAG python package. Which reflects my belief that what langchain offers is fast-changing and learning one thing today, might be something similar, but different, in 1 month because the API has changed, or the specific function is provided natively by another package. Anyway ~ this blog post is something to read, maybe on next stream (Monday) or by meself - I will see.
+
+---
+
+I want to try to convert the pandas code and models from the Udemy course to PySpark - kind of what part of my role was back during my 1st year at Lloyds Banking Group. I decided to start doing it, but of course I faced pyspark issues. I spend ~1hr fixing them but finally I got it done. I will follow along the notebooks that do it in pandas and translate that to pyspark ~ but that is for tomorrow/next few days.
+
+That is all for today!
+
+See you tomorrow :)
+
+P.S. had the post ready to post, but was playing around with translating the pandas code to pyspark and got stuck at doing OHE in pyspark. 
+
+So far, I have
+
+```python
+from pyspark.ml.feature import StringIndexer, OneHotEncoder
+from pyspark.ml import Pipeline
+
+indexer = StringIndexer(inputCol='grade', outputCol='grade_index')
+encoder = OneHotEncoder(inputCol='grade_index', outputCol='grade_ohe', dropLast=True)
+pipeline = Pipeline(stages=[indexer, encoder])
+pipeline_model = pipeline.fit(df)
+df = pipeline_model.transform(df)
+
+df.select('grade', 'grade_index', 'grade_ohe').show()
+```
+
+Which returns:
+
+```
++-----+-----------+-------------+
+|grade|grade_index|    grade_ohe|
++-----+-----------+-------------+
+|    B|        0.0|(6,[0],[1.0])|
+|    C|        1.0|(6,[1],[1.0])|
+|    C|        1.0|(6,[1],[1.0])|
+|    C|        1.0|(6,[1],[1.0])|
+|    B|        0.0|(6,[0],[1.0])|
+|    A|        3.0|(6,[3],[1.0])|
+|    C|        1.0|(6,[1],[1.0])|
+|    E|        4.0|(6,[4],[1.0])|
+|    F|        5.0|(6,[5],[1.0])|
+```
+
+And apparently for spark ML models the features need to be put in a vector, so that's why this is returned like that. I need to think if this is ok for me and will continue tomorrow.

From 41a36d619658bc586618820e4d6e0e16def9c6dd Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 25 Oct 2024 00:29:07 +0900
Subject: [PATCH 475/638] Rename 2024-10-25-day-297.md to 2024-10-24-day-297.md

---
 _posts/{2024-10-25-day-297.md => 2024-10-24-day-297.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename _posts/{2024-10-25-day-297.md => 2024-10-24-day-297.md} (100%)

diff --git a/_posts/2024-10-25-day-297.md b/_posts/2024-10-24-day-297.md
similarity index 100%
rename from _posts/2024-10-25-day-297.md
rename to _posts/2024-10-24-day-297.md

From e856cec12d0b5f78232b57974185b63af67d9fc8 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 25 Oct 2024 22:23:50 +0900
Subject: [PATCH 476/638] Create 2024-10-25-day-298.md

---
 _posts/2024-10-25-day-298.md | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 _posts/2024-10-25-day-298.md

diff --git a/_posts/2024-10-25-day-298.md b/_posts/2024-10-25-day-298.md
new file mode 100644
index 000000000..9c589eb8f
--- /dev/null
+++ b/_posts/2024-10-25-day-298.md
@@ -0,0 +1,35 @@
+---
+layout: post
+title: (Day 298) From Pandas to PySpark
+categories: [applying-knowledge,data-eng]
+---
+
+# Hello :) Today is Day 298!
+A quick summary of today:
+* starting to translate pandas data transformations to pyspark
+
+In my 1st year in Lloyds Banking Group I was part of a project that built an innovative webapp for the risk reporting world - and part of my role in the team was tranlsating code from sas to python (pandas). More than SAS to python, though, we had to switch from pandas to pyspark and I had to learn how to translate pandas functions into pyspark, and also I helped other colleagues translate their code directly from sas to pyspark.
+
+Doing the Credit Modelling with Python course on Udemy this past week, reminded me of this great time in my professional life and I decided to try to convert the pandas code from the course to pyspark. The reason we switched from pandas to pyspark back then (2019-2020) was because our data was millions and millions and millions of rows so pyspark was a much better choice. Now... the dataset used in the course is not that big so I most likely won't see any benefit of switching from pandas to pyspark, I think it is still a good exercise to translate the models. 
+
+**Disclaimer**
+
+I cannot share the pandas code from the course because the course author does not have it public anywhere so I do not want to share it either. However, I will share the pyspark notebooks/scripts as I write them.
+
+Here is my project structure
+
+<img width="393" alt="image" src="https://github.com/user-attachments/assets/5d1a45f0-a47f-4f7b-862b-2fe1434bb6b7">
+
+The pandas folder contains the code from the course, and the pyspark folder is my pyspark code.
+
+For now, rather than uploading it to github, I will keep it local and just share my code through colab. [Here is what I have so far](https://colab.research.google.com/drive/1rz3ySHhE1lV5vAqIjE95l6HtqPNiylTa?usp=sharing)
+
+What I finished at is figuring out how to do WoE and IV calculations with PySpark. However, I think I might do just data transformation and model translation, rather than the WoE and IV data analysis as well. For the sake of continuing and getting to the modelling part, I might assume that the output of the WoE and IV analysis is a given and I will just translate those transformations.
+
+I could not do much more today because of my Friday class. But I invite you to check the colab notebook for the pyspark code ^^ It reminded me of my great time in my risk analyst job. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 407f4522b5ca4591a99738d64ff4f3fa4314ef1a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 26 Oct 2024 21:31:17 +0900
Subject: [PATCH 477/638] Create 2024-10-26-day-299.md

---
 _posts/2024-10-26-day-299.md | 708 +++++++++++++++++++++++++++++++++++
 1 file changed, 708 insertions(+)
 create mode 100644 _posts/2024-10-26-day-299.md

diff --git a/_posts/2024-10-26-day-299.md b/_posts/2024-10-26-day-299.md
new file mode 100644
index 000000000..4aebec17c
--- /dev/null
+++ b/_posts/2024-10-26-day-299.md
@@ -0,0 +1,708 @@
+---
+layout: post
+title: (Day 299) ML in PySpark
+categories: [applying-knowledge,data-eng,nlp]
+---
+
+# Hello :) Today is Day 299!
+A quick summary of today:
+* read a bit of [Data Analysis with Python and PySpark](https://learning.oreilly.com/library/view/data-analysis-with/9781617297205/)
+* evaluate gpt4o-mini's finetuning results
+
+## [Data Analysis with Python and PySpark](https://learning.oreilly.com/library/view/data-analysis-with/9781617297205/)
+
+I found this book and its part 3 is _Get confident: Using machine learning with PySpark_, which consists of these 3 chapters:
+
+* 12 Setting the stage: Preparing features for machine learning
+* 13 Robust machine learning with ML Pipelines
+* 14 Building custom ML transformers and estimators
+
+## Chapter 12 - Preparing features for machine learning
+
+Dataset used is about food nutrition with ~20K samples and 680 columns - [here](https://www.kaggle.com/hugodarwood/epirecipes)
+
+### Preparing the data for making features
+
+Because the raw column names vary in their format (i.e. include -, _, *, etc), we first need to clean them:
+
+```python
+def sanitize_column_name(name):
+    """Drops unwanted characters from the column name.
+ 
+    We replace spaces, dashes and slashes with underscore,
+    and only keep alphanumeric characters.
+    """
+    answer = name
+    for i, j in ((" ", "_"), ("-", "_"), ("/", "_"), ("&", "and")):   
+        answer = answer.replace(i, j)
+    return "".join([char for char in answer if char.isalpha() or char.isdigit() or char == "_"])
+ 
+food = food.toDF(*[sanitize_column_name(name) for name in food.columns])
+```
+
+**Next is identifying which columns need what kind of transformation. For example, identifying binary columns vs multi-categorical columns for encoding.**
+
+For binary cols, the author suggests:
+
+```python
+import pandas as pd
+ 
+pd.set_option("display.max_rows", 1000)             
+ 
+is_binary = food.agg(*[(F.size(F.collect_set(x)) == 2).alias(x) for x in food.columns]).toPandas()
+ 
+is_binary.unstack()
+```
+
+Which returns (I am pasting a truncated version):
+
+```
+title                     0    False
+rating                    0    False
+calories                  0    False
+protein                   0    False
+fat                       0    False
+sodium                    0    False
+cakeweek                  0    False
+wasteless                 0    False
+22_minute_meals           0     True
+3_ingredient_recipes      0     True
+30_days_of_groceries      0     True
+```
+
+Of course, the above might not be able to detect hidden categorical features, so we need to be careful and try to understand the data before moving onto more in-depth feature engineering.
+
+**Next, 4 sets of columns are created:**
+
+* identifiers, which are the column(s) that contain the information unique to each record
+
+* targets, which are the column(s) (most often one) that contain the value we wish to predict
+
+* continuous columns, containing continuous features
+
+* binary columns, containing binary features
+
+```python
+IDENTIFIERS = ["title"]
+ 
+CONTINUOUS_COLUMNS = [
+    "rating",
+    "calories",
+    "protein",
+    "fat",
+    "sodium",
+]
+ 
+TARGET_COLUMN = ["dessert"]   
+ 
+BINARY_COLUMNS = [
+    x
+    for x in food.columns
+    if x not in CONTINUOUS_COLUMNS
+    and x not in TARGET_COLUMN
+    and x not in IDENTIFIERS
+]
+```
+
+**Next is deleting rows where either the target is null or majority of the features are null, and also imputing some missing values.**
+
+Ignoring the target and the recipe name column:
+
+```python
+food = food.dropna(how="all", subset=[x for x in food.columns if x not in IDENTIFIERS])
+ 
+food = food.dropna(subset=TARGET_COLUMN)
+```
+
+Which results in just 5 records being deleted. 
+
+**Next, if we want we can impute a value of 0 to any missing values in the binary columns:**
+
+```python
+food = food.fillna(0.0, subset=BINARY_COLUMNS)
+```
+
+**Next is taking care of extreme values among the continuous features**
+
+Before this, we need to make sure all numerical cols are read as numerical by pyspark. We can check for non-numerical values using:
+
+```python
+from typing import Optional
+ 
+@F.udf(T.BooleanType())
+def is_a_number(value: Optional[str]) -> bool:
+    if not value:
+        return True
+    try:
+        _ = float(value)                         
+    except ValueError:
+        return False
+    return True
+ 
+food.where(~is_a_number(F.col("rating"))).select(*CONTINUOUS_COLUMNS).show()
+```
+
+Which returns this rogue row:
+
+```
++---------+------------+-------+----+------+
+|   rating|    calories|protein| fat|sodium|
++---------+------------+-------+----+------+
+| Cucumber| and Lemon "|   3.75|NULL|  NULL|
++---------+------------+-------+----+------+
+```
+
+Now that this is fixed, the decided approach to deal with extreme values is to harcode a maximum:
+
+```python
+maximum = {
+    "calories": 3203.0,                                  
+    "protein": 173.0,                                    
+    "fat": 207.0,                                        
+    "sodium": 5661.0,                                    
+}
+ 
+for k, v in maximum.items():
+    food = food.withColumn(k, F.when(F.isnull(F.col(k)), F.col(k)).otherwise(F.least(F.col(k), F.lit(v))))
+```
+
+**Next is weeding out the rare binary occurrence columns**
+
+For this model, 10 is chosen to be the threshold. We do not want binary features with less than 10 of 0.0 or 1.0 in the model.
+
+```python
+inst_sum_of_binary_columns = [F.sum(F.col(x)).alias(x) for x in BINARY_COLUMNS]
+ 
+sum_of_binary_columns = (food.select(*inst_sum_of_binary_columns).head().asDict())
+ 
+num_rows = food.count()
+too_rare_features = [k for k, v in sum_of_binary_columns.items() if v < 10 or v > (num_rows - 10)]
+ 
+len(too_rare_features)
+print(too_rare_features)
+```
+
+```python
+BINARY_COLUMNS = list(set(BINARY_COLUMNS) - set(too_rare_features))
+```
+
+Now, the binary columns are 506 (167 removed)
+
+### Feature creation and refinement
+
+**Creating custom features**
+
+```python
+food = food.withColumn(
+    "protein_ratio", F.col("protein") * 4 / F.col("calories")  
+).withColumn(
+    "fat_ratio", F.col("fat") * 9 / F.col("calories")
+)                                                              
+ 
+food = food.fillna(0.0, subset=["protein_ratio", "fat_ratio"])
+ 
+CONTINUOUS_COLUMNS += ["protein_ratio", "fat_ratio"]
+```
+
+We need to be careful not to introduce multicollinear features
+
+**Removing highly correlated features**
+
+First, we need to create a vector of the continuous features:
+
+```python
+from pyspark.ml.feature import VectorAssembler
+ 
+continuous_features = VectorAssembler(
+    inputCols=CONTINUOUS_COLUMNS, outputCol="continuous_features"
+)
+ 
+vector_food = food.select(CONTINUOUS_COLUMNS)
+for x in CONTINUOUS_COLUMNS:
+    vector_food = vector_food.where(~F.isnull(F.col(x)))     
+ 
+vector_variable = continuous_features.transform(vector_food)
+ 
+vector_variable.select("continuous_features").show(3, False)
+```
+
+```
++---------------------------------------------------------------------+
+|continuous_features                                                  |
++---------------------------------------------------------------------+
+|[2.5,426.0,30.0,7.0,559.0,0.28169014084507044,0.14788732394366197]   |
+|[4.375,403.0,18.0,23.0,1439.0,0.17866004962779156,0.5136476426799007]|
+|[3.75,165.0,6.0,7.0,165.0,0.14545454545454545,0.38181818181818183]   |
++---------------------------------------------------------------------+
+```
+
+Once we have that we can use pyspark's Correlation.corr() 
+
+```python
+from pyspark.ml.stat import Correlation
+ 
+correlation = Correlation.corr(
+    vector_variable, "continuous_features"                 
+)
+ 
+correlation.printSchema()
+
+# |-- pearson(continuous_features): matrix (nullable = false)
+```
+
+The Correlation.corr() returns returns a DenseMatrix column type, which is like a two-dimensional vector. In order to extract the values in an easy-to-read format, we have to do a little method juggling:
+
+1. we extract a single record as a list of Row using head()
+
+2. a Row is like an ordered dictionary, so we can access the first (and only) field containing our correlation matrix using list slicing
+
+3. a DenseMatrix can be converted into a pandas-compatible array by using the toArray() method on the matrix
+
+4. we can directly create a pandas DataFrame from our Numpy array. Inputting our column names as an index (in this case, they’ll play the role of 'row names') makes the correlation matrix very readable
+
+```python
+correlation_array = correlation.head()[0].toArray()         
+ 
+correlation_pd = pd.DataFrame(
+    correlation_array,                                      
+    index=CONTINUOUS_COLUMNS,                               
+    columns=CONTINUOUS_COLUMNS,                             
+)
+ 
+print(correlation_pd.iloc[:, :4])
+```
+
+```
+                 rating  calories   protein       fat
+rating         1.000000  0.102257  0.113292  0.111536
+calories       0.102257  1.000000  0.757837  0.918052
+protein        0.113292  0.757837  1.000000  0.664899
+fat            0.111536  0.918052  0.664899  1.000000
+sodium         0.065225  0.516818  0.585450  0.421920
+protein_ratio  0.094429  0.164735  0.600182  0.125572
+fat_ratio      0.129946  0.176823  0.109188  0.424986
+```
+
+```python
+print(correlation_pd.iloc[:, 4:])
+```
+
+```
+                 sodium  protein_ratio  fat_ratio
+rating         0.065225       0.094429   0.129946
+calories       0.516818       0.164735   0.176823
+protein        0.585450       0.600182   0.109188
+fat            0.421920       0.125572   0.424986
+sodium         1.000000       0.339067   0.033819
+protein_ratio  0.339067       1.000000   0.024854
+fat_ratio      0.033819       0.024854   1.000000
+```
+
+### Feature preparation with transformers and estimators
+
+![image](https://github.com/user-attachments/assets/224e2cb8-628f-472e-be12-b6226fa20ad3)
+
+Estimators are like a transformers factory. The next parts introduce estimators through the usage of Imputer and MinMaxScaler.
+
+**Imputing continuous features using the Imputer estimator**
+
+![image](https://github.com/user-attachments/assets/6ace274e-60a0-4e13-b903-2e081ed9dc1f)
+
+The image shows the Imputer and its companion transformer/model, ImputerModel. When fit() is called on an instantiated estimator, a fully parameterized transformer (called Model) is created.
+
+As an example, we want our Imputer to impute the mean value to every record in the calories, protein, fat, and sodium columns when the record is null
+
+```python
+from pyspark.ml.feature import Imputer
+ 
+OLD_COLS = ["calories", "protein", "fat", "sodium"]
+NEW_COLS = ["calories_i", "protein_i", "fat_i", "sodium_i"]
+ 
+imputer = Imputer(
+    strategy="mean",                                            
+    inputCols=OLD_COLS,                                         
+    outputCols=NEW_COLS,                                        
+)
+ 
+imputer_model = imputer.fit(food)                               
+ 
+CONTINUOUS_COLUMNS = (
+    list(set(CONTINUOUS_COLUMNS) - set(OLD_COLS)) + NEW_COLS    
+)
+
+# Using the imputer:
+food_imputed = imputer_model.transform(food)
+```
+
+**Scaling our features using the MinMaxScaler estimator**
+
+![image](https://github.com/user-attachments/assets/98bd31a4-c21e-4b0e-8768-78c281f0f858)
+
+The MinMaxScaler and its companion model, the MinMaxScalerModel—the same model of operation as the Imputer estimator
+
+```python
+from pyspark.ml.feature import MinMaxScaler
+ 
+CONTINUOUS_NB = [x for x in CONTINUOUS_COLUMNS if "ratio" not in x]
+ 
+continuous_assembler = VectorAssembler(
+    inputCols=CONTINUOUS_NB, outputCol="continuous"
+)
+ 
+food_features = continuous_assembler.transform(food_imputed)
+ 
+continuous_scaler = MinMaxScaler(
+    inputCol="continuous",
+    outputCol="continuous_scaled",
+)
+ 
+food_features = continuous_scaler.fit(food_features).transform(
+    food_features
+)
+```
+
+## Chapter 13 - Robust machine learning with ML Pipelines
+
+### Transformers and estimators: The building blocks of ML in Spark
+
+**Data comes in, data comes out: The Transformer**
+
+The transformer is the first building block of an ML pipeline
+
+![image](https://github.com/user-attachments/assets/55df0549-3510-4a92-afbf-a5a8b07dd27c)
+
+In PySpark, transformers use a `Params` mechanism for configuration. When initializing transformers like `VectorAssembler`, keyword-only arguments define its `Params`. To access or modify these `Params`, getters and setters are used, which differ from typical Python attributes by returning a `Param` object, not the direct value. To get the actual value, the getter method (e.g., `getOutputCol()`) is used, while `explainParam()` details the `Param`, including its name, description, default, and current value.
+
+For changing values, setter methods (like `setOutputCol()`) modify `Params` in place, while `setParams()` allows updating multiple `Params` simultaneously. To reset a `Param` to default, the `clear()` method is used. Transformers are passed by reference, so using `copy()` ensures modifications on one instance don’t affect others.
+
+**Data comes in, transformer comes out: The Estimator**
+
+![image](https://github.com/user-attachments/assets/a7f7fae2-68de-483e-be4e-f2ab64ad7b3a)
+
+In PySpark, an estimator is an essential ML pipeline component that, unlike a transformer, uses the `fit()` method to train on an input DataFrame and returns a transformer model. This distinction allows an estimator to adjust parameters based on the input data, whereas a transformer relies solely on pre-set parameters to apply transformations. For example, the `MinMaxScaler` estimator takes `min` and `max` (defaulting to 0.0 and 1.0) and the `inputCols` and `outputCols` parameters. During `fit()`, it calculates the actual minimum (`E_min`) and maximum (`E_max`) values of the input column, scales them to the min-max range, and outputs a configured transformer.
+
+This `fit()/transform()` pattern is foundational in PySpark ML, where estimators—including models—adapt to data specifics before transforming. If `E_min` equals `E_max`, scaling cannot proceed as usual since the range is zero, a scenario to handle as a special case.
+
+### Building a (complete) machine learning pipeline
+
+Pipelines build on transformers and estimators to make training, evaluating, and optimizing ML models much clearer and more explicit
+
+The Pipeline estimator has only one Param, called stages, which takes a list of transformers and estimators.
+
+When using pipelines, we need remember that the data frame will travel each stage. For instance, the continuous_scaler stage will have the output of the data frame transformed by continuous_scaler. For estimator stages, the data frame stays identical, as fit() does not transform the data frame but returns a Model instead.
+
+![image](https://github.com/user-attachments/assets/14c2ba63-e27d-4453-b3cc-bff0b349bca6)
+
+When calling fit() using a data frame, that data frame gets passed as an argument to the first stage. Each estimator stage gets evaluated (transformers are passed verbatim), and the resulting transformers/models form the stages of the PipelineModel.
+
+**Assembling the final data set with the vector column type**
+
+```python
+preml_assembler = MF.VectorAssembler(
+    inputCols=BINARY_COLUMNS                          
+    + ["continuous_scaled"]
+    + ["protein_ratio", "fat_ratio"],
+    outputCol="features",
+)
+ 
+food_pipeline.setStages(
+    [imputer, continuous_assembler, continuous_scaler, preml_assembler]
+)
+ 
+food_pipeline_model = food_pipeline.fit(food)         
+food_features = food_pipeline_model.transform(food)
+```
+
+Our data frame is ready for machine learning! We have a number of records, each with
+
+* a target (or label) column, dessert, containing a binary input (1.0 if the recipe is a dessert, 0.0 otherwise)
+
+* a vector of features, called features, containing all the information we want to train our machine learning model with
+
+We provide 513 distinct features with a large number of zeroes. When storing vectors, PySpark has two choices for representing vectors:
+
+* a dense representation, where a Vector in PySpark is simply a numpy single-dimensional array object
+
+* a sparse representation, where a Vector in PySpark is an optimized sparse vector compatible with the scipy scipy.sparse matrix
+
+In practice, we don’t decide if a Vector is sparse or dense: PySpark will convert between the two as needed.
+
+```
+Dense:  [0.0, 1.0, 4.0, 0.0]
+Sparse: (4, [1,2], [1.0, 4.0])
+```
+
+We can further see the schema of our features using:
+
+```python
+print(food_features.schema["features"])
+ 
+# StructField(features,VectorUDT,true)
+ 
+print(food_features.schema["features"].metadata)
+```
+
+```
+{'ml_attr': {'attrs': {'numeric': [{'idx': 0, 'name': 'ham'}, {'idx': 1, 'name': 'frangelico'}, {'idx': 2, 'name': 'lunch'}, {'idx': 3, 'name': 'cherry'}, {'idx': 4, 'name': 'cookies'}, {'idx': 5, 'name': 'rice'}, {'idx': 6, 'name': 'rutabaga'}, {'idx': 7, 'name': 'kosher_for_passover'}, {'idx': 8, 'name': 'potluck'}, {'idx': 9, 'name': 'fruit'}, {'idx': 10, 'name': 'nut'}, {'idx': 11, 'name': 'lemon_juice'}, {'idx': 12, 'name': 'nutmeg'}, {'idx': 13, 'name': 'simmer'}, {'idx': 14, 'name': 'egg'}, {'idx': 15, 'name': 'meat'}, {'idx': 16, 'name': 'harpercollins'}, {'idx': 17, 'name': 'walnut'}, {'idx': 18, 'name': 'arugula'}, {'idx': 19, 'name': 'caraway'}, {'idx': 20, 'name': 'pear'}, {'idx': 21, 'name': 'missouri'}, {'idx': 22, 'name': 'thanksgiving'},
+
+...
+```
+
+The column schema for an assembled vector will keep track of the features making its composition under the metadata attribute.
+
+For scaled variables, since they originate from a VectorAssembler, PySpark gives them a generic name, but we can track their name from the original vector column (here continuous_assembled) as needed.
+
+**Training an ML model using a LogisticRegression classifier**
+
+```python
+from pyspark.ml.classification import LogisticRegression
+ 
+lr = LogisticRegression(
+    featuresCol="features", labelCol="dessert", predictionCol="prediction"
+)
+ 
+food_pipeline.setStages(
+    [
+        imputer,
+        continuous_assembler,
+        continuous_scaler,
+        preml_assembler,
+        lr,
+    ]
+)
+```
+
+Before fitting the pipeline, we cache() the training data frame. We do this because ML uses the data frame repeatedly, so caching in memory provides an increase in speed if your cluster has enough memory.
+
+```python
+train, test = food.randomSplit([0.7, 0.3], 13)   
+ 
+train.cache()
+ 
+food_pipeline_model = food_pipeline.fit(train)   
+results = food_pipeline_model.transform(test)
+```
+
+_WARNING! Although PySpark will use the same seed, which should guarantee that the split will be consistent across runs, there are some cases where PySpark will break that consistency. If you want to be 100% certain about your splits, split your data frame, write each one to disk, and then read them from the disk location._
+
+### Evaluating and optimizing our model
+
+**Assessing model accuracy: Confusion matrix and evaluator object**
+
+```python
+results.groupby("dessert").pivot("prediction").count().show()
+```
+
+```
++-------+----+---+
+|dessert| 0.0|1.0|
++-------+----+---+
+|    0.0|4944| 87|
+|    1.0|  93|991|
++-------+----+---+
+```
+
+In Spark 3.1+ we can use the below for precision and recall:
+
+```python
+lr_model = food_pipeline_model.stages[-1]                  
+metrics = lr_model.evaluate(results.select("title", "dessert", "features"))
+ 
+print(f"Model precision: {metrics.precisionByLabel[1]}")   
+print(f"Model recall: {metrics.recallByLabel[1]}")
+
+# Model precision: 0.9192949907235621
+# Model recall: 0.9142066420664207
+```
+
+Otherwise, we can use the RDD-based API:
+
+```python
+from pyspark.mllib.evaluation import MulticlassMetrics
+ 
+predictionAndLabel = results.select("prediction", "dessert").rdd
+ 
+metrics_rdd = MulticlassMetrics(predictionAndLabel)
+ 
+print(f"Model precision: {metrics_rdd.precision(1.0)}")
+print(f"Model recall: {metrics_rdd.recall(1.0)}")
+
+# Model precision: 0.9192949907235621
+# Model recall: 0.9142066420664207
+```
+
+**True positives vs. false positives: The ROC curve**
+
+```python
+from pyspark.ml.evaluation import BinaryClassificationEvaluator
+ 
+evaluator = BinaryClassificationEvaluator(
+    labelCol="dessert",                     
+    rawPredictionCol="rawPrediction",       
+    metricName="areaUnderROC",
+)
+ 
+accuracy = evaluator.evaluate(results)
+print(f"Area under ROC = {accuracy} ")
+
+# Area under ROC = 0.9927985236918562
+```
+
+```python
+import matplotlib.pyplot as plt
+ 
+plt.figure(figsize=(5, 5))
+plt.plot([0, 1], [0, 1], "r--")
+plt.plot(
+    lr_model.summary.roc.select("FPR").collect(),
+    lr_model.summary.roc.select("TPR").collect(),
+)
+plt.xlabel("False positive rate")
+plt.ylabel("True positive rate")
+plt.show()
+```
+
+<img width="377" alt="image" src="https://github.com/user-attachments/assets/e9f4e9f9-29c3-4de2-9c23-237e43afb7d6">
+
+**Optimizing hyperparameters with cross-validation**
+
+To build the set of hyperparameters we wish to evaluate our model against, we use the ParamGridBuilder, which assists in creating a Param Map. For this, we start with the builder class which can take a series of addGrid() methods taking two parameters:
+
+* the Param of the stage we want to modify. In this case, our LogisticRegression estimator was assigned to the variable lr, so lr.elasticNetParam is the Param in question
+* the values we wish to assign the hyperparameter on, passed as a list
+
+Once we are done, we call build(), and a list of Param Maps is returned. Each element of the list is a dictionary (called Map in Scala, hence the name) of the Params that will be passed to our pipeline when fitting. Often, we want to set the hyperparameters for the model estimator, but nothing prevents us from changing the Params from another stage, for instance the preml_assembler, should we want to remove features. We just need to be sure to be consistent if you meddle with inputCol/outputCol to avoid missing column errors.
+
+```python
+from pyspark.ml.tuning import ParamGridBuilder
+ 
+grid_search = (
+    ParamGridBuilder()                           
+    .addGrid(lr.elasticNetParam, [0.0, 1.0])     
+    .build()                                     
+)
+ 
+print(grid_search)
+```
+
+```
+[{Param(parent='LogisticRegression_e87ad8468abf', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0.0}, {Param(parent='LogisticRegression_e87ad8468abf', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 1.0}]
+```
+
+Now onto cross-validation. PySpark provides out-of-the-box K-fold cross-validation through the CrossValidator class
+
+Code-wise, the CrossValidator object combines everything under a single abstraction. To build a cross-validator, we need three elements, all of which we’ve encountered so far:
+
+* an estimator, which contains the model we wish to evaluate (here: food_ pipeline)
+* an estimatorParamMaps set, which is the list of Param Maps we created earlier in the section
+* an evaluator, which carries the metric we wish to optimize against
+
+```python
+from pyspark.ml.tuning import CrossValidator
+ 
+cv = CrossValidator(
+    estimator=food_pipeline,
+    estimatorParamMaps=grid_search,
+    evaluator=evaluator,
+    numFolds=3,
+    seed=13,
+    collectSubModels=True,
+)
+ 
+cv_model = cv.fit(train)                     
+print(cv_model.avgMetrics)
+# [0.9895381256961793, 0.9895383283033521]
+
+pipeline_food_model = cv_model.bestModel
+```
+
+### Getting the biggest drivers from our model: Extracting the coefficients
+
+```python
+import pandas as pd
+ 
+feature_names = ["(Intercept)"] + [                     
+    x["name"]
+    for x in (
+        food_features
+        .schema["features"]
+        .metadata["ml_attr"]["attrs"]["numeric"]
+    )
+]
+feature_coefficients = [lr_model.intercept] + list(     
+    lr_model.coefficients.values
+)
+ 
+ 
+coefficients = pd.DataFrame(
+    feature_coefficients, index=feature_names, columns=["coef"]
+)
+ 
+coefficients["abs_coef"] = coefficients["coef"].abs()  
+ 
+print(coefficients.sort_values(["abs_coef"]))
+```
+
+Sample:
+
+```
+                               coef   abs_coef
+hanukkah                   0.001985   0.001985
+house_and_garden           0.004944   0.004944
+pernod                    -0.015873   0.015873
+```
+
+**Writing and Reading ML models**
+
+```python
+pipeline_food_model.write().overwrite().save("am_I_a_dessert_the_model")
+```
+
+and
+
+```python
+from pyspark.ml.pipeline import PipelineModel
+ 
+loaded_model = PipelineModel.load("am_I_a_dessert_the_model")
+```
+
+### Creating our own transformer and estimators
+
+Creating a custom transformer is not hard, but there are a lot of moving parts and a set of conventions to follow to make it consistent with the other transformers provided by PySpark. Our blueprint for this section follows this plan:
+
+1. Design our transformer: Params, inputs, and outputs
+2. Create the Params, inheriting some preconfigured ones as necessary
+3. Create the necessary getters and setters to get
+4. Create the initialization function to instantiate our transformer
+5. Create the transformation function
+
+For Estimators the plan is:
+
+1. Outline the design or the estimator, taking into account the resulting model: inputs, outputs, fit(), and transform()
+2. Create the companion model class as a Model (which is a specialized Transformer) subclass
+3. Create the estimator as an Estimator subclass
+
+The full code for this example is [here](https://github.com/jonesberg/DataAnalysisWithPythonAndPySpark/blob/trunk/code/Ch14/custom_feature.py)
+
+---
+
+I saw that this book has a [video edition](https://learning.oreilly.com/videos/data-analysis-with/9781617297205VE/) so I might watch some of the videos of the other chapters that just go over basic pyspark functions and see if there is anything interest. I will bookmark this part for later ^^
+
+## Evaluating gpt4o-mini 
+
+A few days ago I mentioned about starting to use company review data and fine-tuning a model for that task. I had fine-tuned gpt4o-mini with the company dataset, however I was yet to evaluate it based on the preset metrics - RougeLSum for text and RMSE for scores. Well today, after some debugging mainly due to data formatting, I got the results:
+
+```
+RougeLSum: 0.8768	(1 is max meaning identical, 0 is the min)
+RMSE: 0.5924 (lower is better)
+```
+
+And [here](https://platform.openai.com/playground/chat?preset=x8k1WrYsu6Mn2v9r7XfqPXW6) is a link to the model on OpenAI's platform
+
+---
+
+Also, tomorrow is day 300 which is exciting but I don't have anything special planned haha
+
+That is all for today!
+
+See you tomorrow :)

From b8b11ef119e02e346278c025aae423c523f026bb Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 27 Oct 2024 22:34:17 +0900
Subject: [PATCH 478/638] Create 2024-10-27-day-300.md

---
 _posts/2024-10-27-day-300.md | 181 +++++++++++++++++++++++++++++++++++
 1 file changed, 181 insertions(+)
 create mode 100644 _posts/2024-10-27-day-300.md

diff --git a/_posts/2024-10-27-day-300.md b/_posts/2024-10-27-day-300.md
new file mode 100644
index 000000000..facfb9a78
--- /dev/null
+++ b/_posts/2024-10-27-day-300.md
@@ -0,0 +1,181 @@
+---
+layout: post
+title: (Day 300) Trying out Neo4j's GraphRAG package
+categories: [applying-knowledge,nlp,gnn]
+---
+
+# Hello :) Today is Day 300!
+A quick summary of today:
+* path finding in neo4j
+* streamed
+
+
+## [Path Finding with GDS](https://graphacademy.neo4j.com/courses/gds-shortest-paths/)
+
+Before streaming today I had some free time and decided to do these ~1hr course from Neo4j.
+
+### Shortest path calculation
+
+Graphs allow for calculating the shortest paths by representing cities, intersections, or other points as *nodes* and the connections between them (such as roads or routes) as *edges*. For instance, in a transportation network, one can determine the fastest or least expensive route between cities by considering attributes like travel time or distance. Graph databases like Neo4j can quickly identify these optimal paths by leveraging each relationship’s properties, such as cost or travel time.
+
+![image](https://github.com/user-attachments/assets/db06b1c7-9600-4d8b-be6b-b5a16c8a58fe)
+
+### Graphs vs. Relational dbs
+
+In traditional relational databases, pathfinding across multiple connections is challenging. We must hypothesize table joins and create complex queries, which can be computationally expensive. Moreover, because relational databases lack inherent awareness of connections between data points, traversing unknown or numerous relationships adds to the complexity and processing time. 
+
+In contrast, graph databases like Neo4j are designed to handle paths and relationships directly, often achieving the same result with a few lines of code, making them ideal for finding optimal paths.
+
+### Performance advantages of graph dbs
+
+Neo4j and similar graph databases use a concept called *Index-free Adjacency*, where each node knows all incoming and outgoing relationships. This design avoids costly table joins, allowing traversals through nodes via efficient in-memory pointer chasing. As a result, query performance remains consistent, regardless of data size, which isn’t the case with relational databases that slow down as they grow. This feature is particularly advantageous for real-time pathfinding and large datasets.
+
+### Applications of graph pathfinding
+
+- logistics and routing: determining the most efficient delivery routes
+- infrastructure management: mapping out the best paths for utilities and road planning
+- social and networking: identifying the shortest paths to make new contacts or connections
+- web link analysis: finding optimal link structures for SEO or content discovery
+
+### Shortest paths with Cypher
+
+**Unweighted shortest path**
+
+Cypher supports the calculation of the shortest unweighted path between a pair of nodes with the shortestPath() function. In an unweighted path, the traversal of each relationship has an identical cost, so the shortest path between two nodes will always be the sum of the total relationships in a path between them.
+
+The shortestPath() function expects a definition of a Cypher path between an origin and destination node. The calculation is then made by traversing through the graph from both nodes, finding where the two paths meet in the middle. The traversal is breadth first, meaning that all relationships from a node will be followed before traversing further through the graph.
+
+The state of the traversal is held in memory, pruning or removing paths that are longer than the current shortest path as it goes. The overall shortest path is calculated as the shortest path length between the two nodes, or in other words, the smallest number of relationships in the path.
+
+The output of the function is the single shortest path between the two nodes.
+
+Example:
+
+```cypher
+MATCH (source:Airport {city:"Baltimore"}),
+      (target:Airport {city:"Frankfurt"}) 
+MATCH p = shortestPath((source)-[:HAS_ROUTE*1..10]->(target)) 
+RETURN p
+```
+
+It is good practice to specify a range of relationships (i.e. [:HAS_ROUTE*1..10]). This ensures that excessively long relationships are not queried or returned - improving query performance and constraining the result.
+
+![image](https://github.com/user-attachments/assets/86fdea09-091e-47f3-a92a-0337098e8c1f)
+
+For multiple relationships:
+
+```cypher
+MATCH (source:Airport {city:"Baltimore"}),
+      (target:Airport {city:"Frankfurt"})
+MATCH p=shortestPath((source)-[:HAS_ROUTE|CAN_WALK_TO|CAN_SWIM_TO*1..10]->(target))
+RETURN p
+```
+
+Or for finding all shortest paths:
+
+```cypher
+MATCH (source:Airport {city:"Baltimore"}),
+      (target:Airport {city:"Frankfurt"})
+MATCH p = allShortestPaths((source)-[:HAS_ROUTE*1..10]->(target))
+RETURN p
+```
+
+This will return all paths that contain the shortest number of relationships
+
+There was a simple challenge after this: How many flights you have to take to get from BNA to HKT?
+
+```cypher
+MATCH (source:Airport {iata: 'BNA'}), (target:Airport {iata: 'HKT'})  
+MATCH p=shortestPath((source)-[:HAS_ROUTE*]->(target))
+RETURN length(p) AS result
+```
+
+-> 3
+
+And also: How many options do you have to get from Nashville to Phuket with the smallest amount of flights?
+
+```cypher
+MATCH (source:Airport {iata: 'BNA'}), (target:Airport {iata: 'HKT'})  
+MATCH p=allShortestPaths((source)-[:HAS_ROUTE*]->(target))
+RETURN count(p) AS result
+```
+
+-> 206
+
+### Using the Graph Data Science (GDS) library for weighted relationships
+
+![image](https://github.com/user-attachments/assets/ed7c72ea-b0ee-4e48-b770-f214f100d117)
+
+In Neo4j, relationship properties in a property graph, called 'weights', represent traversal costs like distance, time, or financial cost. For example, a flight route between Paris and Lisbon may be cheaper via Madrid due to combined relationship weights, allowing applications to find the most cost-effective, time-saving, or eco-friendly routes.
+
+To use weighted shortest-path algorithms, Neo4j's GDS library requires creating an in-memory graph projection using `gds.graph.project()`. We can specify properties (like cost or distance) for relationships, allowing flexible path calculations.
+
+#### Common Weighted Shortest Path Algorithms
+
+1. Dijkstra’s algorithm: calculates the shortest path between two nodes, working only with positive weights. In Neo4j, it’s accessed with `gds.shortestPath.dijkstra.stream()` and returns nodes and costs in the path
+   
+2. Yen’s k-Shortest Paths Algorithm: an extension of Dijkstra’s, Yen’s algorithm finds multiple shortest paths (up to a specified k number). It ensures each discovered path is unique, ideal for alternative route suggestions
+
+These algorithms are valuable in scenarios where users seek efficient paths based on cost, distance, or other weighted factors, such as in transportation networks or logistics planning.
+
+Next, is a challenge using Djikstra's algorithm:
+
+First, a graph projection is created: 
+
+```cypher
+CALL gds.graph.project(
+  'routes',
+  'Airport',
+  'HAS_ROUTE',
+  {relationshipProperties:'distance'}
+);
+```
+
+Then, we can run the below to find the shortest **weighted** path:
+
+```cypher
+MATCH (source:Airport {iata: "BNA"}),
+      (target:Airport {iata:"HKT"})
+
+CALL gds.shortestPath.dijkstra.stream(
+  'routes',
+  {
+    sourceNode:source,
+    targetNode:target,
+    relationshipWeightProperty:'distance'
+  }
+) YIELD totalCost
+RETURN totalCost;
+```
+
+-> 9387.0
+
+Another challenge: In cases where the shortest path isn’t ideal (i.e. due to inconvenient timing or insufficient services), providing multiple route options can enhance user choice. Using Yen’s algorithm, we can calculate the three shortest paths between Nashville (BNA) and Phuket (HKT) in Neo4j. The Cypher query below identifies the total cost of the third shortest path by setting k:3
+
+```cypher
+MATCH (source:Airport {iata: "BNA"}),
+      (target:Airport {iata:"HKT"})
+
+CALL gds.shortestPath.yens.stream(
+  'routes',
+  {
+    sourceNode:source,
+    targetNode:target,
+    relationshipWeightProperty:'distance',
+    k:3
+  }
+) YIELD totalCost
+RETURN totalCost
+```
+
+## [Stream](https://youtu.be/Gi9fyRaI_kI)
+
+On stream I watched [GraphRAG Python Package: Accelerate GenAI with Knowledge Graphs](https://go.neo4j.com/WBR-241024-GenAI-Python-Package-EMEA_On-Demand.html), which covers neo4j's GraphRAG python package. It was around 30 mins, and after that I decided to check the [example notebook](https://github.com/neo4j-product-examples/graphrag-python-examples/blob/main/end-to-end-lupus.ipynb) by meself. However, instead of the provided pdf files I downloaded 3 random papers from sciencedirect. However, if you check the stream, it did not work for some reason. I believe because the code for the creation of the nodes and edges is generated through an LLM. And even though in the sample notebook they manage to get that working, I used different pdfs and also maybe because LLMs are not deterministic. But I could not generate nodes and edges for the knowledge graph for my 3 papers of choice. Still a cool package that will for sure become better (I think it was officially released recently)
+
+---
+
+I did not have anything special planned for day 300 😆 just normal study day
+
+That is all for today!
+
+See you tomorrow :)

From a110028f4e346882d0f0c52961cd91c09e5915eb Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 28 Oct 2024 23:57:18 +0900
Subject: [PATCH 479/638] Create 2024-10-28-day-301.md

---
 _posts/2024-10-28-day-301.md | 162 +++++++++++++++++++++++++++++++++++
 1 file changed, 162 insertions(+)
 create mode 100644 _posts/2024-10-28-day-301.md

diff --git a/_posts/2024-10-28-day-301.md b/_posts/2024-10-28-day-301.md
new file mode 100644
index 000000000..320018b48
--- /dev/null
+++ b/_posts/2024-10-28-day-301.md
@@ -0,0 +1,162 @@
+---
+layout: post
+title: (Day 301) LLM Engineer's Handbook
+categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+---
+
+# Hello :) Today is Day 301!
+A quick summary of today:
+* streamed and talked about translating pandas PD model code to pyspark
+* started reading a new promising book - LLM Engineer's Handbook
+
+## [Stream - translating a pd model from pandas to pyspark](https://youtu.be/0AiD5QxxP9I)
+
+[Here](https://colab.research.google.com/drive/1NJ_gCS1wXy8eUhS4BquJhVCeu1zaK6hi?usp=sharing) is a colab notebook with the code from today's work and stream. 
+
+Overall, it was a nice pyspark exercise, using pyspark's ML library, and also going from model coefficients to a scorecard to assess a customer's credit score (i.e. FICO). There are a couple extra steps that can be taken to improve the above notebook - adding a test dataset, using the model to assess the whole customer base' credit scores and overall PDs. The other models for calculating Expected Loss - EAD and LGD ~ I guess I will leave them for the next few days.
+
+## [RAG on the scikit-learn docs](https://youtu.be/7wXv2-6OM6U)
+
+I finally got time to check out probabl's last Friday stream that shows the functionality of a custom built [RAG tool](https://papers.probabl.ai/a-rag-from-scratch-to-query-the-scikit-learn-documentation) for the scikit-learn docs. I had already checked the paper and the repo some time ago, but Vincent (the instructor) showed the actual app as well as anyone can clone the open-source repo and run it locally - so this was nice 
+
+## [LLM Engineer's Handbook](https://learning.oreilly.com/library/view/llm-engineers-handbook/9781836200079/)
+
+I saw the author of this book advertising it on linkedin and it was getting lots of traction from other known llm/ml/data people so I had it bookmarked. I was refreshing the pages on my chrome tabs now and turns out that all of the chapter came out (last time I checked - last week, only the first 2-3 had come out). So ~ I'm excited to read it as it talks about LLM Ops.
+
+## Chapter 1 - Understanding the LLM Twin Concept and Architecture
+
+### Understanding the LLM Twin concept
+
+In a nutshell, An LLM twin is me in an AI version of me that has learned my writing style, voice, and personality
+
+To teach an AI to act like me, I could use:
+
+* LinkedIn posts and X threads: specialize the LLM in writing social media content
+* messages with friends and family: adapt the LLM to an unfiltered version of myself
+* academic papers and articles: calibrate the LLM in writing formal and educative content
+* code: specialize the LLM in implementing code as I would
+
+All these can be reduced to one core strategy: collecting my digital data (or some parts of it) and feeding it to an LLM using different algorithms. Ultimately, the LLM reflects the voice and style of the collected data
+
+Unfortunately, this raises many technical and moral issues. First, on the technical side, how can we access this data? Do we have enough digital data to project ourselves into an LLM? What kind of data would be valuable? Secondly, on the moral side, is it OK to do this in the first place? Do I want to create a copycat of myself? Will it write using my voice and personality, or just try to replicate it?
+
+_Building an LLM Twin is entirely moral._ The LLM will be fine-tuned only on my personal digital data, and it won't use other people's data. 
+
+### Planning the MVP of the LLM Twin product
+
+_An MVP is a version of a product that includes just enough features to draw in early users and test the viability of the product concept in the initial stages of development_
+
+An MVP is a powerful strategy because of the following reasons:
+
+* accelerated time-to-market: launch a product quickly to gain early traction
+* idea validation: test it with real users before investing in the full development of the product
+* market research: gain insights into what resonates with the target audience
+* risk minimization: reduces the time and resources needed for a product that might not achieve market success
+
+Features for the LLM twin MVP:
+
+* collect data from LinkedIn, Medium, Substack, and GitHub profiles 
+* fine-tune an open-source LLM using the collected data
+* populate a vector database (DB) using our digital data for RAG
+* create LinkedIn posts leveraging the following:
+  * user prompts
+  * RAG to reuse and reference old content
+  * new posts, articles, or papers as additional knowledge to the LLM
+* Have a simple web interface to interact with the LLM Twin and be able to do the following:
+  * configure our social media links and trigger the collection step
+  * send prompts or links to external resources
+
+### Building ML systems with feature/training/inference (FTI) pipelines
+
+**The problem with building ML systems**
+
+Common elements/problems to consider as an MLOps engineer:
+
+![image](https://github.com/user-attachments/assets/d5930c87-1dea-4598-b8bb-e227f608e3d6)
+
+ The critical question is this: How do we connect all these components into a single homogenous system? There are existing solutions
+
+ **The issue with previous solutions**
+
+The main issue with monolithic batch architectures in machine learning (ML) applications is that while they solve the training-serving skew by using the same code for feature creation in training and inference, they have limitations when scaling. These architectures lack feature reusability, require code refactoring for large data, and complicate sharing and modularizing tasks across teams. Transitioning to real-time processing is also difficult, as client applications must pass all state data, leading to inefficiencies and errors.
+
+Real-time architectures further complicate predictions, as clients need full access to user state data. For example, recommending movies for a user requires transferring not just an ID but a complete user profile. This setup is complex, inefficient, and unsuitable for client applications. Although platforms like GCP offer a scalable production-ready architecture, it is complex, making it challenging to scale gradually or understand without advanced experience in ML production.
+
+**The solution – ML pipelines for ML systems**
+
+The FTI pipeline solution simplifies ML systems by dividing them into three core pipelines: feature, training, and inference. This approach mirrors the structure of traditional software layers (database, business logic, UI), creating a straightforward framework for building ML systems. Each pipeline—feature, training, and inference—serves a specific role and can be independently managed, scaled, or developed using different technologies, making the design flexible.
+
+![image](https://github.com/user-attachments/assets/72b4ede0-1801-4c08-bae7-99d5beb1febf)
+
+1. Feature pipe: processes raw data into features and stores them in a feature store, ensuring versioning and consistency across training and inference
+2. Training pipe: uses stored features to train models, which are then stored in a model registry with metadata, tracking versions and the data used for training
+3. Inference pipe: combines features from the feature store and models from the registry to generate predictions, supporting both batch and real-time modes. Versioning across features, labels, and models allows for controlled updates or rollbacks
+
+This modular design avoids issues like training-serving skew and supports easy adjustments to models and features.
+
+**Benefits of the FTI architecture**
+
+* as there are just three components, it is intuitive to use and easy to understand
+* each component can be written into its tech stack, so we can quickly adapt them to specific needs, such as big or streaming data. Also, it allows us to pick the best tools for the job
+* as there is a transparent interface between the three components, each one can be developed by a different team (if necessary), making the development more manageable and scalable
+* every component can be deployed, scaled, and monitored independently
+
+In most cases, the system will include more than just 3 pipelines. For example, the feature pipeline can be composed of a service that computes the features and one that validates the data. Also, the training pipeline can be composed of the training and evaluation components.
+
+### Designing the system architecture of the LLM Twin
+
+**Listing the technical details of the LLM Twin architecture**
+
+Data 
+
+- Collect data from LinkedIn, Medium, Substack, and GitHub automatically and on a schedule
+- Standardize the crawled data and store it in a data warehouse
+- Clean the raw data
+- Create instruction datasets for fine-tuning an LLM
+- Chunk and embed the cleaned data, then store the vectorized data in a vector database for RAG
+
+Training Tasks
+
+- Fine-tune LLMs of various sizes (7B, 14B, 30B, or 70B parameters)
+- Fine-tune on instruction datasets of multiple sizes
+- Switch between LLM types (e.g.Mistral, Llama, GPT)
+- Track and compare experiments
+- Test potential production LLM candidates before deployment
+- Automatically start training when new instruction datasets are available
+
+Inference Code Properties
+
+- Provide a REST API for client interactions with the LLM Twin
+- Enable real-time access to the vector database for RAG
+- Support inference with LLMs of various size
+- Autoscale based on user requests
+- Automatically deploy LLMs that pass evaluation
+
+LLMOps Features
+
+- Instruction dataset versioning, lineage, and reusability
+- Model versioning, lineage, and reusability
+- Experiment tracking
+- Continuous training, continuous integration, and continuous delivery (CT/CI/CD)
+- Prompt and system monitoring
+
+**How to design the LLM Twin architecture using the FTI pipeline design**
+
+![image](https://github.com/user-attachments/assets/d8fa8d83-99b8-4f81-afd0-7014182d4240)
+
+This pipeline involves four stages:
+
+1. data collection pipeline: collects text data (articles from Medium/Substack, posts from LinkedIn, and code from GitHub) via ETL (Extract, Transform, Load), stored in a NoSQL database as the data warehouse. Data is categorized by type (article, post, code) for modularity
+2. feature pipeline: processes collected data by cleaning, chunking, and embedding, storing processed data in a vector database (logical feature store) for model fine-tuning and retrieval-augmented generation (RAG)
+3. training pipeline: fine-tunes the LLM using artifacts from the feature store. Experiments determine optimal hyperparameters, with the final model registered for production and tested rigorously before deployment
+4. inference pipeline: uses the fine-tuned LLM and vector DB for RAG-based query responses through a REST API. It includes prompt monitoring for quality control and automated alerts based on set criteria.
+
+---
+
+Wow, this book seems *extremely* promising. The authors say that, for learning purposes, they share their data. And that is nice because I do not have much public data on social media haha. In my case I would have just used my personal chats maybe, and also this blog which is pretty substantial in size to be fair.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 42c93223478001d992e45bb61838fea254f58e0a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 29 Oct 2024 22:00:05 +0900
Subject: [PATCH 480/638] Create 2024-10-29-day-302.md

---
 _posts/2024-10-29-day-302.md | 222 +++++++++++++++++++++++++++++++++++
 1 file changed, 222 insertions(+)
 create mode 100644 _posts/2024-10-29-day-302.md

diff --git a/_posts/2024-10-29-day-302.md b/_posts/2024-10-29-day-302.md
new file mode 100644
index 000000000..97ce0ddbf
--- /dev/null
+++ b/_posts/2024-10-29-day-302.md
@@ -0,0 +1,222 @@
+---
+layout: post
+title: (Day 302) Data engineering pipeline from the LLM Engineer's handbook
+categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+---
+
+# Hello :) Today is Day 302!
+A quick summary of today:
+* covered ch. 2 and 3 of the book
+* streamed
+* small update on the company reviewer project
+
+## LLM Engineer's handbook Chapter 2 - Tooling and Installation
+
+As the name of the chapter suggests, this was information about how to setup the [code](https://github.com/PacktPublishing/LLM-Engineers-Handbook) locally.
+
+After some environment setup, I ran a ZenML pipeline using poetry, and in the UI we can see:
+
+![image](https://github.com/user-attachments/assets/f851d2c2-cde0-41f2-9b07-5b07385d95b4)
+
+A run looks like:
+
+![image](https://github.com/user-attachments/assets/414fe7b0-2337-4121-a8f9-bce313e8692e)
+
+### MLOPs and LLMOps tools:
+
+* HuggingFace is used as a model registry
+* ZenML is used as an orchestrator and to save artifacts and metadata
+* Comet ML is used for experiment tracking
+* Opik is used for prompt monitoring
+
+### Databases
+
+* MongoDB as a noSQL db
+* Qdrant as a vector db
+
+### AWS
+
+Optionally, I can use AWS SageMaker for training and inference
+
+## Chapter 3 - Data Engineering
+
+### Designing the LLM Twin’s data collection pipeline
+
+![image](https://github.com/user-attachments/assets/2a1c982b-335a-44cf-a29d-50b9ae318d9b)
+
+* input: a list of links and their associated user (the author)
+* output: a list of raw documents stored in the NoSQL data warehouse
+
+The ETL pipeline will detect the domain of each link, based on which it will call a specialized crawler
+
+![image](https://github.com/user-attachments/assets/73f345ce-5d25-426b-a4cb-dc2465f144d1)
+
+* Medium crawler: used for data from Medium. It outputs an article document. It logs in Medium and crawls the HTML of the article, then, extracts, cleans, and normalises the text from the HTML and loads the standardised text of the article into mongodb (our nosql data warehouse)
+* custom article crawler: similar to the Merdium crawler, but more generic. It doesn't do any log ins, it just gathers all the HTML from a particular link
+* github crawler: collects data from github. It outputs a repository document. It clones the repository, parses the repository file tree, cleans and normalises the files, and loads them to the db
+* linkedin crawler: collects data from linkedin. It outputs multiple post documents. It logs in linkedin, navigates to the user's feed, and crawls all the user's latest posts. For each post, it extracts its HTML, cleans and normalises it, and loads it to the db
+
+**How is the ETL process connected to the feature pipeline?**
+
+The feature pipeline ingests the raw data from the MongoDB data warehouse, cleans it further, processes it into features, and stores it in the Qdrant vector DB to make it accessible for the LLM training and inference pipelines. The ETL process is independent of the feature pipeline. The two pipelines communicate with each other strictly through the MongoDB data warehouse. Thus, the data collection pipeline can write data for MongoDB, and the feature pipeline can read from it independently and on different schedules.
+
+**Why is MongoDB used as a data warehouse?**
+
+Using a transactional database, such as MongoDB, as a data warehouse is uncommon. However, in this use case, we are working with small amounts of data, which MongoDB can handle. Even if we plan to compute statistics on top of our MongoDB collections, it will work fine at the scale of our LLM Twin’s data (hundreds of documents). By mainly working with unstructured text, selecting a NoSQL database that doesn’t enforce a schema will make the development easier and faster. However, when working with big data (millions of documents or more), using a dedicated data warehouse such as Snowflake or BigQuery will be ideal.
+
+#### Implementing the LLM Twin’s data collection pipeline
+
+Below is the digital_data_etl pipeline:
+
+```python
+from zenml import pipeline
+from steps.etl import crawl_links, get_or_create_user
+
+@pipeline
+def digital_data_etl(user_full_name: str, links: list[str]) -> str:
+    user = get_or_create_user(user_full_name)
+    last_step = crawl_links(user=user, links=links)
+
+    return last_step.invocation_id
+```
+
+Diving deeper ~ we will explore each of the two steps in the above pipeline
+
+Firstly, `get_or_create_user`
+
+```python
+from loguru import logger
+from typing_extensions import Annotated
+from zenml import get_step_context, step
+
+from llm_engineering.application import utils
+from llm_engineering.domain.documents import UserDocument
+
+@step
+def get_or_create_user(user_full_name: str) -> Annotated[UserDocument, "user"]:
+    logger.info(f"Getting or creating user: {user_full_name}")
+
+    first_name, last_name = utils.split_user_full_name(user_full_name)
+
+    user = UserDocument.get_or_create(first_name=first_name, last_name=last_name)
+
+    step_context = get_step_context()
+    step_context.add_output_metadata(output_name="user", metadata=_get_metadata(user_full_name, user))
+
+    return user
+
+
+def _get_metadata(user_full_name: str, user: UserDocument) -> dict:
+    return {
+        "query": {
+            "user_full_name": user_full_name,
+        },
+        "retrieved": {
+            "user_id": str(user.id),
+            "first_name": user.first_name,
+            "last_name": user.last_name,
+        },
+    }
+```
+
+Next is the `crawl_links` step:
+
+```python
+from urllib.parse import urlparse
+
+from loguru import logger
+from tqdm import tqdm
+from typing_extensions import Annotated
+from zenml import get_step_context, step
+
+from llm_engineering.application.crawlers.dispatcher import CrawlerDispatcher
+from llm_engineering.domain.documents import UserDocument
+
+@step
+def crawl_links(user: UserDocument, links: list[str]) -> Annotated[list[str], "crawled_links"]:
+    dispatcher = CrawlerDispatcher.build().register_linkedin().register_medium().register_github()
+
+    logger.info(f"Starting to crawl {len(links)} link(s).")
+
+    metadata = {}
+    successfull_crawls = 0
+    for link in tqdm(links):
+        successfull_crawl, crawled_domain = _crawl_link(dispatcher, link, user)
+        successfull_crawls += successfull_crawl
+
+        metadata = _add_to_metadata(metadata, crawled_domain, successfull_crawl)
+
+    step_context = get_step_context()
+    step_context.add_output_metadata(output_name="crawled_links", metadata=metadata)
+
+    logger.info(f"Successfully crawled {successfull_crawls} / {len(links)} links.")
+
+    return links
+
+
+def _crawl_link(dispatcher: CrawlerDispatcher, link: str, user: UserDocument) -> tuple[bool, str]:
+    crawler = dispatcher.get_crawler(link)
+    crawler_domain = urlparse(link).netloc
+
+    try:
+        crawler.extract(link=link, user=user)
+
+        return (True, crawler_domain)
+    except Exception as e:
+        logger.error(f"An error occurred while crowling: {e!s}")
+
+        return (False, crawler_domain)
+
+
+def _add_to_metadata(metadata: dict, domain: str, successfull_crawl: bool) -> dict:
+    if domain not in metadata:
+        metadata[domain] = {}
+    metadata[domain]["successful"] = metadata.get(domain, {}).get("successful", 0) + successfull_crawl
+    metadata[domain]["total"] = metadata.get(domain, {}).get("total", 0) + 1
+
+    return metadata
+```
+
+I will go over the code on stream (link below) because there is a lot of code related to all the crawlers and interacting with mongodb.
+
+After running the data etl pipeline, we can see the data in mongodb. VSCode has a nice mongodb extension that allows to view the base inside vs code:
+
+![image](https://github.com/user-attachments/assets/fa8cbe55-db52-41f2-88b8-5499bef6bf86)
+
+And I can see I have two users (because I ran the pipeline for the two authors' data)
+
+All of the book's code can be found on the book's [github repo](https://github.com/PacktPublishing/LLM-Engineers-Handbook)
+
+
+## Company reviewer
+
+Today, I gave a brief report on my progress for the company reviewer LLM fine-tuning as tomorrow he invited me to meet with the client company. Below is a summary of the email I sent to him. 
+
+**As for OpenAI’s gpt4o-mini fine-tuning**
+
+| Model               | Text RougeLSum | Ratings RMSE |
+|---------------------|----------------|--------------|
+| gpt4o-mini          | 0.8768         | 0.5924       |
+| gpt4o-mini-fewshot  | 0.8499         | 0.6797       |
+
+ 
+I suspect a bit of the decrease in performance with the few-shot is due to the larger prompt and maybe it could be improved with some different prompts. I might need some extra credits because these fine-tunings and evaluations are draining my account haha. I have seen online that gpt4o-mini’s performance is very close if not the same to gpt4o, but it might also be worth exploring gpt4o if you think that is something of interest.
+ 
+**As for open-source model LoRA fine-tuning**
+
+I have been facing some issues related to the LoRA fine-tuning library I was using ([unsloth](https://github.com/unslothai/unsloth/issues/1181#issuecomment-2442981841)). I reported the problem to the library developers, it got fixed but it has problems again and I have reported it again just this morning. When there were no errors I got to fine-tune two open-source models - Llama3.2 and Llama3.1 but the responses were not good – even just doing manual reviews I noticed the text did not make sense and was not related at all to our data. This is something that I think requires a more powerful GPU, or to purchase colab GPU hours as the free T4 GPUs are not suitable for the bigger data (bigger compared to the movie review data I was using). As the a single case is bigger I need to increase the input context of the open-source models like Llama, and Qwen, but this increase in input context requires more compute power.
+ 
+**As for prompt tuning**
+
+I found [this notebook](https://huggingface.co/learn/cookbook/en/prompt_tuning_peft) that is an implementation and example of doing prompt tuning. However, whenever I run it on my laptop, or colab’s free CPU/GPU or Kaggle’s free GPU, I run out of memory – which I guess requires me to have a more powerful PC in order to do it. Or maybe this is just not a good implementation of doing PEFT prompt tuning ~ 
+
+## [Stream](https://youtu.be/UqJuQ1yHYX4)
+
+I streamed a bit late today as I had some university engagements. I covered some of the code from chapter 3 of LLM Engineer's Handbook. 
+
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f24bc90c5e2e746eb19b6127fdf2f5f3c9d867e6 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 30 Oct 2024 23:05:57 +0900
Subject: [PATCH 481/638] Create 2024-10-30-day-303.md

---
 _posts/2024-10-30-day-303.md | 164 +++++++++++++++++++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 _posts/2024-10-30-day-303.md

diff --git a/_posts/2024-10-30-day-303.md b/_posts/2024-10-30-day-303.md
new file mode 100644
index 000000000..1f5e651e6
--- /dev/null
+++ b/_posts/2024-10-30-day-303.md
@@ -0,0 +1,164 @@
+---
+layout: post
+title: (Day 303) Using my professor's A6000 GPU for LLM fine-tuning
+categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+---
+
+# Hello :) Today is Day 303!
+A quick summary of today:
+* using a powerful GPU to fine-tune models
+* reading more of the LLM engineer's handbook
+
+## Using my professor's A6000 GPU
+
+![image](https://github.com/user-attachments/assets/9e82102f-1d5a-405d-9d61-1bd531658bca)
+
+Today, I went to the professor who is leading the whole project that involves other parts, but mine is developing the company reviewer LLM which I have been talking about. Up until now, I have been using the resources available online, but today because the client came for a meeting - I went to meet them and also use my professor's 2nd pc which has a 49GB A6000 GPU 🤯
+
+I had to setup conda, and unsloth. For unsloth I ended up unstalling WSL as unsloth is not really supported for Windows (there is a way to do it but it is not that optimal). No proble, I installed WSL with Ubuntu and set it all up. 
+
+Because I am using WSL I faced some weird issues. [Turns out other people using WSL face it as well](https://github.com/unslothai/unsloth/issues/338). There is a workaround but it makes it so that I am not utilising unsloth's full potential. The issue is quite recent, but there have been no updates on fixing it. Nonetheless, I first ran a llama 3.1 8B with 4K context and the results were subpar just by looking at them. This took 1 hour, and if I were using T4 - it would have taken 7-8hrs 🤯
+
+After the above did not succeed, I started a 8K context llama 3.1 8b. It trained for ~1.5hrs and the results look promising. I saved the results in a df and just by doing a manual review (human evaluation) they seem good and follow the output format that we need.
+
+Here are the results (alongside the gpt ones):
+
+| Model               | Text RougeLSum | Ratings RMSE |
+|---------------------|----------------|--------------|
+| gpt4o-mini          | 0.8768         | 0.5924       |
+| gpt4o-mini-fewshot  | 0.8499         | 0.6797       |
+| llama-3.1-8b-8K     | 0.7827         | 0.8580       |
+
+I call it 8K because that's the max sequence length I used when training the model. [Here](https://huggingface.co/divakaivan/llama31-8b-KR-8192-company-reviewer) is a link to it on HuggingFace.
+
+During the models' training time, I read the below chapter.
+
+## LLM Engineer's handbook - Chapter 4 - RAG Feature Pipeline
+
+The book explains a bit about RAG and its benefits, and shows the below 'vanilla' framework:
+
+* ingestion pipeline: a batch or streaming pipeline used to populate the vector DB
+* retrieval pipeline: a module that queries the vector DB and retrieves relevant entries to the user’s input
+* generation pipeline: the layer that uses the retrieved data to augment the prompt and an LLM to generate answers
+
+_How are these 3 connected?_
+
+1. On the backend side, the ingestion pipeline runs either on a schedule or constantly to populate the vector DB with external data
+2. On the client side, the user asks a question
+3. The question is passed to the retrieval module, which preprocesses the user’s input and queries the vector DB
+4. The generation pipelines use a prompt template, user input, and retrieved context to create the prompt
+5. The prompt is passed to an LLM to generate the answer
+6. The answer is shown to the user
+
+![image](https://github.com/user-attachments/assets/525c7f42-984b-4af0-b4b4-bbacfa8ea408)
+
+### An overview of advanced RAG
+
+The 'vanilla' design can be optimised:
+
+* pre-retrieval: this focuses on how to structure and preprocess your data for data indexing optimizations as well as query optimizations
+* retrieval: this revolves around improving the embedding models and metadata filtering to improve the vector search step
+* post-retrieval: this mainly targets different ways to filter out noise from the retrieved documents and compress the prompt before feeding it to an LLM for answer generation
+
+The 3 stages of advanced RAG apps:
+
+![image](https://github.com/user-attachments/assets/65af2474-e48e-43cb-959f-45d36762950c)
+
+#### The pre-retrieval steps are performed in two different ways:
+
+* data indexing: it is part of the RAG ingestion pipeline. It is mainly implemented within the cleaning or chunking modules to preprocess the data for better indexing
+* query optimization: the algorithm is performed directly on the user’s query before embedding it and retrieving the chunks from the vector DB
+
+Data indexing techniques focus on better preprocessing and structuring the data to improve retrieval efficiency, these include:
+
+* sliding window
+* enhancing data granularity: involves data cleaning, such as removing irrelevant details, verifying accuracy, and updating outdated information. Clean, accurate datasets lead to sharper retrieval
+* metadata: adding tags like dates, URLs, and external IDs helps filter results efficiently during retrieval
+* optimising index structures: uses varied chunk sizes and multi-indexing strategies to refine retrieval precision
+* small-to-big: decouples the chunk size for embedding from the larger context in the final prompt. Small chunks improve retrieval accuracy, while the larger context adds depth to the generated response
+
+The intuition behind this is that if we use the whole text for computing the embedding, we might introduce too much noise, or the text could contain multiple topics, which results in a poor overall semantic representation of the embedding.
+
+On the query optimization side, we can leverage techniques such as query routing, query rewriting, and query expansion to refine the retrieved information for the LLM further:
+
+* query routing: based on the user’s input, we might have to interact with different categories of data and query each category differently. Query rooting is used to decide what action to take based on the user’s input, similar to if/else statements. Still, the decisions are made solely using natural language instead of logical statements
+
+![image](https://github.com/user-attachments/assets/699f6829-922c-49d6-9652-da938ab53e3c)
+
+* query rewriting: sometimes, the user’s initial query might not perfectly align with the way your data is structured. Query rewriting tackles this by reformulating the question to match the indexed information better
+
+* Hypothetical document embeddings (HyDE): this technique involves having an LLM create a hypothetical response to the query. Then, both the original query and the LLM’s response are fed into the retrieval stage
+* query expansion: this approach aims to enrich the user’s question by adding additional terms or concepts, resulting in different perspectives of the same initial question. For example, when searching for 'disease', we can leverage synonyms and related terms associated with the original query words and also include 'illnesses” or “ailments'
+* self-query: the core idea is to map unstructured queries into structured ones. An LLM identifies key entities, events, and relationships within the input text. These identities are used as filtering parameters to reduce the vector search space (i.e. identify cities within the query, for example, 'Paris' and add it to our filter to reduce your vector search space)
+
+#### Retrieval
+
+The retrieval step can be optimized in two fundamental ways:
+
+* improving the embedding models used in the RAG ingestion pipeline to encode the chunked documents and, at inference time, transform the user’s input
+* leveraging the DB’s filter and search features: this step will be used solely at inference time when you have to retrieve the most similar chunks based on user input
+  * hybrid search: combines vector and keyword-based searches. It uses keyword search for exact matches and vector search for broader semantic similarities. The balance between the two is controlled by a parameter (often called alpha), and results from each method are normalized and combined
+  * filtered vector search: uses metadata filtering with vector search by applying specific keyword filters to metadata either before or after the vector search, reducing the search space without performing a separate keyword-based search
+
+#### Post-retrieval
+
+The post-retrieval optimizations are solely performed on the retrieved data to ensure that the LLM’s performance is not compromised by issues such as limited context windows or noisy data. This is because the retrieved context can sometimes be too large or contain irrelevant information, both of which can distract the LLM.
+
+Two popular methods performed at the post-retrieval step are:
+
+* prompt compression: eliminate unnecessary details while keeping the essence of the data
+* re-ranking: use a cross-encoder ML model to give a matching score between the user’s input and every retrieved chunk. The retrieved items are sorted based on this score. Only the top N results are kept as the most relevant. This works because the re-ranking model can find more complex relationships between the user input and some content than a simple similarity search. However, we can’t apply this model at the initial retrieval step because it is costly. That is why a popular strategy is to retrieve the data using a similarity distance between the embeddings and refine the retrieved information using a re-raking model
+
+_Bi-encoder (the standard embedding model) versus cross-encoder_
+
+![image](https://github.com/user-attachments/assets/0ad719bb-b3ec-4912-bd53-075c380cfd1b)
+
+_The re-ranking algorithm_
+
+![image](https://github.com/user-attachments/assets/f5470af9-914b-4dc6-9dd9-8f75b44b6166)
+
+### Exploring the LLM Twin’s RAG feature pipeline architecture
+
+Any RAG system is split into two independent components:
+
+* the ingestion pipeline takes in raw data, cleans, chunks, embeds, and loads it into a vector DB
+* the inference pipeline queries the vector DB for relevant context and ultimately generates an answer by levering an LLM
+
+Going back to the main goal - the LLM twin, the next step is to create a **feature store**
+
+The feature store will be the central access point for all the features used within the training and inference pipelines. The training pipeline will use the cleaned data from the feature store (stored as artifacts) to fine-tune LLMs. The inference pipeline will query the vector DB for chunked documents for RAG. That is why we are designing a feature pipeline and not only a RAG ingestion pipeline. In practice, the feature pipeline contains multiple subcomponents, one of which is the RAG logic.
+
+Here is an overview of the architecture of the RAG feature pipeline:
+
+![image](https://github.com/user-attachments/assets/485af829-2da8-475a-8b3f-5aa22523c058)
+
+Batch vs Streaming pipelines are discussed and a nice graph is shown for tools depending on the use case:
+
+![image](https://github.com/user-attachments/assets/8d46cbb7-2fee-4558-8d64-7760204374ff)
+
+In this case, the pipeline will be batch, as new data does not require immediate data processin, and for simplicity. 
+
+#### Change data capture: syncing the data warehouse and feature store
+
+Data is constantly changing, which can result in dbs, data lakes, data warehouses, and feature stores getting out of sync. Change data capture (CDC) is a strategy that allows you to optimally keep two or more data storage types in sync without computing and I/O overhead. It captures any CRUD operation done on the source DB and replicates it on a target db.
+
+The syncing issues also apply when building a feature pipeline. One key design choice concerns how to sync the data warehouse with the feature store to have data fresh enough for our particular use case.
+
+In this LLM Twin use case, a naive batch processing pipeline reads and processes all raw data from the data warehouse in batches, updating or inserting records in the Qdrant vector db. This approach works well for smaller datasets but raises scalability concerns as data volumes grow. Specific issues include handling millions of records, reflecting deletions from the data warehouse, and processing only new or updated items.
+
+A Change Data Capture (CDC) pattern addresses these issues through two main approaches:
+
+- push: the source DB actively identifies and sends changes to target systems for near-instant updates, with messaging systems buffering data if targets are unavailable
+- pull: target systems periodically request changes from a passive source DB, better for large-scale transfers where immediate updates aren't required but also requiring messaging systems to avoid data loss
+
+Push is best for real-time needs, while pull fits non-real-time, large-scale data updates. To detect changes there are timestamp, trigger and log-based approaches.
+
+### Implementing the LLM Twin’s RAG feature pipeline
+
+This step I wanted to cover on stream but I was in the lab until ~9pm so I could not stream today.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 5f64298b381f7f3541aafeb5c1ed8c66047095e1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 31 Oct 2024 23:22:53 +0900
Subject: [PATCH 482/638] Create 2024-10-31-day-304.md

---
 _posts/2024-10-31-day-304.md | 703 +++++++++++++++++++++++++++++++++++
 1 file changed, 703 insertions(+)
 create mode 100644 _posts/2024-10-31-day-304.md

diff --git a/_posts/2024-10-31-day-304.md b/_posts/2024-10-31-day-304.md
new file mode 100644
index 000000000..c81e01773
--- /dev/null
+++ b/_posts/2024-10-31-day-304.md
@@ -0,0 +1,703 @@
+---
+layout: post
+title: (Day 304) LLMs + some dbt
+categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+---
+
+# Hello :) Today is Day 304!
+A quick summary of today:
+* reading more of the LLM Engineer's handbook
+* training a 16k llama 3.2 llm on a6000
+* doing dbt fundamentals course
+
+## Re-reading the explanations about Implementing the LLM Twin’s RAG feature pipeline
+
+This is the 2nd part of the 3rd chapter of the LLM Engineer's handbook. It introduces some not so familiar with me concepts so I wanted to re-read them before covering them on stream and checking my understanding.
+
+These are the concepts they cover:
+
+* settings: using `pydantic`'s BaseSettings, a global settings class that loads sensitive or non-sensitive variables from a .env file is used.
+* ZenML pipeline and steps
+
+### ZenML pipeline and steps
+
+The pipeline performs the five core phases of RAG: extracting raw docs, cleaning, chinking, embedding, and loading them to the logical feature store
+
+```python
+from zenml import pipeline
+from llm_engineering.interfaces.orchestrator.steps import feature_engineering as fe_steps
+
+@pipeline
+def feature_engineering(author_full_names: list[str]) -> None:
+    raw_documents = fe_steps.query_data_warehouse(author_full_names)
+    cleaned_documents = fe_steps.clean_documents(raw_documents)
+    last_step_1 = fe_steps.load_to_vector_db(cleaned_documents)
+    embedded_documents = fe_steps.chunk_and_embed(cleaned_documents)
+    last_step_2 = fe_steps.load_to_vector_db(embedded_documents)
+
+    return [last_step_1.invocation_id, last_step_2.invocation_id]
+```
+
+In ZenML it looks like:
+
+![image](https://github.com/user-attachments/assets/f762b664-e96d-4d17-b883-7f7e49ea32a9)
+
+And each elipse provides metadata to the type of docs from that state and how to load them as artifacts
+
+The pipeline accepts `author_full_names`. This data is injected through a yaml file at runtime, and we don't need to modify the code for different input values, we just need to adjust the yaml file and pass it.
+
+#### Querying  the data warehouse
+
+```python
+# other imports
+from zenml import get_step_context, step
+
+@step
+def query_data_warehouse(
+    author_full_names: list[str],
+) -> Annotated[list, "raw_documents"]:
+
+    documents = []
+    authors = []
+    for author_full_name in author_full_names:
+        logger.info(f"Querying data warehouse for user: {author_full_name}")
+        first_name, last_name = utils.split_user_full_name(author_full_name)
+        logger.info(f"First name: {first_name}, Last name: {last_name}")
+        user = UserDocument.get_or_create(first_name=first_name, last_name=last_name)
+        authors.append(user)
+        results = fetch_all_data(user)
+        user_documents = [doc for query_result in results.values() for doc in query_result]
+        documents.extend(user_documents)
+    step_context = get_step_context()
+    step_context.add_output_metadata(output_name="raw_documents", metadata=_get_metadata(documents))
+
+    return documents
+```
+
+1. it tries to get or create a UserDocument instance using the 1st and last name; if a user doesn't exist, an error is thrown
+2. it fetches all the raw data for the user from the data warehouse and extends the documents list to include these user docs
+3. it computes a descriptive metadata dictionary logged and tracked in ZenML
+
+The fetch function leverages a thread pool that runs each query on a different thread. As we have multiple data categories, we have to make a different query for the articles, posts, and repositories, as they are stored in different collections. Each query calls the data warehouse, which is bounded by the network I/O and data warehouse latency, not by the machine’s CPU. Thus, by moving each query to a different thread, we can parallelize them. Ultimately, instead of adding the latency of each query as the total timing, the time to run this fetch function will be the max between all the calls.
+
+In Python, we want to parallelize things with processes only when the operations are CPU or memory-bound because the GIL(Global Interpreter Lock) affects them. Each process has a different GIL. Thus, parallelizing  computing logic, such as processing a batch of documents or images already loaded in memory, isn’t affected by Python’s GIL limitations.
+
+```python
+def fetch_all_data(user: UserDocument) -> dict[str, list[NoSQLBaseDocument]]:
+
+    user_id = str(user.id)
+    with ThreadPoolExecutor() as executor:
+        future_to_query = {
+            executor.submit(__fetch_articles, user_id): "articles",
+            executor.submit(__fetch_posts, user_id): "posts",
+            executor.submit(__fetch_repositories, user_id): "repositories",
+        }
+        results = {}
+        for future in as_completed(future_to_query):
+            query_name = future_to_query[future]
+            try:
+                results[query_name] = future.result()
+            except Exception:
+                logger.exception(f"'{query_name}' request failed.")
+                results[query_name] = []
+
+    return results
+```
+
+The `_get_metadata()` function takes a list of queries docs and authors and counts the number of them relative to each data category:
+
+```python
+def _get_metadata(documents: list[Document]) -> dict:
+    metadata = {
+        "num_documents": len(documents),
+    }
+    for document in documents:
+        collection = document.get_collection_name()
+        if collection not in metadata:
+            metadata[collection] = {}
+        if "authors" not in metadata[collection]:
+            metadata[collection]["authors"] = list()
+        metadata[collection]["num_documents"] = metadata[collection].get("num_documents", 0) + 1
+        metadata[collection]["authors"].append(document.author_full_name)
+    for value in metadata.values():
+        if isinstance(value, dict) and "authors" in value:
+            value["authors"] = list(set(value["authors"]))
+
+    return metadata
+```
+
+This is the metadata that is exposed in the ZenML UI. 
+
+#### Cleaning the documents
+
+Here, we iterate over the docs and a `CleaningDispatcher` who knows what cleaning logic to apply based on the data category is used. 
+
+```python
+@step
+def clean_documents(
+    documents: Annotated[list, "raw_documents"],
+) -> Annotated[list, "cleaned_documents"]:
+
+    cleaned_documents = []
+    for document in documents:
+        cleaned_document = CleaningDispatcher.dispatch(document)
+        cleaned_documents.append(cleaned_document)
+    step_context = get_step_context()
+    step_context.add_output_metadata(output_name="cleaned_documents", metadata=_get_metadata(cleaned_documents))
+
+    return cleaned_documents
+```
+
+#### Chunk and embed the cleaned documents
+
+Similar to the cleaning dispatcher, the chinking and embedding logic is delegated to a dispatcher who knows how to handle each data category. 
+
+```python
+@step
+def chunk_and_embed(
+    cleaned_documents: Annotated[list, "cleaned_documents"],
+) -> Annotated[list, "embedded_documents"]:
+
+    metadata = {"chunking": {}, "embedding": {}, "num_documents": len(cleaned_documents)}
+    embedded_chunks = []
+    for document in cleaned_documents:
+        chunks = ChunkingDispatcher.dispatch(document)
+        metadata["chunking"] = _add_chunks_metadata(chunks, metadata["chunking"])
+        for batched_chunks in utils.misc.batch(chunks, 10):
+            batched_embedded_chunks = EmbeddingDispatcher.dispatch(batched_chunks)
+            embedded_chunks.extend(batched_embedded_chunks)
+    metadata["embedding"] = _add_embeddings_metadata(embedded_chunks, metadata["embedding"])
+    metadata["num_chunks"] = len(embedded_chunks)
+    metadata["num_embedded_chunks"] = len(embedded_chunks)
+    step_context = get_step_context()
+    step_context.add_output_metadata(output_name="embedded_documents", metadata=metadata)
+
+    return embedded_chunks
+```
+
+#### Loading the documents to the vector DB
+
+As each article, post, or code repository sits in a different collection inside the vector DB, we have to group all the documents based on their data category. Then, we load each group in bulk in the Qdrant vector db:
+
+```python
+@step
+def load_to_vector_db(
+    documents: Annotated[list, "documents"],
+) -> None:
+
+    logger.info(f"Loading {len(documents)} documents into the vector database.")
+    grouped_documents = VectorBaseDocument.group_by_class(documents)
+    for document_class, documents in grouped_documents.items():
+        logger.info(f"Loading documents into {document_class.get_collection_name()}")
+        for documents_batch in utils.misc.batch(documents, size=4):
+            try:
+                document_class.bulk_insert(documents_batch)
+            except Exception:
+                return False
+
+    return True
+```
+
+### Pydantic domain entities
+
+To some extent, in implementing the LLM Twin, the domain-driven design (DDD) principles are followed. They state that domain entities are the core of an app.
+
+Pydantic is used to model all domain entities as it is the go-to python package for writing data structures with out-of-the-box type validation.
+
+The domain of the LLM Twin application is split into two dimensions:
+
+* the data category: post, article, and repository
+* the state of the data: cleaned, chunked, and embedded
+
+A base class is created for each state of the document:
+
+* `class CleanedDocument(VectorBaseDocument, ABC)`
+* `class Chunk(VectorBaseDocument, ABC)`
+* `class EmbeddedChunk(VectorBaseDocument, ABC)`
+
+All of them inherit the VectorBaseDocument, which is the book's custom OVM implementation (explained in a bit). Each also inherits from ABC, which makes the class abstract. 
+
+Each base abstract class from above will have a subclass that adds the data category dimension. For instance, the CleanedDocument class will have the below subclasses:
+
+* `class CleanedPostDocument(CleanedDocument)`
+* `class CleanedArticleDocument(CleanedDocument)`
+* `class CleanedRepositoryDocument(CleanedDocument)`
+
+This is repeated for Chunk and EmbeddedChunk
+
+![image](https://github.com/user-attachments/assets/1e92ede0-2def-4985-b6c1-25b72b64ed5a)
+
+For example, when ingesting a raw document, the cleaning step will yield a `CleanedArticleDocument` instance, the chunking step will return a list of `ArticleChunk` objects, and the embedding operation will return `EmbeddedArticleChunk` instances that encapsulate the embedding and all the necessary metadata to ingest in the vector DB.
+
+The authors say they chose this type of design because the list of states will rarely change, and that they want to exted the list of data categories. Therefore, structuring the classes after the state allows us to plub another data caegory by inheriting these base abstract classes. 
+
+All the attributes of a cleaned document will be saved within the metadata of the vector DB. For example, the metadata of a cleaned article document will always contain the content, platform, author id, author full name, and link of the article.
+
+Another fundamental aspect is the Config internal class, which defines the name of the collection within the vector DB, the data category of the entity, and whether to leverage the vector index when creating the collection:
+
+```python
+class CleanedDocument(VectorBaseDocument, ABC):
+    content: str
+    platform: str
+    author_id: UUID4
+    author_full_name: str
+
+class CleanedPostDocument(CleanedDocument):
+    image: Optional[str] = None
+    class Config:
+        name = "cleaned_posts"
+        category = DataCategory.POSTS
+        use_vector_index = False
+
+class CleanedArticleDocument(CleanedDocument):
+    link: str
+    class Config:
+        name = "cleaned_articles"
+        category = DataCategory.ARTICLES
+        use_vector_index = False
+
+class CleanedRepositoryDocument(CleanedDocument):
+    name: str
+    link: str
+    class Config:
+        name = "cleaned_repositories"
+        category = DataCategory.REPOSITORIES
+        use_vector_index = False
+```
+
+The last but of this section is to llok at the base abstract class of the chunk and embeeded chunk:
+
+```python
+class Chunk(VectorBaseDocument, ABC):
+    content: str
+    platform: str
+    document_id: UUID4
+    author_id: UUID4
+    author_full_name: str
+    metadata: dict = Field(default_factory=dict)
+# PostChunk, ArticleChunk, RepositoryChunk
+
+class EmbeddedChunk(VectorBaseDocument, ABC):
+    content: str
+    embedding: list[float] | None
+    platform: str
+    document_id: UUID4
+    author_id: UUID4
+    author_full_name: str
+    metadata: dict = Field(default_factory=dict)
+# EmbeddedPostChunk, EmbeddedArticleChunk, EmbeddedRepositoryChunk
+```
+
+An enum that aggregates all our data categories in a single structure of constants, is also defined (benefits of StrEnum: distinct enumeration members; restricted set of values; comparison and identity; iterability; access to members and values):
+
+```python
+class DataCategory(StrEnum):
+    POSTS = "posts"
+    ARTICLES = "articles"
+    REPOSITORIES = "repositories"
+```
+
+The last step to fully understand how the domain objects work is to zoom into the VectorBaseDocument OVM class.
+
+#### OVM
+
+This is inspired by ORM, but the authors call it OVM because of working with embeddings and vector dbs instead of structured data and sql tables. 
+
+The code is similar to the ODM from chapter 3 (where we interacted with mongodb). 
+
+The OVM base class is called VectorBaseDocument:
+
+```python
+from pydantic import UUID4, BaseModel
+from typing import Generic
+from llm_engineering.infrastructure.db.qdrant import connection
+
+T = TypeVar("T", bound="VectorBaseDocument")
+
+class VectorBaseDocument(BaseModel, Generic[T], ABC):
+    id: UUID4 = Field(default_factory=uuid.uuid4)
+
+    @classmethod
+    def from_record(cls: Type[T], point: Record) -> T:
+        _id = UUID(point.id, version=4)
+        payload = point.payload or {}
+        attributes = {
+            "id": _id,
+            **payload,
+        }
+        if cls._has_class_attribute("embedding"):
+            payload["embedding"] = point.vector or None
+        return cls(**attributes)
+
+    def to_point(self: T, **kwargs) -> PointStruct:
+        exclude_unset = kwargs.pop("exclude_unset", False)
+        by_alias = kwargs.pop("by_alias", True)
+        payload = self.dict(exclude_unset=exclude_unset, by_alias=by_alias, **kwargs)
+        _id = str(payload.pop("id"))
+        vector = payload.pop("embedding", {})
+        if vector and isinstance(vector, np.ndarray):
+            vector = vector.tolist()
+
+        return PointStruct(id=_id, vector=vector, payload=payload)
+```
+
+The VectorBaseDocument class inherits from Pydantic’s BaseModel and helps us structure a single record’s attributes from the vector DB. Every OVM will be initialized by default with UUID4 as its unique identifier. Using generics—more precisely, by inheriting from Generic[T]—the signatures of all the subclasses of the VectorBaseDocument class will adapt to that given class.
+
+The create and read methods:
+* the from_record() method adapts a data point from Qdrant’s format to our internal structure based on pydantic
+* the to_point() method takes the attributes of the current instance and adapts them to Qdrant’s PointStruct() format
+
+Ultimately, all operations made to Qdrant will be done through the connection instance, which is instantiated in the application’s infrastructure layer.
+
+### The dispatcher layer
+
+A dispatcher inputs a document and applies dedicated handlers based on its data category (article, post, or repository). A handler can either clean, chunk, or embed a document.
+
+Firstly, the `CleaningDispatcher` - it mainly implements a `dispatch()` method that inputs a raw document. Based on its data category, it instantiates and calss a handler that applies the cleaning logic specific to that data point:
+
+```python
+class CleaningDispatcher:
+    cleaning_factory = CleaningHandlerFactory()
+
+    @classmethod
+    def dispatch(cls, data_model: NoSQLBaseDocument) -> VectorBaseDocument:
+        data_category = DataCategory(data_model.get_collection_name())
+        handler = cls.cleaning_factory.create_handler(data_category)
+        clean_model = handler.clean(data_model)
+        logger.info(
+            "Data cleaned successfully.",
+            data_category=data_category,
+            cleaned_content_len=len(clean_model.content),
+        )
+
+        return clean_model
+```
+
+The key is the CleaningHandlerFactory, which instantiates a different cleaning handler based on the document's data category:
+
+```python
+class CleaningHandlerFactory:
+    @staticmethod
+    def create_handler(data_category: DataCategory) -> CleaningDataHandler:
+        if data_category == DataCategory.POSTS:
+            return PostCleaningHandler()
+        elif data_category == DataCategory.ARTICLES:
+            return ArticleCleaningHandler()
+        elif data_category == DataCategory.REPOSITORIES:
+            return RepositoryCleaningHandler()
+        else:
+            raise ValueError("Unsupported data type")
+```
+
+The Dispatcher or Factory classes are nothing fancy, but they offer an intuitive and simple interface for applying various operations to our documents. When manipulating documents, instead of worrying about their data category and polluting our business logic with if-else statements, we have a class dedicated to handling that. We have a single class that cleans any document, which respects the DRY (don’t repeat yourself) principles from software engineering. By respecting DRY, we have a single point of failure, and the code can easily be extended. For example, if we add an extra type, we must extend only the Factory class instead of multiple occurrences in the code.
+
+The ChunkingDispatcher and EmbeddingDispatcher follow the same pattern. They use a ChunkingHandlerFactory and, respectively, an EmbeddingHandlerFactory that initializes the correct handler based on the data category of the input document. Afterward, they call the handler and return the result.
+
+The Factory class leverages the [abstract factory creational pattern](https://refactoring.guru/design-patterns/abstract-factory), which instantiates a family of classes implementing the same interface. In our case, these handlers implement the clean() method regardless of the handler type.
+
+Also, the Handler class family leverages the [strategy behavioral pattern](https://refactoring.guru/design-patterns/strategy) used to instantiate when we want to use different variants of an algorithm within an object and be able to switch from one algorithm to another during runtime.
+
+Intuitively, in our dispatcher layer, the combination of the factory and strategy patterns works as follows:
+
+* initially, we knew we wanted to clean the data, but as we knew the data category only at runtime, we couldn’t decide on what strategy to apply
+* we can write the whole code around the cleaning code and abstract away the logic under a Handler() interface, which will represent our strategy
+* when we get a data point, we apply the abstract factory pattern and create the correct cleaning handler for its data type
+* ultimately, the dispatcher layer uses the handler and executes the right strategy
+
+The last component of the RAG feature pipeline is the implementation of the cleaning, chunking, and embedding handlers.
+
+### The handlers
+
+The handler has a one-on-one structure with the domain, meaning that every entity has its own handler
+
+![image](https://github.com/user-attachments/assets/06b8c8d8-ccab-4e3b-8bb6-cede355e47bd)
+
+#### The cleaning handlers
+
+```python
+# Other imports.
+from typing import Generic, TypeVar
+
+DocumentT = TypeVar("DocumentT", bound=Document)
+CleanedDocumentT = TypeVar("CleanedDocumentT", bound=CleanedDocument)
+
+class CleaningDataHandler(ABC, Generic[DocumentT, CleanedDocumentT]):
+    @abstractmethod
+    def clean(self, data_model: DocumentT) -> CleanedDocumentT:
+        pass
+```
+
+And for every post, article and repository a different handler is implemented:
+
+```python
+class PostCleaningHandler(CleaningDataHandler):
+    def clean(self, data_model: PostDocument) -> CleanedPostDocument:
+        return CleanedPostDocument(
+            id=data_model.id,
+            content=clean_text(" #### ".join(data_model.content.values())),
+            # Copy the rest of the parameters from the data_model object.
+        )
+
+class ArticleCleaningHandler(CleaningDataHandler):
+    def clean(self, data_model: ArticleDocument) -> CleanedArticleDocument:
+        valid_content = [content for content in data_model.content.values() if content]
+        return CleanedArticleDocument(
+            id=data_model.id,
+            content=clean_text(" #### ".join(valid_content)),
+            platform=data_model.platform,
+            link=data_model.link,
+            author_id=data_model.author_id,
+            author_full_name=data_model.author_full_name,
+        )
+
+class RepositoryCleaningHandler(CleaningDataHandler):
+    def clean(self, data_model: RepositoryDocument) -> CleanedRepositoryDocument:
+        return CleanedRepositoryDocument(
+            id=data_model.id,
+            content=clean_text(" #### ".join(data_model.content.values())),
+            # Copy the rest of the parameters from the data_model object.
+        )
+```
+
+The handlers input a raw document domain entity, clean the content, and return a cleaned document. All the handlers use the clean_text() function to clean the text. Out of simplicity, the same cleaning technique for all the data categories is used.
+
+#### The chunking handlers
+
+The handler takes cleaned documents as input and returns chunk entities.
+
+This is the abstract class:
+
+```python
+# Other imports.
+from typing import Generic, TypeVar
+
+CleanedDocumentT = TypeVar("CleanedDocumentT", bound=CleanedDocument)
+ChunkT = TypeVar("ChunkT", bound=Chunk)
+
+class ChunkingDataHandler(ABC, Generic[CleanedDocumentT, ChunkT]):
+    @property
+    def metadata(self) -> dict:
+        return {
+            "chunk_size": 500,
+            "chunk_overlap": 50,
+        }
+
+    @abstractmethod
+    def chunk(self, data_model: CleanedDocumentT) -> list[ChunkT]:
+        pass
+```
+
+#### The embedding handlers
+
+The embedding handlers differ slightly from the others as the EmbeddingDataHandler() interface contains most of the logic. This is done so when the embedding model is called, as many as samples are batched to optimise the inference process. When running the model on a GPU, the batched samples are processed independently and in parallel. Thus, by batching the chunks, we can optimize the inference process by 10x or more, depending on the batch size and hardware we use
+
+An `embed()` method is implemented for running inference on a single data point, along with an `embed_batch()` method. The `embed_batch()` method takes chunked documents as input, gathers their content into a list, passes them to the embedding model, and maps the results to an embedded chunk domain entity. The mapping is done through the `map_model()` abstract method, which must be customized for every data category.
+
+```python
+# Other imports.
+from typing import Generic, TypeVar, cast
+from llm_engineering.application.networks import EmbeddingModelSingleton
+
+ChunkT = TypeVar("ChunkT", bound=Chunk)
+EmbeddedChunkT = TypeVar("EmbeddedChunkT", bound=EmbeddedChunk)
+embedding_model = EmbeddingModelSingleton()
+
+class EmbeddingDataHandler(ABC, Generic[ChunkT, EmbeddedChunkT]):
+    """
+    Abstract class for all embedding data handlers.
+    All data transformations logic for the embedding step is done here
+    """
+    def embed(self, data_model: ChunkT) -> EmbeddedChunkT:
+        return self.embed_batch([data_model])[0]
+    def embed_batch(self, data_model: list[ChunkT]) -> list[EmbeddedChunkT]:
+        embedding_model_input = [data_model.content for data_model in data_model]
+        embeddings = embedding_model(embedding_model_input, to_list=True)
+        embedded_chunk = [
+            self.map_model(data_model, cast(list[float], embedding))
+            for data_model, embedding in zip(data_model, embeddings, strict=False)
+        ]
+        return embedded_chunk
+
+    @abstractmethod
+    def map_model(self, data_model: ChunkT, embedding: list[float]) -> EmbeddedChunkT:
+        pass
+```
+
+The last step is to understand how the EmbeddingModelSingleton() works. It is a wrapper over the SentenceTransformer() class from Sentence Transformers that initializes the embedding model.
+
+Writing a wrapper over external packages is often good practice. Thus, when we want to change the third-party tool, you have to modify only the internal logic of the wrapper instead of the whole code base.
+
+This is designed with the [singleton pattern](https://refactoring.guru/design-patterns/singleton) in mind.
+
+Next, I read the refactoring guru links referenced in the book:
+
+* [builder](https://refactoring.guru/design-patterns/builder)
+* [abstract factory](https://refactoring.guru/design-patterns/abstract-factory)
+* [strategy](https://refactoring.guru/design-patterns/strategy)
+* [singleton](https://refactoring.guru/design-patterns/singleton)
+
+## The Builder design pattern
+
+**Intent**
+
+It is a creational design pattern that lets us construct complex objects step by step. The patterns allows us to produce different types and representations of an object using the same construction code
+
+**Problem**
+
+1. complex initialization: it simplifies the construction of complex objects by allowing step-by-step configuration, avoiding unwieldy constructors
+2. telescoping constructor issue: it prevents cluttered constructors with many optional parameters, improving readability
+3. avoiding subclass explosion: it eliminates the need for multiple subclasses for different configurations, reducing complexity in the class hierarchy
+4. separation of concerns: it separates the object’s construction from its representation, allowing for easier maintenance and flexibility
+5. enhanced usability: client code becomes more intuitive, as parameters can be set explicitly, making the code clearer and easier to understand
+
+**Solution**
+
+The Builder pattern suggests that we extract the object construction code out of its own class and move it to separate objects called builders.
+
+The pattern organizes object construction into a set of steps. To create an object, we execute a series of these steps on a builder object. The important part is that we don’t need to call all of the steps. We can call only those steps that are necessary for producing a particular configuration of an object.
+
+Some of the construction steps might require different implementation when we need to build various representations of the product. For example, walls of a cabin may be built of wood, but the castle walls must be built with stone.
+
+In this case, we can create several different builder classes that implement the same set of building steps, but in a different manner. Then we can use these builders in the construction process to produce different kinds of objects.
+
+_Director_
+
+We can go further and extract a series of calls to the builder steps we use to construct a product into a separate class called director. This type of class defines the order in which to execute the building steps, while the builder provides  the implementation for those steps.
+
+**Structure**
+
+![image](https://github.com/user-attachments/assets/9e0f7924-0c0d-443c-b872-48c967306a4a)
+
+**Applicability**
+
+* use the Builder pattern to get rid of a 'telescoping(overloaded) constructor'
+* use the Builder pattern when we want our code to be able to create different representations of some product
+* it can be applied when construction of various representations of the product involves similar steps that differ only in the details
+* the Builder allows us to construct products step-by-step
+
+## The Abstract Factory design pattern
+
+![image](https://github.com/user-attachments/assets/3e7f6e4d-905b-47ed-b6b1-f53eb8b7efd2)
+
+Abstract Factory is a creational design pattern that lets us produce families of related objects without specifying their concrete classes.
+
+**Solution**
+
+The first thing the Abstract Factory pattern suggests is to explicitly declare interfaces for each distinct product of the product family. Then we can make all variants of products follow those interfaces. For example, all chair variants can implement the Chair interface; all coffee table variants can implement the CoffeeTable interface, and so on
+
+![image](https://github.com/user-attachments/assets/e4337b23-3da5-492c-8182-ed4d9fc4ca5f)
+
+The next move is to declare the Abstract Factory—an interface with a list of creation methods for all products that are part of the product family (for example, createChair, createSofa and createCoffeeTable). These methods must return abstract product types represented by the interfaces we extracted previously: Chair, Sofa, CoffeeTable and so on.
+
+![image](https://github.com/user-attachments/assets/b3257bed-5f4a-4ec4-9445-e791d9469e8f)
+
+If the client is only exposed to the abstract interfaces, what creates the actual factory objects? Usually, the application creates a concrete factory object at the initialization stage. Just before that, the app must select the factory type depending on the configuration or the environment settings.
+
+**Structure**
+
+![image](https://github.com/user-attachments/assets/ebe2806c-b041-4609-9f8c-32f8bccd3414)
+
+**Applicability**
+
+* use this design pattern when our code needs to work with various families of related products, but we don't want it to depend on the concrete classes of these products - they might be unknown beforehand or we simply want to allow for future extensibility
+* consider implementing the Abstract Factory when we have a class with a set of factory methods that blue its primary responsibility
+
+## The Strategy design pattern
+
+Strategy is a behavioral design pattern that lets us define a family of algorithms, put each of them into a separate class, and make their objects interchangeable.
+
+**Problem**
+
+![image](https://github.com/user-attachments/assets/912d5abf-94b4-46e2-a66e-bbb3e0bec547)
+
+Imagine we build a navigator and over time we need to add new and new features. After some time, the code of the navigator becomes bloated.
+
+**Solution**
+
+The Strategy pattern suggests that you take a class that does something specific in a lot of different ways and extract all of these algorithms into separate classes called strategies.
+
+The original class, called context, must have a field for storing a reference to one of the strategies. The context delegates the work to a linked strategy object instead of executing it on its own.
+
+The context isn’t responsible for selecting an appropriate algorithm for the job. Instead, the client passes the desired strategy to the context. In fact, the context doesn’t know much about strategies. It works with all strategies through the same generic interface, which only exposes a single method for triggering the algorithm encapsulated within the selected strategy.
+
+This way the context becomes independent of concrete strategies, so you can add new algorithms or modify existing ones without changing the code of the context or other strategies.
+
+![image](https://github.com/user-attachments/assets/e8df8779-c699-4ba3-ab41-1eb56f2cd682)
+
+**Pros**
+
+* runtime flexibility: algorithms can be swapped at runtime, allowing for dynamic behavior
+* encapsulation: implementation details of algorithms are isolated from the client code, promoting cleaner architecture
+* composition over inheritance: replaces inheritance with composition, reducing complexity and increasing flexibility
+* open/closed principle: new strategies can be introduced without modifying existing code, enhancing maintainability
+
+**Cons**
+
+* overhead for few algorithms: if there are only a couple of algorithms that rarely change, introducing new classes and interfaces may complicate the code unnecessarily
+* client awareness: clients must understand the differences between strategies to select the appropriate one, which can lead to confusion
+* functional alternatives: modern languages may support functional types that allow implementing algorithms with anonymous functions, reducing the need for the strategy pattern and avoiding code bloat
+
+## The Singleton design pattern
+
+Singleton is a creational design pattern that lets us ensure that a class has only one instance, while providing a global access point to this instance.
+
+**Problem/Reason to Use**
+
+* ensure that a class has just a single instance: why would anyone want to control how many instances a class has? The most common reason for this is to control access to some shared resource—for example, a database or a file
+
+* provide a global access point to that instance: remember those global variables that someone used to store some essential objects? While they’re very handy, they’re also very unsafe since any code can potentially overwrite the contents of those variables and crash the app
+
+Nowadays, the Singleton pattern has become so popular that people may call something a singleton even if it solves just one of the listed problems.
+
+**Structure**
+
+![image](https://github.com/user-attachments/assets/c90315c6-ff77-4691-9810-1119180495f0)
+
+I found that the website has all of its design patterns in python with examples - https://refactoring.guru/design-patterns/python
+
+Definitely something to checkout in the future. 
+
+## Went to my professor's lab to use the A6000 again
+
+I tried tuning Gemma 2 today, but to do that I had to install the flash attention package. Not sure if it's bugged or something or it is just big but the 'Building wheel for flash-attn' part took forever. And in the meantime I just decided to fine-tune llama 3.2-3B and using 16K context worked thanks to it being a smaller model.
+
+The results are:
+
+| Model             | Text RougeLSum | Ratings RMSE |
+|-------------------|----------------|--------------|
+| gpt4o-mini        | 0.8768         | 0.5924       |
+| gpt4o-mini-fewshot| 0.8499         | 0.6797       |
+| llama-3.1-8b-8k   | 0.7827         | 0.8580       |
+| llama-3.2-3b-16k  | 0.8537         | 0.7636       |
+
+
+## [Streamed](https://youtu.be/ctUY3N2HIvI)
+
+On stream I started covering the [dbt fundamentals](https://learn.getdbt.com/courses/dbt-fundamentals) course. It's dbt's education team official course website. For the practical part it requires me to set up a connection using dbt cloud (which is not hard) with some cloud provider (or a data warehouse tool) (and this is the unfortunate part). So I ended up just watching the videos and doing the quizes. 
+
+After I ended the stream I slept and then decided to finish the dbt fundamentals course. 
+
+<img width="433" alt="image" src="https://github.com/user-attachments/assets/8eeeeb5d-2feb-489b-9d98-3c78afddb555">
+
+At the end I saw this:
+
+```
+The dbt Certified Developer Path
+The courses on the path are:
+
+dbt fundamentals 
+Refactoring SQL for Modularity
+Jinja, Macros, and Packages
+Advanced Materializations
+Analyses and Seeds
+Advanced Testing
+Advanced Deployment
+```
+
+I will continue with the 2nd course next. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 2c5f2e194513b35826159212eb8263f17daa4435 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Nov 2024 20:18:22 +0900
Subject: [PATCH 483/638] Create 2024-11-01-day-305.md

---
 _posts/2024-11-01-day-305.md | 175 +++++++++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100644 _posts/2024-11-01-day-305.md

diff --git a/_posts/2024-11-01-day-305.md b/_posts/2024-11-01-day-305.md
new file mode 100644
index 000000000..770553379
--- /dev/null
+++ b/_posts/2024-11-01-day-305.md
@@ -0,0 +1,175 @@
+---
+layout: post
+title: (Day 305) More of the LLM Engineer's handbook
+categories: [nlp,theory]
+---
+
+# Hello :) Today is Day 305!
+A quick summary of today:
+* read chapters 5 and 6 of the handbook
+
+## LLM Engineer's Handbook Chapter 5 - Supervised Fine-Tuning
+
+Overview of the post-training data pipeline covered in this chapter
+
+![image](https://github.com/user-attachments/assets/35ae6428-db01-4053-8161-e25ab43e3c2f)
+
+### Creating our own instruction dataset
+
+To create a high-quality instruction dataset, we need to address two main issues:
+
+* the instructured nature of our data
+* the limited number of articles we can crawl
+
+The unstructured nature comes from the fact that we are dealing with raw text, instead of pairs of intructions and answers. To address this, an LLM is used to perform backtranslation and rephrasing. Backtranslation refers to the process of providing the expected answer as output and generating its corresponding instruction. However, using a chunk of text like a paragraph as an answer might not always be appropriate. This is why we want to rephrase the raw text to ensure we’re outputting properly formatted, high-quality answers. Additionally, we can ask the model to follow the author’s writing style to stay close to the original paragraph. While this process involves extensive prompt engineering, it can be automated and used at scale.
+
+The second issue regarding the limited number of samples is quite common in real-world use cases. To address this problem, we will divide our articles into chunks and generate three instruction-answer pairs for each chunk. This will multiply the number of samples we create while maintaining diversity in the final dataset
+
+LLMs are not reliable when it comes to producing structured output. To ensure properly structured results, we can employ structured generation techniques. Structured generation is an effective method to force an LLM to follow a predefined template, such as JSON, pydantic classes, or regular expressions
+
+_Synthetic data generation pipeline from raw text to instruction dataset_
+
+![image](https://github.com/user-attachments/assets/37d4c81f-dd6f-4ded-8f8f-38b6ff125776)
+
+After the python implementation, the authors share the generated dataset on [huggingface](https://huggingface.co/datasets/mlabonne/llmtwin)
+
+![image](https://github.com/user-attachments/assets/b6404635-95b2-43f1-8ecd-4a6744054e62)
+
+### Exploring SFT and its techniques
+
+'When to fine-tune?' - basic chart:
+
+![image](https://github.com/user-attachments/assets/0be587d2-87d1-4776-bb9b-f6e260fd2529)
+
+### Parameter-efficient fine-tuning techniques
+
+Differences between full, LoRA and QLoRA:
+
+![image](https://github.com/user-attachments/assets/c72937c3-527b-4c3f-a338-ad74e3d81069)
+
+This part gave a very nice overview of the parameters used when doing (LoRA) fine-tuning. Which was interesting as I see all the parameters when using unsloth for the company reviewer LLM. They also show how to do LoRA fine-tuning and it is the same code as I use haha.
+
+LoRA (Low-Rank Adaptation) and QLoRA (Quantized LoRA) are techniques designed to make fine-tuning large language models (LLMs) more memory-efficient and accessible, especially for users with limited computational resources.
+
+- LoRA enables efficient fine-tuning by introducing trainable, low-rank matrices A and B to model layers, preserving the original parameters while reducing memory and speeding up training. LoRA focuses primarily on attention layers but can be extended to other parts, like the feed-forward layers. With LoRA, tuning a model with 7B parameters can be done on a single GPU with 14–18 GB of VRAM
+  
+- QLoRA combines LoRA with 4-bit quantization to further minimize memory usage, making it possible to fine-tune even large models on modest GPUs. QLoRA's double quantization and paged optimizers reduce memory by up to 75%, though with a 30% training speed penalty compared to LoRA
+
+Key Components of Training LLMs with LoRA and QLoRA:
+
+1. hyperparams: LoRA’s rank r and scaling factor alpha control the size and impact of the added matrices, with rank values from 8 to 256 offering flexibility between task diversity and efficiency
+2. training params:
+   - learning rate and scheduler: acrucial factor, often in the range of 1e-5, adjusted over time using linear or cosine decay
+   - batch size and gradient accumulation: effective batch sizes depend on GPU memory, with larger sizes increasing stability
+   - max sequence length and packing: length caps affect memory and performance, with packing optimizing short sequences for training
+   - number of epochs: typically 2–5, balancing overfitting and underfitting
+   - optimizers**: AdamW is widely used; paged or quantized versions help manage memory on constrained GPUs
+3. regularization techniques: weight decay (0.01-0.1) and gradient checkpointing reduce memory load by recalculating intermediate steps, improving efficiency without losing performance
+
+## Chapter 6 - Fine-Tuning with Preference Alignment
+
+SFT sturggles to capture the nuance of human preferences and the long tail of potential interactions that a model might encounter. This limitation has led to the development of more advanced techniques for aligning Ai systems with human preferences - _preference alignment_
+
+Preference alignment addresses the shortcomings of SFT by incorporating direct human or AI feedback into the training process. This method allows a more nuanced understanding of human preferences, especially in complex scenarios where simple supervised learning falls short. While numerous techniques exist for preference alignment, this chapter will primarily focus on **Direct Preference Optimization** (DPO) for simplicity and efficiency.
+
+### Understanding preference datasets
+
+Preference datasets lack the standardization of instruction datasets due to varying data requirements across different training algorithms. Preference data comprises a collection of responses to a given instruction, ranked by humans or language models
+
+The structure of DPO datasets is straightforward: each instruction is paired with one preferred answer and one rejected answer. The objective is to train the model to generate the preferred response rather than the rejected one.
+
+![image](https://github.com/user-attachments/assets/3eb2a050-21be-4fb9-83b9-17580b412579)
+
+In preference datasets, the rejected response is as important as the chosen one. Without the rejected response, the dataset would be a simple instruction set. Rejected responses represent the behavior we aim to eliminate from the model. This provides a lot of flexibility and allows us to use preference datasets in many contexts. Here is a list of examples where preference datasets are more beneficial to use compared to using SFT alone:
+
+* chatbots
+* content moderation
+* summarisations
+* code generation
+* creative writing
+* translation
+
+DPO datasets typically require fewer samples than instruction datasets to significantly impact model behavior. As with instruction datasets, the required sample count depends on model size and task complexity. Larger models are more sample-efficient and thus require less data, while complex tasks demand more examples to capture the desired behavior. Once again, data quality is crucial, and a large number of preference pairs is generally beneficial.
+
+DPO datasets can be created using various methods, each with its own trade-offs between quality, cost, and scalability. These methods can be tailored to specific applications and require varying degrees of human feedback. The book divides them into 4 methods:
+
+* human-generated, human-evaluated
+* human-generated, LLM-evaluated
+* LLM-generated, human-evaluated
+* LLM-generated, LLM-evaluated
+
+Note that evaluation is not always required and preferences can emerge naturally from the generation process. For instance, it is possible to use a high-quality model to generate preferred outputs and a lower-quality or intentionally flawed model to produce less preferred alternatives. This creates a clear distinction in the preference dataset, allowing more effective training of AI systems to recognize and emulate high-quality outputs. The Intel/orca_dpo_pairs dataset available on the Hugging Face Hub was created with this process.
+
+[Here](https://huggingface.co/datasets/mlabonne/llmtwin-dpo) is the preference dataset created by the authors. 
+
+<img width="821" alt="image" src="https://github.com/user-attachments/assets/355ba054-a910-44fb-bd80-5e75ed24fa09">
+
+### Preference alignment
+
+#### RLHF
+
+At its core, RLHF works by iteratively improving both a reward model and a policy:
+
+* reward model learning
+* policy optimisation
+* iterative improvement
+
+A key innovation in RLHF is its approach to handling the high cost of human feedback. Rather than requiring constant human oversight, RLHF allows for asynchronous and sparse feedback. The learned reward model serves as a proxy for human preferences, enabling the RL algorithm to train continuously without direct human input for every action.
+
+Proximal Policy Optimization (PPO) algorithm is one of the most famous RLHF algorithms. Here, the reward model is used to score the text that is generated by the trained model. This reward is regularized by an additional Kullback–Leibler (KL) divergence factor, ensuring that the distribution of tokens stays similar to the model before training (frozen model).
+
+![image](https://github.com/user-attachments/assets/3a9eea03-99e9-440f-84c7-14296275010a)
+
+While RLHF has proven effective for aligning AI systems with human preferences, it faces challenges due to its iterative nature and reliance on a separate reward model, which can be computationally expensive and potentially unstable.
+
+#### Direct Preference Optimization (DPO)
+
+Direct Preference Optimization (DPO), introduced by Rafailov et al. in 2023, offers a simpler alternative to the traditional Reinforcement Learning from Human Feedback (RLHF) methods for training language models. Rather than relying on separate reward models and complex RL algorithms, DPO reformulates preference learning by deriving a closed-form expression for the optimal policy under RLHF’s typical reward-maximizing objective with a KL-divergence constraint. This reformulation allows DPO to handle preference learning directly, without separate reward models or reinforcement learning.
+
+![image](https://github.com/user-attachments/assets/a7b61c7a-5dca-4216-8693-5c7b072f8759)
+
+In practice, DPO functions as a binary cross-entropy loss on the model’s output probabilities, encouraging higher probabilities for preferred responses and lower for non-preferred ones, while remaining close to a reference model, controlled by a tunable beta parameter. The approach enables simple gradient descent optimization, streamlining the process and eliminating the need for sampling or other RL complexities. This simplicity reduces engineering complexity and computational cost, especially when using adapters like LoRA or QLoRA, which allow one model to be trained without separating frozen and trained models, saving VRAM.
+
+DPO can match RLHF’s performance but is often more stable and less sensitive to hyperparameters, making it accessible to smaller teams lacking deep RL expertise. Although RLHF methods have a higher performance ceiling for large-scale applications, DPO offers most benefits with less computational and engineering effort. Both methods benefit from synthetic data, which enables iterative improvements in model training.
+
+However, DPO does have limitations. It still requires paired preference data and lacks some of the theoretical guarantees of RL approaches. While RLHF may be more advantageous for complex or dynamic tasks, DPO serves as an effective, lower-cost alternative for preference alignment in many applications.
+
+## scikit-learn official certification 🥳
+
+I saw this morning that probabl (one of the companies run by scikit-learn devs) announced an official certification program - [here](https://certification.probabl.ai/scikit-learn-associate)
+
+The link takes you to the 1st level of the certification (there are 3 overall). And I think I will review the materials suggested to pass this 1st level on stream. Here they are:
+
+<img width="1041" alt="image" src="https://github.com/user-attachments/assets/fe1c35cb-b35d-43bd-87b9-8eb768e65641">
+
+* **Data Preprocessing**
+   - Loading parquet datasets
+   - Visualizing data with basic plotting techniques: scatterplot, boxplot
+   - Identifying wrongly encoded predictive columns (e.g., float encoded as string)
+   - Handling missing values using imputation (`SimpleImputer`)
+   - Correct choice of feature scaling: `StandardScaler`, `MinMaxScaler`
+   - Encoding categorical data: `OrdinalEncoder`, `OneHotEncoder`
+   - Combining preprocessing steps with `ColumnTransformer`
+
+* **Model Building and Evaluation**
+   - Splitting datasets into training and testing sets using `train_test_split`
+   - Training ML models using the `fit()` method
+   - Making predictions using the `predict()` method
+   - Evaluating model performance with common metrics: accuracy, precision, recall, F1 score, confusion matrix, mean squared error, R-squared
+   - Interpreting scores with respect to dummy models
+
+* **Model Selection and Validation**
+   - Understanding and implementing cross-validation techniques: `KFold`, `ShuffleSplit`
+   - Learning and validation curves
+   - Performing hyperparameter tuning using `GridSearchCV`, `RandomSearchCV`
+   - Stability of learned coefficients across splits
+
+* **Interpretation of Results & Communication**
+   - Visualizing model results using basic plotting techniques (`matplotlib`, `seaborn`)
+   - Interpreting and communicating model outputs and performance metrics to non-technical stakeholders
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f0d40488da2b6a23739ca12a0b0477739f67309e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 2 Nov 2024 22:41:18 +0900
Subject: [PATCH 484/638] Create 2024-11-02-day-306.md

---
 _posts/2024-11-02-day-306.md | 199 +++++++++++++++++++++++++++++++++++
 1 file changed, 199 insertions(+)
 create mode 100644 _posts/2024-11-02-day-306.md

diff --git a/_posts/2024-11-02-day-306.md b/_posts/2024-11-02-day-306.md
new file mode 100644
index 000000000..d0d69b0e4
--- /dev/null
+++ b/_posts/2024-11-02-day-306.md
@@ -0,0 +1,199 @@
+---
+layout: post
+title: (Day 306) Checking out scikit-learn's MOOC
+categories: [nlp,theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 306!
+A quick summary of today:
+* read a bit more of the LLM engineer's handbook
+* started covering sklearn's MOOC
+
+## LLM Engineer's handbook Chapter 7 - Evaluating LLMs
+
+### Model evaluation
+
+In model evaluation, the objective is to assess the capabilities of a single model without any prompt engineering, RAG pipeline, and so on.
+
+General-purpose LLM metrics are:
+
+* train loss
+* validation loss
+* perplexity
+* gradient norm
+
+After pre-training, it is common to use a suite of evaluations to evaluate the base model. This suite can include internal and public benchmarks. Here’s a non-exhaustive list of common public pre-training evaluations:
+
+* MMLU (knowledge)
+* HellaSwag (reasoning)
+* ARC-C (reasoning)
+* Winogrande (reasoning)
+* PIQA (reasoning)
+
+Domain-specific LLM evaluations are:
+
+* Open Medical-LLM Leaderboard
+* BigCodeBench Leaderboard
+* Hallucinations Leaderboard
+* Enterprise Scenarios Leaderboard
+
+Task-specific LLM evaluations are:
+
+* ROUGE
+* for classification: accuracy, precision, recall, f1
+* LLM-as-a-judge
+
+### RAG evaluation
+
+The evaluation of RAG systems goes beyond assessing a standalone LLM. It requires examining the entire system’s performance, including:
+
+* retrieval accuracy
+* integration quality
+* factuality and relevance
+
+Below, two methods are covered on how to evaluate how well RAG models incorporate external information into their responses
+
+#### RAGAS
+
+Retrieval-Augmented Generation Assessment (Ragas) is an open-source toolkit designed to provide developers with a comprehensive set of tools for RAG evaluation and optimization. It’s designed around the idea of metrics-driven development (MDD), a product development approach that relies on data to make well-informed decisions, involving the ongoing monitoring of essential metrics over time to gain valuable insights into an application’s performance
+
+![image](https://github.com/user-attachments/assets/5bc110bc-24fc-4081-b01a-c459fb4fc517)
+
+Ragas provides a suite of LLM-assisted evaluation metrics designed to objectively measure different aspects of RAG system performance. These metrics include:
+
+* faithfulness: this metric measures the factual consistency of the generated answer against the given context. It works by breaking down the answer into individual claims and verifying if each claim can be inferred from the provided context. The faithfulness score is calculated as the ratio of verifiable claims to the total number of claims in the answer
+* answer relevancy: this metric evaluates how pertinent the generated answer is to the given prompt. It uses an innovative approach where an LLM is prompted to generate multiple questions based on the answer and then calculates the mean cosine similarity between these generated questions and the original question. This method helps identify answers that may be factually correct but off-topic or incomplete
+* context precision: this metric evaluates whether all the ground-truth relevant items present in the contexts are ranked appropriately. It considers the position of relevant information within the retrieved context, rewarding systems that place the most pertinent information at the top
+* context recall: this metric measures the extent to which the retrieved context aligns with the annotated answer (ground truth). It analyzes each claim in the ground truth answer to determine whether it can be attributed to the retrieved context, providing insights into the completeness of the retrieved information
+
+#### ARES
+
+ARES (an automated evaluation framework for RAG systems) is a comprehensive tool designed to evaluate RAG systems. It offers an automated process that combines synthetic data generation with fine-tuned classifiers to assess various aspects of RAG performance, including context relevance, answer faithfulness, and answer relevance.
+
+The ARES framework operates in three main stages: synthetic data generation, classifier training, and RAG evaluation. Each stage is configurable, allowing users to tailor the evaluation process to their specific needs and datasets.
+
+![image](https://github.com/user-attachments/assets/482373ed-3643-4cfd-8cad-6a0aa2c96ca5)
+
+## Chapter 8 - Inference Optimization
+
+### Model optimization strategies
+
+Below, several optimization strategies that are commonly used to speed up inference and reduce Video Random-Access Memory (VRAM) usage, such as implementing a (static) KV cache, continuous batching, speculative decoding, and optimized attention mechanisms, are discussed.
+
+#### KV cache
+
+LLMs generate text token by token, which is slow because each new prediction depends on the entire previous context. For example, to predict the 100th token in a sequence, the model needs the context of tokens 1 through 99. When predicting the 101st token, it again needs the information from tokens 1 through 99, plus token 100. This repeated computation is particularly inefficient.
+
+The key-value (KV) cache addresses this issue by storing key-value pairs produced by self-attention layers. Instead of recalculating these pairs for each new token, the model retrieves them from the cache, significantly speeding up the generation.
+
+![image](https://github.com/user-attachments/assets/89e6450c-f772-41f9-80d2-1c25daed2dd4)
+
+The size of the cache is:
+
+![image](https://github.com/user-attachments/assets/6ed0b931-020b-48d8-a0e9-51cc3e784033)
+
+It grows with each generation step and is dynamic, it prevents us from taking advantage of torch.compile, a powerful optimization tool that fuses PyTorch code into fast and optimized kernels. The static KV cache solves this issue by pre-allocating the KV cache size to a maximum value, which allows us to combine it with torch.compile for up to a 4x speedup in the forward pass.
+
+Efficiently managing the KV cache is essential, as it can quickly exhaust available GPU memory and limit the batch sizes that can be processed. 
+
+#### Continuous batching
+
+Continuous batching improves the efficiency of inference by processing new requests as soon as previous ones finish, rather than waiting for all requests in a batch to complete. This method avoids idle time, keeping the hardware fully utilized. It works by evicting completed requests from the batch and immediately replacing them with new ones. This approach is especially useful for decoder-only models with varying input and output lengths, which can otherwise lead to underutilization. Most inference frameworks, such as Hugging Face’s TGI, vLLM, and NVIDIA TensorRT-LLM, support continuous batching natively.
+
+#### Speculative decoding
+
+Another powerful optimization technique is speculative decoding, also called assisted generation. The key insight is that even with continuous batching, the token-by-token generation process fails to fully saturate the parallel processing capabilities of the accelerator. Speculative decoding aims to use this spare compute capacity to predict multiple tokens simultaneously, using a smaller proxy model
+
+![image](https://github.com/user-attachments/assets/a8f0d526-a8e9-4a30-98ed-4d2158e16320)
+
+The general approach is:
+
+* apply a smaller model, like a distilled or pruned version of the main model, to predict multiple token completions in parallel. This could be 5–10 tokens predicted in a single step
+* feed these speculative completions into the full model to validate which predictions match what the large model would have generated
+* retain the longest matching prefix from the speculative completions and discard any incorrect tokens
+
+The result is that, if the small model approximates the large model well, multiple tokens can be generated in a single step. This avoids running the expensive large model for several iterations. The degree of speedup depends on the quality of the small model’s predictions – a 90% match could result in a 3–4X speedup.
+
+#### Optimized attention mechanisms
+
+PagedAttention and FlashAttention-2 are optimized attention mechanisms that address memory inefficiencies in traditional Transformers, especially for longer sequences.
+
+**PagedAttention** (Kwon, Li, et al., 2023) tackles memory challenges by using a block-based approach, inspired by virtual memory. It partitions the key-value (KV) cache, enabling efficient memory utilization and reducing the need for contiguous memory allocation. This approach supports batching more sequences and allows memory sharing across sequences, enhancing parallel processing. Results show memory overhead reduction up to 55% and throughput improvement by 2.2x, with implementations in libraries like vLLM, TGI, and TensorRT-LLM.
+
+**FlashAttention-2** (Tri Dao, 2023) optimizes runtime and memory by dividing input/output matrices into blocks stored in GPU SRAM, which minimizes data transfers. It also employs 'online softmax', allowing block-wise computation of attention scores without large intermediate matrices, lowering memory requirements from quadratic to linear for backpropagation.
+
+### Model parallelism
+
+Model parallelism allows us to distribute the memory and compute requirements of LLMs across multiple GPUs. This enables the training and inference of models too large to fit on a single device, while also improving performance in terms of throughput (tokens per second).
+
+There are three main approaches to model parallelism, each involving splitting the model weights and computation in different ways: data parallelism, pipeline parallelism, and tensor parallelism.
+
+Although these approaches were originally developed for training, we can reuse them for inference by focusing on the forward pass only.
+
+**Data parallelism**
+
+This is the simplest type of model parallelism. It involves making copies of the model and distributing these replicas across different GPUs
+
+![image](https://github.com/user-attachments/assets/b9f47b1e-7485-456c-a70d-52a042ade730)
+
+**Pipeline parallelism**
+
+This is a strategy for distributing the computational load of training and running large neural networks across multiple GPUs.
+
+![image](https://github.com/user-attachments/assets/1b554a2b-8ad1-4df5-899c-7dffc7f39f02)
+
+**Tensor parallelism**
+
+This is another popular technique to distribute the computation of LLM layers across multiple devices. In contrast to pipeline parallelism, TP splits the weight matrices found in individual layers. This enables simultaneous computations, significantly reducing memory bottlenecks and increasing processing speed.
+
+![image](https://github.com/user-attachments/assets/cf04a9e3-7dbd-4ad6-91bb-132e9edf7d08)
+
+### Model quantization
+
+(side note: this book definitely feels like an LLM engineer's handbook - especially after covering yesterday and today's chapters. It covers crucial information for anyone working with an LLM)
+
+Model quantization reduces the precision of neural network weights and activations to lower bit formats, minimizing memory usage and speeding up large language models (LLMs) without significantly compromising performance. For example, large models (over 30B parameters) can maintain quality with 2- or 3-bit precision.
+
+There are two primary methods: **Post-Training Quantization (PTQ)**, a simpler approach that doesn’t require retraining, and **Quantization-Aware Training (QAT)**, which incorporates quantization during training to maintain better accuracy.
+
+**Quantization Techniques**
+
+* GGUF and llama.cpp: these enable CPU inference with optional GPU offloading, running on diverse hardware. llama.cpp, a popular open-source library, also supports optimizations like FlashAttention-2
+* GPTQ and EXL2: both tailored for GPU, they enhance speed over llama.cpp. GPTQ uses a refined quantization algorithm, while EXL2 adds flexibility by mixing quantization levels (2–8 bits) to minimize errors and optimize memory, allowing large models (up to 70B parameters) to run on single GPUs
+
+## Chapter 9 - The RAG inference pipeline
+
+### Understanding the LLM Twin’s RAG inference pipeline and Advanced RAG feature techniques
+
+![image](https://github.com/user-attachments/assets/d751fe56-c67e-4b6b-add0-a13e69e7b545)
+
+1. User query: We begin with the user who makes a query, such as “Write an article about...”
+2. Query expansion: We expand the initial query to generate multiple queries that reflect different aspects or interpretations of the original user query. Thus, instead of one query, we will use xN queries. By diversifying the search terms, the retrieval module increases the likelihood of capturing a comprehensive set of relevant data points. This step is crucial when the original query is too narrow or vague.
+3. Self-querying: We extract useful metadata from the original query, such as the author’s name. The extracted metadata will be used as filters for the vector search operation, eliminating redundant data points from the query vector space (making the search more accurate and faster).
+4. Filtered vector search: We embed each query and perform a similarity search to find each search’s top K data points. We execute xN searches corresponding to the number of expanded queries. We call this step a filtered vector search as we leverage the metadata extracted from the self-query step as query filters.
+5. Collecting results: We get up to xK results closest to its specific expanded query interpretation for each search operation. Further, we aggregate the results of all the xN searches, ending up with a list of N x K results containing a mix of articles, posts, and repositories chunks. The results include a broader set of potentially relevant chunks, offering multiple relevant angles based on the original query’s different facets.
+6. Reranking: To keep only the top K most relevant results from the list of N x K potential items, we must filter the list further. We will use a reranking algorithm that scores each chunk based on the relevance and importance relative to the initial user query. We will leverage a neural cross-encoder model to compute the score, a value between 0 and 1, where 1 means the result is entirely relevant to the query. Ultimately, we sort the N x K results based on the score and pick the top K items. Thus, the output is a ranked list of K chunks, with the most relevant data points situated at the top.
+7. Build the prompt and call the LLM: We map the final list of the most relevant K chunks to a string used to build the final prompt. We create the prompt using a prompt template, the retrieved context, and the user’s query. Ultimately, the augmented prompt is sent to the LLM (hosted on AWS SageMaker exposed as an API endpoint).
+8. Answer: We are waiting for the answer to be generated. After the LLM processes the prompt, the RAG logic finishes by sending the generated response to the user.
+
+Most of this chapter was just code from the repo, and it covers the above 8 steps. [Here](https://github.com/PacktPublishing/LLM-Engineers-Handbook/tree/main/llm_engineering/application/rag) is the code.
+
+## [Streamed](https://www.youtube.com/watch?v=PTB7YI0TufI)
+
+I covered the 1st module - [The predictive modeling pipeline](https://inria.github.io/scikit-learn-mooc/predictive_modeling_pipeline/predictive_modeling_module_intro.html). I don't know how I just find about this 😆. It is *such* a great tool for someone just entering the field. This was just the 1st module and it was so comprehensive. 
+
+These are the main takeaways from it and what someone can learn:
+
+* to create a scikit-learn predictive model;
+* about the scikit-learn API to train and test a predictive model;
+* to process numerical data, notably using a Pipeline
+* to process categorical data, notably using a OneHotEncoder and an OrdinalEncoder;
+* to handle and process mixed data types (i.e. numerical and categorical data), notably using a ColumnTransformer
+
+I will continue with this before taking the exam. Might be an overkill ~ but it's cool
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From bd8dc798b7ce859f2f117591418f2d963d9c9efa Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 4 Nov 2024 00:18:20 +0900
Subject: [PATCH 485/638] Create 2024-11-03-day-307.md

---
 _posts/2024-11-03-day-307.md | 50 ++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 _posts/2024-11-03-day-307.md

diff --git a/_posts/2024-11-03-day-307.md b/_posts/2024-11-03-day-307.md
new file mode 100644
index 000000000..27f167948
--- /dev/null
+++ b/_posts/2024-11-03-day-307.md
@@ -0,0 +1,50 @@
+---
+layout: post
+title: (Day 307) Covering sklearn's MOOC on stream
+categories: [theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 307!
+A quick summary of today:
+* streamed twice and covered modules 2 and 3 of sklearn's MOOC
+
+## [Stream](https://youtu.be/dky6_YmP-3s)
+
+Today's stream was longer than expected ~ 3 hours. 
+
+### I covered a dbt course:
+
+* [Refactoring SQL for Modularity](https://learn.getdbt.com/courses/refactoring-sql-for-modularity)
+
+However, I just watched the videos :/ Again, because I do not want to deal with GCP/AWS. Nevertheless it was a decent course.
+
+### After that I covered module 2 of sklearn's MOOC:
+
+* [Selecting the best model](https://inria.github.io/scikit-learn-mooc/overfit/overfit_module_intro.html)
+
+Here is the overview:
+
+* understand the concept of overfitting and underfitting
+* understand the concept of generalization
+* understand the general cross-validation framework used to evaluate a model
+
+It is all basic concepts, however they are very nicely explain, and also I want to do a comprehensive review before taking sklearn's official exam. 
+
+### I decided to do a late night stream as well, and continue with Module 3: Hyperparameter tuning
+
+Here are its main takeaways:
+
+* Hyperparameters have an impact on the models’ performance and should be wisely chosen
+* The search for the best hyperparameters can be automated with a grid-search approach or a randomized search approach
+* A grid-search is expensive and does not scale when the number of hyperparameters to optimize increase. Besides, the combination are sampled only on a regular grid
+* A randomized-search allows a search with a fixed budget even with an increasing number of hyperparameters. Besides, the combination are sampled on a non-regular grid
+
+[Here is the link to the late night stream with no mic/cam](https://youtu.be/SZeE6ZpDyIk) ~1.5hrs
+
+I am glad it was shorter because I got really sleepy towards the end
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From e74323767c6613685e982d9cb8469e2cd1c66a65 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 4 Nov 2024 23:45:09 +0900
Subject: [PATCH 486/638] Create 2024-11-04-day-308.md

---
 _posts/2024-11-04-day-308.md | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 _posts/2024-11-04-day-308.md

diff --git a/_posts/2024-11-04-day-308.md b/_posts/2024-11-04-day-308.md
new file mode 100644
index 000000000..be50cee1d
--- /dev/null
+++ b/_posts/2024-11-04-day-308.md
@@ -0,0 +1,48 @@
+---
+layout: post
+title: (Day 308) Streaming for 5 hours (overall) and reviewing sklearn's MOOC
+categories: [theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 308!
+A quick summary of today:
+* covered module 4 - linear models
+* covered module 5 - decision tree models
+
+## Sklearn MOOC module 4 - Linear models
+
+Main takeaways:
+
+* the predictions of a linear model depend on a weighted sum of the values of the input features added to an intercept parameter
+* fitting a linear model consists in adjusting both the weight coefficients and the intercept to minimize the prediction errors on the training set
+* to train linear models successfully it is often required to scale the input features approximately to the same dynamic range
+* regularization can be used to reduce over-fitting: weight coefficients are constrained to stay small when fitting
+* the regularization hyperparameter needs to be fine-tuned by cross-validation for each new machine learning problem and dataset
+* linear models can be used on problems where the target variable is not linearly related to the input features but this requires extra feature engineering work to transform the data in order to avoid under-fitting
+
+I streamed the whole module, however I was in the library so I could not turn on the mic/cam.
+
+[Part 1](https://www.youtube.com/watch?v=3HJ7jWO53gI&t=1s) and after a brief internet problem I recorder [Part 2](https://www.youtube.com/watch?v=0fYh4XKd-Sw)
+
+Using the hosted env (through binder) is so nice because there is no copilot and if I want to write code I have to find the functions/imports/params myself which again makes me learn about functions' params and creates a virtuous cycle of learning. Copilot can really have a bad influence when learning/reviewing for a coding exam 😆
+
+## Sklearn MOOC module 5 - Decision tree models
+
+Main takeaways, they are:
+
+* are suited for both regression and classification problems
+* are non-parametric models
+* are not able to extrapolate
+* are sensitive to hyperparameter tuning
+
+Very nice module again. The stream was 2 hours ~ [Here](https://youtu.be/3XMeS0aEFjw) is a link.
+
+I am really enjoying this MOOC and is a must-do course for a beginner that wants to learn about sklearn and some of the most popular non-DL models. And for sure in the future if I have to teach some of the material, I can refer people to this or even refer to it when I'm preparing my materials. 
+
+---
+
+Overall, I streamed for 5 hours today which is amazing, and also makes me a bit exhausted 😆
+
+That is all for today!
+
+See you tomorrow :)

From 19fe565624a3734da2d2c184abd5a2d48570d222 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 5 Nov 2024 23:18:49 +0900
Subject: [PATCH 487/638] Create 2024-11-05-day-309.md

---
 _posts/2024-11-05-day-309.md | 200 +++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)
 create mode 100644 _posts/2024-11-05-day-309.md

diff --git a/_posts/2024-11-05-day-309.md b/_posts/2024-11-05-day-309.md
new file mode 100644
index 000000000..4dec2a109
--- /dev/null
+++ b/_posts/2024-11-05-day-309.md
@@ -0,0 +1,200 @@
+---
+layout: post
+title: (Day 309) Finishing scikit-learn's MOOC + AI fairness
+categories: [theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 309!
+A quick summary of today:
+* reading some of the fairness python lib docs
+* stream studying the last 2 modules of sklearn's MOOC
+
+## [Loan Approval Model fairness](https://fairlearn.org/v0.11/auto_examples/plot_credit_loan_decisions.html#)
+
+I got this post somewhere from sklearn's MOOC and I kept it for a few days, and today I decided to read it.
+
+The library `fairlearn` is about assessing and improving the fairness of models - like not to disciminate between genders/race/etc for loan approvals, and the above example link in their docs is exactly about credit loan decisions (they have other examples too)
+
+In the US (Apple to be specific) there was a backlash in 2019 when Apple's Apple Card allegedly gave men higher credit limits than women, even for married couples with joint assets. It shows that while AI systems can comply with the law, they can still exhibit fairness issues. Some fairness interventions may even conflict with anti-discrimination laws
+
+### Dataset
+
+It includes demographic features, payment history, bill statements, and default information. The task is to develop a model to predict whether an applicant will default on a personal loan, using credit card payment history as a proxy for creditworthiness
+
+The dataset is imbalanced when it comes to the `SEX` feature:
+
+```
+SEX
+female    0.603733
+male      0.396267
+```
+
+An additional synthetic feature is create - 'interest rate'. It adds realism by simulating financial factors in loan approvals, allowing to demonstrate how additional features can influence fairness assessments and bias in credit decision models.
+
+### Fairness assessment
+
+After an initial fairness-unaware model is trained, the next step is performing fairness assessment. The steps are:
+
+* identify who will be harmed
+* identify the types of harms we anticipage
+* define fairness metrics based on the anticipated harms
+
+**Who will be harmed?**
+
+* based on the US Apple case, the model may incorrectly predict that women will default on their loans
+* the system might unfairly allocate less loans to women and over-allocate loans to men
+
+**Types of harms**
+
+In this case:
+
+* allocation harms: when AI extends or withholds resources, information, or opportunities, such as in hiring or lending
+
+Other: 
+
+* quality-of-service harms: when AI performs inconsistently across groups, like differing accuracy in face recognition or search results
+* stereotyping harms: when AI reinforces stereotypes, often through autocomplete suggestions, shaping expectations based on characteristics instead of individuality
+* erasure harms: when AI omits certain groups or histories, like ignoring notable figures or cultural sites, leading to underrepresentation or misrepresentation
+
+A secondary harm, is the long-term impact on an individual's FICO credit score, getting approved/denied for a loan has an impact on your score and future applications.
+
+One other type of harm we anticipate in this scenario is the long-term effects of denying loans to applicants who would have successfully paid back the loan
+
+**Fairness metrics**
+
+* balanced accuracy
+* false positive rate
+* false negative rate
+
+**Results from the fairness-unaware model**
+
+![image](https://github.com/user-attachments/assets/d1d50b65-7d97-49e3-a151-dcf5f3fc8470)
+
+The model produces a significantly higher false_negative_rate and false_positive_rate for the applicants labeled female compared to those labeled male. In the context of credit decision scenario, this means the model under-allocates loans to women who would have paid the loan, but over-allocates loans to women who go on to default on their loan.
+
+### Mitigating unfa
+
+* postprocessing: in the postprocessing approach, the outputs of a trained classifier are transformed to satisfy some fairness criterion
+* reductions: in the reductions approach, we take in a model class and iteratively create a sequence of models that optimize some fairness constraint. Compared to the postprocessing approach, the fairness constraint is satisfied during the model training time rather than afterwards
+
+**Using the first approach, the model produces:**
+
+![image](https://github.com/user-attachments/assets/4f3e202a-a1cb-47b2-a499-351b177126dd)
+
+The model achieves a much lower disparity between the two groups compared to the unmitigated model. However, this does come with the trade-off of lower balanced_accuracy score for male applicants.
+
+**Using the 2nd approach:**
+
+![image](https://github.com/user-attachments/assets/22302248-6441-44d2-8c46-7dc82bcfe386)
+
+In the reductions model, we see a decrease in the false positive rate for female applicants. This is accompanied by an increase in the false negative rate for male applicants
+
+Overall, the equalized odds difference for the reductions models is lower than that of the original fairness-unaware model.
+
+#### Fairness under unawareness
+
+When proving credit models are compliant with fair lending laws, practitoners may run into the issue of not having access to the sensitive demographic features. As a result, financial institutions are often tasked with proving their models are compliant with fair lending laws by imputing these demographics features. However, some research has shown that these imputation methods often introduce new fairness-related issues.
+
+After this I decided to check some of the other example notebooks in their docs.
+
+## [CorrelationRemover visualization](https://fairlearn.org/v0.11/auto_examples/plot_correlationremover_before_after.html)
+
+They have a `fairlearn.preprocessing.CorrelationRemover()` class which does what the name says I guess. A diabetes dataset hospital is used, and the goal is to reduce the correlation between a sensitive variable of interest (`race_AfricanAmerican` in this case) and the other variables without affecting the correlation between the other variables.
+
+Before:
+
+![image](https://github.com/user-attachments/assets/c49a21e2-68ff-4d63-8142-fb59aeafc1f3)
+
+After:
+
+![image](https://github.com/user-attachments/assets/b146c77d-b702-488c-9162-93458684942d)
+
+After, with alpha=0.5:
+
+![image](https://github.com/user-attachments/assets/cb6d634e-96d3-45ec-b33a-198c87bd554f)
+
+Even though there was not a high amount of correlation to begin with, the CorrelationRemover successfully removed all correlation between 'race_AfricanAmerican' and the other columns while retaining the correlation between the other features. Using a lower value of alpha results in not all the correlation being projected away.
+
+## [MetricFrame visualizations](https://fairlearn.org/v0.11/auto_examples/plot_quickstart.html#)
+
+_This data structure stores and manipulates disaggregated values for any number of underlying metrics. At least one sensitive feature must be supplied, which is used to split the data into subgroups. The underlying metric(s) is(are) calculated across the entire dataset (made available by the overall property) and for each identified subgroup (made available by the by_group property)._
+
+```python
+# import
+from fairlearn.metrics import (
+    MetricFrame,
+    count,
+    false_negative_rate,
+    false_positive_rate,
+    selection_rate,
+)
+
+# prepare diabetes dataset ...
+# use a DecisionTreeClassifier ...
+
+# Analyze metrics using MetricFrame
+metrics = {
+    "accuracy": accuracy_score,
+    "precision": zero_div_precision_score,
+    "false positive rate": false_positive_rate,
+    "false negative rate": false_negative_rate,
+    "selection rate": selection_rate,
+    "count": count,
+}
+metric_frame = MetricFrame(
+    metrics=metrics, y_true=y_test, y_pred=y_pred, sensitive_features=A_test
+)
+
+# plot
+metric_frame.by_group.plot.bar(
+    subplots=True,
+    layout=[3, 3],
+    legend=False,
+    figsize=[12, 8],
+    title="Show all metrics",
+)
+
+# there are different options as well, using ylim, colormap, plot kind
+```
+
+Out:
+
+![image](https://github.com/user-attachments/assets/fb4f621e-ba3b-45d1-b35f-4cd7c266c883)
+
+Here are others that I read:
+
+* [Passing pipelines to mitigation techniques](https://fairlearn.org/v0.11/auto_examples/plot_mitigation_pipeline.html)
+* [Basics & Model Specification of AdversarialFairnessClassifier](https://fairlearn.org/v0.11/auto_examples/plot_adversarial_basics.html)
+* [GridSearch with Census Data](https://fairlearn.org/v0.11/auto_examples/plot_grid_search_census.html)
+* Showed this one on stream - [Plotting Metrics with Errors](https://fairlearn.org/v0.11/auto_examples/plot_plotting_metrics_with_error.html)
+
+## [Stream](https://youtu.be/iLJa9dI71yQ)
+
+Covered [Module 6 - Ensemble of models](https://inria.github.io/scikit-learn-mooc/ensemble/ensemble_module_intro.html)
+
+Just like the previous modules, this was amazing as well. Main overview:
+
+* understanding the principles behind bootstrapping and boosting
+* get intuitions with specific models such as random forest and gradient boosting
+* identify the important hyperparameters of random forest and gradient boosting decision trees as well as their typical values
+
+## [The last module - Evaluating model performance](https://youtu.be/v8vejOmvNJ4)
+
+The goals of this last module were:
+
+* understand the necessity of using an appropriate cross-validation strategy depending on the data
+* get the intuitions behind comparing a model with some basic models that can be used as baseline
+* understand the principles behind using nested cross-validation when the model needs to be evaluated as well as optimized
+* understand the differences between regression and classification metrics
+* understand the differences between metrics
+
+
+Overall ~ I don't have enough nice words to describe how great this MOOC was. It's amazing! And definitely in my 'to-recommend' list for people wanting to get into ML.
+
+---
+
+That is all for today!
+
+I streamed for about 6 and something hours and it was a bit exhausting 😆
+
+See you tomorrow :)

From 1b4ebd03fcb3c5cf371e5ff82055fe9c96aff2a1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 7 Nov 2024 00:09:00 +0900
Subject: [PATCH 488/638] Create 2024-11-06-day-310.md

---
 _posts/2024-11-06-day-310.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 _posts/2024-11-06-day-310.md

diff --git a/_posts/2024-11-06-day-310.md b/_posts/2024-11-06-day-310.md
new file mode 100644
index 000000000..9a5a1bffe
--- /dev/null
+++ b/_posts/2024-11-06-day-310.md
@@ -0,0 +1,32 @@
+---
+layout: post
+title: (Day 310) I passed the Scikit-learn Associate Practitioner Certification
+categories: [theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 310!
+A quick summary of today:
+* did the scikit-learn exam
+* streamed for ~30mins
+
+I was originally scheduled to do the exam at around 1pm. Long story short, I started it at 5.45.
+
+Before the exam I was just re-doing the questions in the sklearn MOOC. Then I could not start when I wanted because I faced several issues with webassessor but thankfully the issues were not on my side so the exam kept getting reschedule without incurring extra fees.
+
+I am waiting for the certificate, so here is just a confirmation from the website ^^
+
+<img width="1107" alt="image" src="https://github.com/user-attachments/assets/4edd9fa0-e9d1-47dc-83f1-e48361ef2360">
+
+I think I know what I got wrong 😆 anyway ~ ...
+
+After the exam, I rested for a bit and wanted to do a short stream to talk about the exam and also [skore](https://github.com/probabl-ai/skore) - a new open-source project by probable.
+
+[Here](https://www.youtube.com/watch?v=HHWiU7GoGCo&t=1387s) is a link to the stream. 
+
+Also ~ I was told by someone from the probabl team that I am the 1st to complete the exam 🥳
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From e2a91fe0783d2d4ca0a04d7622c19ddf6ef782db Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 7 Nov 2024 23:29:24 +0900
Subject: [PATCH 489/638] Create 2024-11-07-day-311.md

---
 _posts/2024-11-07-day-311.md | 109 +++++++++++++++++++++++++++++++++++
 1 file changed, 109 insertions(+)
 create mode 100644 _posts/2024-11-07-day-311.md

diff --git a/_posts/2024-11-07-day-311.md b/_posts/2024-11-07-day-311.md
new file mode 100644
index 000000000..2c7d1885a
--- /dev/null
+++ b/_posts/2024-11-07-day-311.md
@@ -0,0 +1,109 @@
+---
+layout: post
+title: (Day 311) Checking out NODES '24, 10th chapter of LLM engineer's handbook, and linear models in sklearn
+categories: [theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 311!
+A quick summary of today:
+* joining some sessions in NODES 2024
+* reading chapter 10 of LLM engineer's handbook
+* reading about not so popular linear models in sklearn's docs
+
+Before I start, I got my certificate today:
+
+<img width="1116" alt="image" src="https://github.com/user-attachments/assets/53d64f72-e619-419d-99ff-799ed2588210">
+
+
+## NODES 2024
+
+![image](https://github.com/user-attachments/assets/8966a69b-6d73-4115-a26a-e56c3184c352)
+
+![image](https://github.com/user-attachments/assets/632e022d-a54a-4d3c-9474-d9c315b68924)
+
+Today is NODES - Neo4j's free 24-hour online conference. I joined some of the sessions
+
+### Neo4j Product Vision and Roadmap
+
+<img width="982" alt="image" src="https://github.com/user-attachments/assets/c007da26-94f3-493d-b5ff-08bab1f9f5a3">
+
+They are introducing CDC which is nice. 
+
+<img width="747" alt="image" src="https://github.com/user-attachments/assets/a1a737f5-4af7-43a9-b579-cf347871db53">
+
+*Available soon* - text to cypher models will be made available. By looking at [this](https://www.linkedin.com/posts/neo4j_aura-neo4j-aura-activity-7260072464037150721-aHSS?utm_source=share&utm_medium=member_desktop) linkedin post, it seems such natural text to prompts will be available in AuraDB
+
+Most of the things were not for me. If I were working for a company that was trying to use/already using neo4j, then I think everything would be interesting for me 😆 but as I am just a student fan for now, I was just looking forward to any news regarding their Graph Data Science library. But there were non :( 
+
+### Neo4j Visualisation Library (NVL)
+
+I joined a session on [NVL](https://neo4j.com/docs/nvl/current/installation/). Not sure how new it is, but it is a javascript library for ... visualisations. 
+
+
+## LLM Engineer's handbook Chapter 10 - Inference pipeline deployment
+
+### Criteria for choosing deployment types
+
+**Throughput and latency**
+
+Throughput is generally measured in requests per second (RPS), and is crucial when deploying ML models when we expect to process many requests. 
+
+Latency is the time it takes for a system to process a single inference request from when it is received until the result is returned. It is critical in real-time applications where quick response times are essential, such as in live user interactions, fraud detection, or any system requiring immediate feedback
+
+A lower latency translates to higher throughput when the service processes multiple queries in parallel successfully. For example, if the service takes 100 ms to process requests, this translates to a throughput of 10 requests per second. If the latency reaches 10 ms per request, the throughput rises to 100 requests per second.
+
+**Data**
+
+The model’s input and output data—including its format, volume, and complexity—are crucial to the inference process. Data characteristics like size and type directly impact latency and throughput, with more complex or larger data increasing processing time. Different model types require distinct handling based on input-output formats.
+
+Before choosing a specific deployment type, we should ask ourselves:
+
+* what are the throughput requirements?
+* how many requests the system must handle simultaniously?
+* what are the latency requirements?
+* how should the system scale?
+* what are the cost requirements?
+* what is the size of the data we work with?
+
+### Inference deployment types
+
+![image](https://github.com/user-attachments/assets/060267a1-ee00-4ed7-8edf-f5544df21462)
+
+### Monolithic vs. microservices architecture in model serving
+
+![image](https://github.com/user-attachments/assets/4467d994-177d-48e1-b800-b1075590380d)
+
+The choice between monolithic and microservices architectures for serving ML models largely depends on the application’s specific needs. A monolithic approach might be ideal for smaller teams or more straightforward applications where ease of development and maintenance is a priority. It’s also a good starting point for projects without frequent scaling requirements. Also, if the ML models are smaller, don’t require a GPU, or don’t require smaller and cheaper GPUs, the trade-off between reducing costs and complicating your infrastructure is worth considering.
+
+On the other hand, microservices, with their adaptability and scalability, are well suited for larger, more complex systems where different components have varying scaling needs or require distinct tech stacks. This architecture is particularly advantageous when scaling specific system parts, such as GPU-intensive LLM services.
+
+### LLMTwin's inference pipeline
+
+![image](https://github.com/user-attachments/assets/58339f39-3dd6-46a3-b726-91cdfb93bec3)
+
+It is defined by the feature/training/inference (FTI) architecture. For the pipe to run, it needs two things:
+
+* real-time features used for RAG, generated by the feature pipeline, which is queried from our online feature store, more concretely from the Qdrant vector db
+* a fine-tuned LLM generated by the training pipe, which is pulled from our model registry
+
+The flow is:
+
+1. a user sends a query through an HTTP request
+2. the user's input retrieves the proper context by leveraging the advanced RAG retrieval module
+3. the user's input and retrieved context are packed into the final prompt using a dedicated prompt template
+4. the prompt is sent to the LLM microservice through an HTTP request
+5. the busines microservices wait for the generated answer
+6. after the answer is generated, it is sent to the prompt monitoring pipeline along with the user's input and other vital info to monitor
+7. ultimately, the generated answer is sent back to the user
+
+The final parts of the chapter talks about deploying the model service using SageMaker and FastAPI
+
+## [Stream for 2hrs](https://youtu.be/vwlka4CmEIA)
+
+I went over the [linear models](https://scikit-learn.org/stable/modules/linear_model.html#) section of scikit-learn's docs. There were some familar 'faces' but also some concepts which I had not heard of before - like Least Angle Regression, Orthogonal Matching Pursuit (OMP), and others as the list is quite long.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 306aa8af4bc06c8a142689a80c512272c727d515 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 8 Nov 2024 23:41:54 +0900
Subject: [PATCH 490/638] Create 2024-11-08-day-312.md

---
 _posts/2024-11-08-day-312.md | 419 +++++++++++++++++++++++++++++++++++
 1 file changed, 419 insertions(+)
 create mode 100644 _posts/2024-11-08-day-312.md

diff --git a/_posts/2024-11-08-day-312.md b/_posts/2024-11-08-day-312.md
new file mode 100644
index 000000000..1f25f4725
--- /dev/null
+++ b/_posts/2024-11-08-day-312.md
@@ -0,0 +1,419 @@
+---
+layout: post
+title: (Day 312) Reading more of sklearn's Supervised learning doc
+categories: [theory,traditional-machine-learning,statistics]
+---
+
+# Hello :) Today is Day 312!
+A quick summary of today:
+* LDA and QDA
+* kernel ridge regression
+* SVMs
+* SGD
+* nearest neighbours
+
+Today, I continued reading down the 'User Guide' section of the scikit-learn docs.
+
+I could not stream today, so I reviewed the below pages by myself and kind of summarised each of them. While some of the parts are not really new for me, I still think it's valuable to review the pages as there are little details that are interesting to me. And I encourage anyone to checkout sklearn's docs ^^ 
+
+## [Linear and Quadratic Discriminant Analysis](https://scikit-learn.org/stable/modules/lda_qda.html)
+
+LDA and QDA are two classic classifiers with a linear and quadratic decision surface, respectively. Here are 3 examples on 3 different datasets, and as seen the LDA can only learn linear boundaries, while the QDA can learn quadratic and is more flexible. 
+
+![image](https://github.com/user-attachments/assets/4716d6ce-2843-4219-9647-639e8005aa1b)
+
+### Dimensionality reduction using LDA
+
+LDA can be used for supervised dim reduction, by projecting the input to a linear subspace consisting of the directions which maximise the separation between classes. The output dim is less than the number of classes so this is in general considered strong dim reduction, and only makes sense in multiclass settings.
+
+Here is an example between dim reduction via LDA and PCA to a 2d projection on the infamous iris dataset (consisting of 3 kind of flowers, and 4 features)
+
+For LDA, the defaul `n_components` param is None, but here is the full explanation from the docs:
+
+```
+n_components: int, default=None
+
+Number of components (<= min(n_classes - 1, n_features)) for dimensionality reduction. If None, will be set to min(n_classes - 1, n_features). This parameter only affects the transform method.
+```
+
+So, with the iris dataset using just LDA, the n_components becomes `min(n_classes - 1, n_features)` = 2
+
+If we use PCA with n_components=2 as well, we get:
+
+<img width="575" alt="image" src="https://github.com/user-attachments/assets/4ff877ec-b044-4ba9-8fa8-ddefe7bbe7a5">
+
+### QDA relation with GaussianNB
+
+If in the QDA model we assume that the covar matrices of individual inputs are diagonal, then the inputs are assumed to be conditionally independent in each class which matches the Gaussian Naive Bayes classifier.
+
+### Shrinkage and covar estimator
+
+Shrinkage is a regularisation method that improves covar matrix estimation, especially when there are few samples relative to features. This technique is used in LDA to enhance classifier generalisation. Setting the shrinkage parameter to 'auto' enables optimal shrinkage calculation via Ledoit and Wolf’s method, applicable only with lsqr or eigen solvers.
+
+The shrinkage parameter can also be set manually between 0 (no shrinkage) and 1 (full shrinkage), balancing between empirical covariance and a diagonal variance matrix. For normally distributed data, the Oracle Approximating Shrinkage (OAS) estimator often yields lower mean squared error and can improve LDA classification accuracy over Ledoit and Wolf’s approach.
+
+![image](https://github.com/user-attachments/assets/11e92552-fc15-44b8-8548-6d87141da333)
+
+### Estimation algorithms
+
+Using LDA and QDA requires computing the log-posterior which depends on the class priors, the class means, and the covar matrix.
+
+The ‘svd’ solver is the default solver used for LDA, and it is the only available solver for QDA. It can perform both classification and transform (for LDA). As it does not rely on the calculation of the covar matrix, the ‘svd’ solver may be preferable in situations where the number of features is large. The ‘svd’ solver cannot be used with shrinkage.
+
+The ‘lsqr’ solver is an efficient algorithm that only works for classification. It needs to explicitly compute the covar matrix, and supports shrinkage and custom covariance estimators.
+
+The 'eigen' solver is based on the optimisation of the between class scatter to within class scatter ratio. It can be used for both classification and transformation, and supports shrinkage. However, the ‘eigen’ solver needs to compute the covar matrix, so it might not be suitable for situations with a high number of features.
+
+## [Kernel ridge regression](https://scikit-learn.org/stable/modules/kernel_ridge.html#kernel-ridge-regression)
+
+KRR combines Ridge regression and classification (linear least squares with l2 norm reg) with the kernel trick. It learns a linear function in the space induced by the respective kernel and the data. For non-linear kernels, this corresponds to a non-linear function in the original space.
+
+The form of the model learned by KRR is identical to SVR, but the loss functions are different. KRR uses squared error loss, while SVR uses **epsilon-insensitive loss**, both combined with l2 reg.
+
+```
+Note: https://library.fiveable.me/key-terms/predictive-analytics-in-business/epsilon-insensitive-loss-function
+
+1. The epsilon-insensitive loss function allows for a certain degree of error within the epsilon range, which simplifies the training process by ignoring small discrepancies.
+2. By focusing on larger errors, this loss function aids in improving model performance on real-world datasets where noise and outliers are common.
+3. The choice of epsilon can significantly affect the complexity of the model; smaller epsilon values may lead to more complex models, while larger values promote simpler models.
+4. The implementation of an epsilon-insensitive loss function is a core feature that differentiates support vector regression from traditional regression methods.
+5. Utilizing this loss function can enhance computational efficiency by reducing the number of support vectors needed to define the model.
+```
+
+In contrast to SVR, fitting KRR can be done in closed-form and is typically faster for medium-sized datasets. On the other hand, the learned model is non-sparse and thus slower than SVR, which learns a sparse model for epsilon > 0, at prediction-time.
+
+The below image compares KRR and SVR on an artificial dataset with a sin target function and strong noise. The learned functions are very similar, however, fitting KRR is 7x faster, but prediction of 100,000 target values is >3x faster with SVR (since it has learned a sparse model using only approx. 1/3 of the 100 training datapoints as support vectors)
+
+![image](https://github.com/user-attachments/assets/8b8d7c7c-0fd6-472c-885e-a84817fd01e6)
+
+SVR's sparsity depends on C and epsilon.
+
+```
+C: float, default=1.0
+Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2.
+
+epsilon: float, default=0.1
+Epsilon in the epsilon-SVR model. It specifies the epsilon-tube within which no penalty is associated in the training loss function with points predicted within a distance epsilon from the actual value. Must be non-negative.
+```
+
+They even say: 
+
+_The fit time complexity is more than quadratic with the number of samples which makes it hard to scale to datasets with more than a couple of 10000 samples. For large datasets consider using LinearSVR or SGDRegressor instead, possibly after a Nystroem transformer or other Kernel Approximation._
+
+## [Support Vector Machines](https://scikit-learn.org/stable/modules/svm.html)
+
+SVMs are a set of supervised learning methods used for classification, regression and outlier detection
+
+**Advantages:**
+
+* effective in high dim spaces
+* still effective in cases where number of dims is greater than the number of samples
+* uses a subset of training points in the decision function (called support vectors), so it is memory efficient
+* versatile: different kernel functions can be specified for the decision function. Common kernels are provided, but there is also the option for custom ones
+
+**Disadvantages:**
+
+* if the number of features is much greater than the number of samples, avoid overfitting in choosing kernel functions and regularisation term is crucial
+* SVMs do not directly provide probability estimates, these are calculated using an expensive 5-fold CV
+
+Sklearn's SVMs support both dense and sparse sample vectors as input. However, to use an SVM to make preds on sparse data, it must have been fit on such data. 
+
+### Classification
+
+In sklearn, SVC, NuSVC and LinearSVC are classes capable of performing binary and multi-class classification.
+
+SVC and NuSVC are similar methods, but accept slightly different sets of parameters and have different mathematical formulations. On the other hand, LinearSVC is another (faster) implementation of Support Vector Classification for the case of a linear kernel. It also lacks some of the attributes of SVC and NuSVC, like support_ (which gives the indices of support vectors). LinearSVC uses squared_hinge loss and due to its implementation in liblinear it also regularizes the intercept, if considered. This effect can however be reduced by carefully fine tuning its intercept_scaling parameter, which allows the intercept term to have a different regularisation behavior compared to the other features. The classification results and score can therefore differ from the other two classifiers.
+
+#### Multi-class classification
+
+SVC and NuSVC implement the 'one-versus-one' approach for multi-class classification. In total, `n_classes * (n_classes - 1) / 2` classifiers are constructed and each one trains data from two classes. To provide a consistent interface with other classifiers, the decision_function_shape option allows to monotonically transform the results of the “one-versus-one” classifiers to a “one-vs-rest” decision function of shape (n_samples, n_classes), which is the default setting of the parameter (default=’ovr’).
+
+Here is some extra info about the `decision_function_shape` param:
+
+```
+decision_function_shape: {‘ovo’, ‘ovr’}, default=’ovr’
+
+Whether to return a one-vs-rest (‘ovr’) decision function of shape (n_samples, n_classes) as all other classifiers, or the original one-vs-one (‘ovo’) decision function of libsvm which has shape (n_samples, n_classes * (n_classes - 1) / 2). However, note that internally, one-vs-one (‘ovo’) is always used as a multi-class strategy to train models; an ovr matrix is only constructed from the ovo matrix. The parameter is ignored for binary classification.
+```
+
+#### ([Ref to Scores and probabilities](https://scikit-learn.org/stable/modules/svm.html#scores-and-probabilities) 
+
+To learn more about the `decision_function` method of SVC and NuSVC)
+
+#### Imbalanced problems
+
+When we want to give more importance to certain classes or samples, we can use the `class_weight` and `sample_weight` params.
+
+SVC (but not NuSVC) implements the parameter class_weight in the fit method. It’s a dictionary of the form {class_label : value}, where value is a floating point number > 0 that sets the parameter C of class class_label to C * value. The figure below illustrates the decision boundary of an unbalanced problem, with and without weight correction.
+
+![image](https://github.com/user-attachments/assets/f60aec5f-b05d-4685-b034-a8389f8cf76d)
+
+`sample_weight`, sets the parameter C for the i-th example to C * sample_weight[i], which will encourage the classifier to get these samples right. The figure below illustrates the effect of sample weighting on the decision boundary. The size of the circles is proportional to the sample weights
+
+![image](https://github.com/user-attachments/assets/547df2ad-4444-4cd0-8db5-24c40c8a9324)
+
+### Regression
+
+The model produced by SVC depends only on a subset of the training data, because the cost function for building it doesn't care about training points that lie beyond the margin
+
+Similarly, SVR ignores samples whose prediction is close to their target
+
+There are three different implementations of Support Vector Regression: SVR, NuSVR and LinearSVR. LinearSVR provides a faster implementation than SVR but only considers the linear kernel, while NuSVR implements a slightly different formulation than SVR and LinearSVR. Due to its implementation in liblinear LinearSVR also regularizes the intercept, if considered. This effect can however be reduced by carefully fine tuning its intercept_scaling parameter, which allows the intercept term to have a different regularization behavior compared to the other features. LinearSVR's classification results and score can therefore differ from the other two.
+
+### Outlier detection
+
+[OneClassSVM](https://scikit-learn.org/stable/modules/generated/sklearn.svm.OneClassSVM.html#sklearn.svm.OneClassSVM) is used for unsupervised outlier detection. 
+
+[This](https://scikit-learn.org/stable/auto_examples/applications/plot_outlier_detection_wine.html#sphx-glr-auto-examples-applications-plot-outlier-detection-wine-py) example which I remember reading on stream uses the OCSVM for outlier detection compared to 2 other methods:
+
+![image](https://github.com/user-attachments/assets/a5ced677-d947-41fd-8675-c420c7dd9232)
+
+OCSVM does not assume any parametric form of the data distribution and can model the complex shape better than the other two. There is also a link to [this](https://scikit-learn.org/stable/modules/outlier_detection.html#novelty-and-outlier-detection), but I will check that one later as it's part of the docs' 2nd 'chapter'. 
+
+### Complexity
+
+The core of an SVM is a quadratic programming proglem. The QP solver used by the libsvm-based implementation scales between O(n_features x n_samples^2) and O(n_features x n_samples^3).
+
+For the linear case, the algorithm used in LinearSVC by the liblinear implementation is much more efficient that its libsvm-based SVC couterpart and can scale almost linearly to millions of samples and/or features (hence for linear cases the LinearSVC was suggested above)
+
+### Tips on practical use
+
+* avoiding data copy
+
+For SVC, SVR, NuSVC and NuSVR, if the data passed to certain methods is not C-ordered contiguous and double precision, it will be copied before calling the underlying C implementation. Whether a given np array is C-contigious can be checked by inspecting its `flags` attribute
+
+For LinearSVC (and LogisticRegression for that matter), any input passed as a numpy array will be copied and converted to the liblinear internal sparse data representation. If we want to fit a large-scale linear classifier without copying a dense numpy C-contiguous double precision array as input, using SGDClassifier is suggested
+
+* kernel cache size
+
+For SVC, SVR, NuSVC and NuSVR, the size of the kernel cache has a strong impact on run times for larger problems. If we have enough RAM available, they recommended to set cache_size to a higher value than the default of 200MB, such as 500 or 1000
+
+* setting C
+
+LinearSVC and LinearSVR are less sensitive to C when it becomes large, and prediction results stop improving after a certain threshold. Meanwhile, larger C values will take more time to train, sometimes up to 10 times longer
+
+* scale your data
+
+SVMs are not scale invariant! For example, scale each attribute on the input vector X to [0,1] or [-1,+1], or standardize it to have mean 0 and variance 1. Note that the same scaling must be applied to the test vector to obtain meaningful results.
+
+* the parameter nu in NuSVC, OneClassSVM and NuSVR approximates the fraction of training errors and support vectors
+
+* in SVC, if the data is imbalanced set class_weight='balanced' and/or try different penalty parameters C
+
+* randomness of the underlying implementations
+
+_The underlying implementations of SVC and NuSVC use a random number generator only to shuffle the data for probability estimation (when probability is set to True). This randomness can be controlled with the random_state parameter. If probability is set to False these estimators are not random and random_state has no effect on the results. The underlying OneClassSVM implementation is similar to the ones of SVC and NuSVC. As no probability estimation is provided for OneClassSVM, it is not random.
+The underlying LinearSVC implementation uses a random number generator to select features when fitting the model with a dual coordinate descent (i.e. when dual is set to True). It is thus not uncommon to have slightly different results for the same input data. If that happens, try with a smaller tol parameter. This randomness can also be controlled with the random_state parameter. When dual is set to False the underlying implementation of LinearSVC is not random and random_state has no effect on the results._
+
+### Kernel functions
+
+<img width="709" alt="image" src="https://github.com/user-attachments/assets/cafcaa92-359b-4593-b556-8106907f38a1">
+
+#### Params of the RBF kernel
+
+* C trades off misclassification of training examples against simplicity of the decision surface
+* gamma defines how much influence a single training example has. The larger gamma is, the closer other examples must be to be affected
+
+Proper choice of C and gamma is critical, and the docs advise to use GridSearchCV with C and gamma spaced exponentially far apart to choose good values.
+
+![image](https://github.com/user-attachments/assets/49739809-d00c-44cd-b3b0-9001d4850af6)
+
+([pic source: RBF SVM parameters](https://scikit-learn.org/stable/auto_examples/svm/plot_rbf_parameters.html#sphx-glr-auto-examples-svm-plot-rbf-parameters-py))
+
+## [Stochastic Gradient Descent](https://scikit-learn.org/stable/modules/sgd.html#stochastic-gradient-descent)
+
+_Stochastic Gradient Descent (SGD) is a simple yet very efficient approach to fitting linear classifiers and regressors under convex loss functions such as (linear) Support Vector Machines and Logistic Regression. Even though SGD has been around in the machine learning community for a long time, it has received a considerable amount of attention just recently in the context of large-scale learning._
+
+SGD has seen much success in large-scale and sparse ML problems often encountered in text classification and NLP. Given that the data is sparse, the classifiers in this module easily scale to problems with 100,000 training samples and more than 100,000 features.
+
+SGD is an optimization technique and doesn't correspond to a specific family of machine learning models.
+
+Often, an instance of SGDClassifier or SGDRegressor will have an equivalent estimator in the scikit-learn API, potentially using a different optimization technique. For example, using SGDClassifier(loss='log_loss') results in logistic regression. Similarly, SGDRegressor(loss='squared_error', penalty='l2') and Ridge solve the same optimization problem, via different means.
+
+**Advantages:**
+
+* efficiency
+* ease of implementation (lots of opportunities for code tuning)
+
+**Disadvantages:**
+
+* it requires a number of hyperparams such as the regularisation param and the number of iterations
+* it is sensitive to feature scaling
+
+**Warning**
+
+_Make sure you permute (shuffle) your training data before fitting the model or use shuffle=True to shuffle after each iteration (used by default). Also, ideally, features should be standardized using e.g. make_pipeline(StandardScaler(), SGDClassifier())_
+
+### Classification
+
+The class SGDClassifier implements a plain SGD learning routine which supports different loss functions and penalties for classification. 
+
+If we train it using the hinge loss, it is equivalent to a linear SVM.
+
+**The loss param**
+
+* ‘hinge’ gives a linear SVM
+* ‘log_loss’ gives logistic regression, a probabilistic classifier
+* ‘modified_huber’ is another smooth loss that brings tolerance to outliers as well as probability estimates
+* ‘squared_hinge’ is like hinge but is quadratically penalized
+* ‘perceptron’ is the linear loss used by the perceptron algorithm.
+
+There are other losses - ‘squared_error’, ‘huber’, ‘epsilon_insensitive’ and ‘squared_epsilon_insensitive’ are designed for regression but can be useful in classification as well
+
+**Penalty**
+
+```
+penalty: {‘l2’, ‘l1’, ‘elasticnet’, None}, default=’l2’
+The penalty (aka regularization term) to be used. Defaults to ‘l2’ which is the standard regularizer for linear SVM models. ‘l1’ and ‘elasticnet’ might bring sparsity to the model (feature selection) not achievable with ‘l2’. No penalty is added when set to None.
+```
+
+SGDClassifier supports multi-class classification by combining multiple binary classifiers in a one-vs-all (OVA) scheme. For each of the classes, a binary classifier is learned that discriminates between that and all other classes. At testing time, we compute the confidence score (i.e. the signed distances to the hyperplane) for each classifier and choose the class with the highest confidence
+
+The below shows the OVA on the iris dataset. The dashed lines show the three OVA classifiers and the background colours show the decision surface induced by each calssifier
+
+![image](https://github.com/user-attachments/assets/98df3666-42f0-41b3-af8e-6a78aa5a3fac)
+
+(Using log_loss and modified_huber enable the usage of predict_proba so they are more suitible for OVA classification)
+
+The SGDClassifier supports both `class_weight` and `sample_weight` which we saw earlier in the SVM part as well.
+
+**Averaged SGD**
+
+This can be enabled by passing `average=True`. ASGD performs the same updates as the normal SGD, but instead of using the last value of the coefs (the values of the last update), the `coef_` attribute is set to the average value of the coefs across all updates. The same is done for the intercept_ attribute as well. When using ASGD the learning rate can be larger and even constant, leading on some datasets to a speed up in training time.
+
+### Regression
+
+_The class SGDRegressor implements a plain stochastic gradient descent learning routine which supports different loss functions and penalties to fit linear regression models. SGDRegressor is well suited for regression problems with a large number of training samples (> 10.000), for other problems we recommend Ridge, Lasso, or ElasticNet._
+
+**Loss**
+
+* loss="squared_error": Ordinary least squares
+* loss="huber": Huber loss for robust regression
+* loss="epsilon_insensitive": linear Support Vector Regression
+
+The Huber and epsilon-insensitive loss functions can be used for robust regression. The width of the insensitive region has to be specified via the parameter epsilon. This parameter depends on the scale of the target variables
+
+**Penalty**
+
+```
+penalty: {‘l2’, ‘l1’, ‘elasticnet’, None}, default=’l2’
+
+The penalty (aka regularization term) to be used. Defaults to ‘l2’ which is the standard regularizer for linear SVM models. ‘l1’ and ‘elasticnet’ might bring sparsity to the model (feature selection) not achievable with ‘l2’. No penalty is added when set to None.
+```
+
+### Online One-Class SVM
+
+_The class sklearn.linear_model.SGDOneClassSVM implements an online linear version of the One-Class SVM using a stochastic gradient descent. Combined with kernel approximation techniques, sklearn.linear_model.SGDOneClassSVM can be used to approximate the solution of a kernelized One-Class SVM, implemented in sklearn.svm.OneClassSVM, with a linear complexity in the number of samples. Note that the complexity of a kernelized One-Class SVM is at best quadratic in the number of samples. sklearn.linear_model.SGDOneClassSVM is thus well suited for datasets with a large number of training samples (> 10,000) for which the SGD variant can be several orders of magnitude faster._
+
+### Stochastic Gradient Descent for sparse data
+
+Sparse and dense formats are supported, but the sparse implementation produces slightly different results from the dense implementation. 
+
+### Complexity
+
+SGD's efficiency is basically linear in the # of training examples. If X is a matrix of size (n, p), training has a cost of O(k x n x p_), where k is the # of iterations, and p_ is the avg # of non-zero attributes per sample
+
+### Stopping criterion
+
+* _with early_stopping=True, the input data is split into a training set and a validation set. The model is then fitted on the training set, and the stopping criterion is based on the prediction score (using the score method) computed on the validation set. The size of the validation set can be changed with the parameter validation_fraction_
+* _with early_stopping=False, the model is fitted on the entire input data and the stopping criterion is based on the objective function computed on the training data_
+
+_In both cases, the criterion is evaluated once by epoch, and the algorithm stops when the criterion does not improve n_iter_no_change times in a row. The improvement is evaluated with absolute tolerance tol, and the algorithm stops in any case after a maximum number of iteration max_iter._
+
+### Tips on practical use
+
+* SGD is sensitive to feature scaling, so scaling to [0,1] or [-1,+1] is recommended
+* finding a good regularisation term alpha is best done using automatic hyperparam search (GridSearchCV or RandomizedSearchCV, usually in the range 10.0**-np.arange(1,7))
+* the docs suggest that SGD converges after observing approximately 10^6 training samples - thus, a reasonable first guess for the number of iterations is max_iter = np.ceil(10**6 / n), where n is the size of the training set
+* if SGD is applied to features extracted using PCA, it is often wise to scale the feature values by some constant c such that the avg L2 norm of the train data equals 1
+
+**Learning rate param**
+
+* ‘constant’: eta = eta0
+
+* ‘optimal’: eta = 1.0 / (alpha * (t + t0)) where t0 is chosen by a heuristic proposed by Leon Bottou.
+
+* ‘invscaling’: eta = eta0 / pow(t, power_t)
+
+* ‘adaptive’: eta = eta0, as long as the training keeps decreasing. Each time n_iter_no_change consecutive epochs fail to decrease the training loss by tol or fail to increase validation score by tol if early_stopping is True, the current learning rate is divided by 5.
+
+For SGDRegressor the default is invscaling, for SGDClassifier the default is optimal. 
+
+`eta` is the initial learning rate which can be set as well. Its default is 0.01.
+
+`power_t` is the exponent for inverse scaling learning rate. 
+
+`t` is the time step
+
+## [Nearest Neighbors](https://scikit-learn.org/stable/modules/neighbors.html#nearest-neighbors)
+
+_sklearn.neighbors provides functionality for unsupervised and supervised neighbors-based learning methods. Unsupervised nearest neighbors is the foundation of many other learning methods, notably manifold learning and spectral clustering. Supervised neighbors-based learning comes in two flavors: classification for data with discrete labels, and regression for data with continuous labels._
+
+The models can handle both dense and sparse input matrices. For dense matrices, a large number of possible distance metrics are supported. For sparse matrices, arbitrary Minkowski metrics (results in the standard Euclidean distance if p=2, if p=1 -> the Manhattan distance is used, other p results in minkowski_distance(l_p) being used) are supported for searches.
+
+### Unsupervised nearest neighbours
+
+`NearestNeighbors` implements these models, and it acts as a uniform interface to 3 different algorithms: BallTree, KDTree, and a brute-force algorithm based on routines in `sklearn.metrics.pairwise`. The hoice is controlled using the `algorithm` param, which is auto, ball_tree, kd_tree, or brute, and auto determines the best approach from the train data. 
+
+### Nearest neighbours classification
+
+_scikit-learn implements two different nearest neighbors classifiers: KNeighborsClassifier implements learning based on the k nearest neighbors of each query point, where k is an integer value specified by the user. RadiusNeighborsClassifier implements learning based on the number of neighbors within a fixed radius r of each training point, where r is a floating-point value specified by the user._
+
+The radious classifier can be an option where the data is not uniformly sampled, and with a specific r points in sparser n'hoods use fewer nearest neighbours for the classification. For high-dim, this becomes less effective due to the curse of dimensionality.
+
+**Weight param**
+
+_The basic nearest neighbors classification uses uniform weights: that is, the value assigned to a query point is computed from a simple majority vote of the nearest neighbors. Under some circumstances, it is better to weight the neighbors such that nearer neighbors contribute more to the fit._
+
+`weights: {'uniform', 'distance'}` (or an user-defined callable)
+
+![image](https://github.com/user-attachments/assets/9b00a138-a4d1-482f-9fcb-1396997e6744)
+
+### Nearest neighbours regression
+
+_scikit-learn implements two different neighbors regressors: KNeighborsRegressor implements learning based on the k nearest neighbors of each query point, where k is an integer value specified by the user. RadiusNeighborsRegressor implements learning based on the neighbors within a fixed radius r of the query point, where r is a floating-point value specified by the user._
+
+Again there is the weights param which can be uniform or distance (or an user-defined callable)
+
+![image](https://github.com/user-attachments/assets/e5098344-ed66-4921-b178-cdac39c44c1a)
+
+### Nearest neighbour algorithms
+
+**Brute force**
+
+The most naive neighbor search implementation involves the brute-force computation of distances between all pairs of points in the dataset - this approach is O(DN^2), D - dimensions, and N samples. This can be god for small datasets, however it can quickly become inefficient as N grows. 
+
+**K-D Tree**
+
+To improve upon the brute force, structures that attempt to reduce the required number of distance calculations started to appear. For instance, if A is far from B, and B is close to C, then the model doesn't need to calculate the distance b/e A and C and can assume they are not close. This can reduce the computational cost to O(DNlog(N)) or better. 
+
+The KD tree (K-dimensional tree) is a binary tree data structure that partitions space along data axes, creating nested regions for data points. It enables fast nearest-neighbour searches in low-dim spaces D < 20 with an O(log(N)) complexity for distance calculations. However, as dims increase, KD trees become inefficient due to the curse of dimensionality.
+
+**Ball Tree**
+
+To address the inefficiencies of the kd_tree algorithm in higher dims, the ball tree structure was developed. This algorithm paritions the data in a series of nesting hyper-spheres, and while the tree construction is morecostly than the KD tree, it results in a data structure which can be very efficient on highly structured data, even in high dimensions. 
+
+A ball tree recursively divides the data into nodes defined by a centroid C and radius r, such that each point in the node lies within the hyper-sphere defined by r and C. Each node contains points within a hyper-sphere around its centroid. By using the triangle inequality (`|x+y| <= |x| + |y|`), the ball tree reduces the number of candidate points for a neighbor search, as it allows quick elimination of nodes that can't contain nearby points. This approach can outperform a KD-tree in higher dimensions, though its efficiency depends on the data structure
+
+There is a [section](https://scikit-learn.org/stable/modules/neighbors.html#choice-of-nearest-neighbors-algorithm) on the choice of nearest neighbour algorithm which I will bookmark and remember to check out with some examples on stream. I can also try out different distance metrics, and just play around with all the different parameters in detail.
+
+### Nearest centroid classifier
+
+_The NearestCentroid classifier is a simple algorithm that represents each class by the centroid of its members. In effect, this makes it similar to the label updating phase of the KMeans algorithm. It also has no parameters to choose, making it a good baseline classifier. It does, however, suffer on non-convex classes, as well as when classes have drastically different variances, as equal variance in all dimensions is assumed_
+
+### Nearest neighbours transformer
+
+_Many scikit-learn estimators rely on nearest neighbors: Several classifiers and regressors such as KNeighborsClassifier and KNeighborsRegressor, but also some clustering methods such as DBSCAN and SpectralClustering, and some manifold embeddings such as TSNE and Isomap.
+All these estimators can compute internally the nearest neighbors, but most of them also accept precomputed nearest neighbors sparse graph, as given by `kneighbors_graph` and `radius_neighbors_graph`. With `mode='connectivity'`, these functions return a binary adjacency sparse graph as required, for instance, in `SpectralClustering`. Whereas with `mode='distance'`, they return a distance sparse graph as required, for instance, in `DBSCAN`. To include these functions in a scikit-learn pipeline, one can also use the corresponding classes KNeighborsTransformer and RadiusNeighborsTransformer._ 
+
+The benefits of this sparse graph API are:
+
+* the precomputed graph can be re-used multiple times
+* precomputing the graph can give finer control on the nearest neighbour estimation
+* the precomputation can be done using custom estimators, such as ANN
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 5bb59794c952bd94b84c5a69a6890b80bec3cf51 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 9 Nov 2024 23:46:13 +0900
Subject: [PATCH 491/638] Create 2024-11-09-day-313.md

---
 _posts/2024-11-09-day-313.md | 511 +++++++++++++++++++++++++++++++++++
 1 file changed, 511 insertions(+)
 create mode 100644 _posts/2024-11-09-day-313.md

diff --git a/_posts/2024-11-09-day-313.md b/_posts/2024-11-09-day-313.md
new file mode 100644
index 000000000..9cd6fd3a5
--- /dev/null
+++ b/_posts/2024-11-09-day-313.md
@@ -0,0 +1,511 @@
+---
+layout: post
+title: (Day 313) More of sklearn's 'user guide'
+categories: [theory,traditional-machine-learning,statistics]
+---
+
+# Hello :) Today is Day 313!
+A quick summary of today:
+* gaussian processes
+* cross decomposition
+* naive Bayes
+* decision trees
+* semi-supervised learning
+* manifold learning
+* clustering
+* biclustering
+
+
+## [Gaussian Processes](https://scikit-learn.org/stable/modules/gaussian_process.html)
+
+_Gaussian Processes (GP) are a nonparametric supervised learning method used to solve regression and probabilistic classification problems._
+
+**Advantages:**
+
+* the prediction interpolates the observations (at least for regular kernels)
+* the prediction is probabilistic (Gaussian) so that one can compute empirical confidence intervals and decide based on those if one should refit the prediction in some region of interest
+* versatile: different kernels can be specified
+
+**Disadvantages:**
+
+* sklearn's implementation is not sparse - it uses the whole feature space to perform predictions
+* they lose efficiency in high dim space - namely when the # of features exceeds a few dozens
+
+### Gaussian Process Regression (GPR)
+
+_The `GaussianProcessRegressor` implements Gaussian processes (GP) for regression purposes. For this, the prior of the GP needs to be specified. GP will combine this prior and the likelihood function based on training samples. It allows to give a probabilistic approach to prediction by giving the mean and standard deviation as output when predicting._
+
+![image](https://github.com/user-attachments/assets/97db5909-348a-4996-9259-0b4883d4c1d4)
+
+**alpha**
+
+Noise level can be specified through setting alpha. An alternative is to use a `WhiteKernel` component in the kernel, which can estimate the global noise level from the data
+
+**normalize_y**
+
+_Whether or not to normalize the target values y by removing the mean and scaling to unit-variance. This is recommended for cases where zero-mean, unit-variance priors are used. Note that, in this implementation, the normalisation is reversed before the GP predictions are reported._
+
+GPR also:
+
+* allows prediction without prior fitting (based on the GP prior)
+* provides an additional method sample_y(X), which evaluates samples drawn from the GPR (prior or posterior) at given inputs
+* exposes a method log_marginal_likelihood(theta), which can be used externally for other ways of selecting hyperparameters, for instance, via Markov chain Monte Carlo
+
+### Gaussian Process Classification (GPC)
+
+_The GaussianProcessClassifier implements Gaussian processes (GP) for classification purposes, more specifically for probabilistic classification, where test predictions take the form of class probabilities. GaussianProcessClassifier places a GP prior on a latent function f, which is then squashed through a link function to obtain the probabilistic classification. The latent function f is a so-called nuisance function, whose values are not observed and are not relevant by themselves. Its purpose is to allow a convenient formulation of the model, and f is removed (integrated out) during prediction. GaussianProcessClassifier implements the logistic link function, for which the integral cannot be computed analytically but is easily approximated in the binary case._
+
+In contrast to GPR, the posterior of the function f is not Gaussian since Gaussian likelihood is inappropriate for discrete class labels. Instead, a non-Gaussian likelihood corresponding to the logistic link function (logit) is used.
+
+GPC supports multi-class classification by performing one-vs-rest or one-vs-one based training and prediction. In one-versus-rest, one binary Gaussian process classifier is fitted for each class, which is trained to separate this class from the rest. In “one_vs_one”, one binary Gaussian process classifier is fitted for each pair of classes, which is trained to separate these two classes. 
+
+### Kernels for GPs
+
+I went on to check out [this youtube video](https://youtu.be/UBDgSHPxVME) and the explanation is so great, and it cleared a lot of confusion about GPs and also about kernels. 
+
+**Constant Kernel**
+
+This kernel scales the magnitude of the other factor (kernel) and depends on a constant value
+
+**WhiteKernel**
+
+The main use-case of this is as part of a sum-kernel where it explains the noise-component of the signal. Tuning its parameter noise_level corresponds to estimating the noise-level.
+
+**Kernel operators**
+
+Sum - takes two kernels and combines them via 
+
+`k_sum(X,Y) = k1(X,Y) + k2(X,Y)`
+
+Product - takes two kernels and combines them via
+
+`k_product(X,Y) = k1(X,Y) x k2(X,Y)`
+
+Exponentiation - takes one base kernel and a scalar param p and combines them via
+
+`k_exp(X,Y) = k(X,Y)^p`
+
+**Radial basis function kernel**
+
+It is known as the squared exponential kernel. It is parametrized by a length-scale param (`l > 0`), which can either be a scalar or a vector with the same dim as the input x
+
+<img width="356" alt="image" src="https://github.com/user-attachments/assets/a32aef80-73ce-49e1-8c18-96345ede7b18">
+
+`d(o,o)` is the Euclidean distance. This kernel is infinitely differentiable, which implies that GPs with this kernel as covar function have mean square derivatives of all orders, and thus are very smooth.
+
+![image](https://github.com/user-attachments/assets/e4c4c6dc-5c7c-4fdf-98f3-7bb25620eb29)
+
+**Matern kernel**
+
+![image](https://github.com/user-attachments/assets/3d7ef802-58cf-4f3d-b1ec-2774d7d6d1e2)
+
+**Rational quadratic kernel**
+
+![image](https://github.com/user-attachments/assets/5252fedb-a79a-4905-b2b7-f6ccdeb3e80d)
+
+**Exp-Sine-Squared kernel**
+
+![image](https://github.com/user-attachments/assets/5e47c284-dcf6-45ad-a61d-7eab26ccfec4)
+
+As mentioned in the video I linked, creating an effective kernel (usually a combination of kernels) and finding the best hyperparameters can be difficult, so it requires trial and error. 
+
+## [Cross decomposition](https://scikit-learn.org/stable/modules/cross_decomposition.html)
+
+_The cross decomposition module contains supervised estimators for dimensionality reduction and regression, belonging to the “Partial Least Squares” family._
+
+_Cross decomposition algorithms find the fundamental relations between two matrices (X and Y). They are latent variable approaches to modeling the covariance structures in these two spaces. They will try to find the multidimensional direction in the X space that explains the maximum multidimensional variance direction in the Y space. In other words, PLS projects both X and Y into a lower-dimensional subspace such that the covariance between transformed(X) and transformed(Y) is maximal._
+
+PLS has similarities with Principal Component Regression (PCR), but PCR is an unsupervised dim reduction method.
+
+Apart from Canonical Correlation Analysis (CCA), the PLS estimators are suited when the matrix of predictors has more variables than observations, and when there is multicollinearity among the features. By contrast, standard linear regression would fail in these cases unless it is regularized.
+
+This module containes the classes: PLSRegression, PLSCanonical, CCA, and PLSSVD
+
+[This example](https://scikit-learn.org/stable/auto_examples/cross_decomposition/plot_compare_cross_decomposition.html#sphx-glr-auto-examples-cross-decomposition-plot-compare-cross-decomposition-py) shows a comparison between the 4 methods.
+
+## [Naive Bayes](https://scikit-learn.org/stable/modules/naive_bayes.html#naive-bayes)
+
+_Naive Bayes methods are a set of supervised learning algorithms based on applying Bayes’ theorem with the “naive” assumption of conditional independence between every pair of features given the value of the class variable._
+
+Regardless of the naive assumption, naive Bayes classifiers have worked quite well in many real-world scenarios (document classification and spam filtering). They require a small amount of train data to estimate the necessary params. Furthermore, naive Bayes learners and classifieers can be extremely fast compared to more sophisticated methods. 
+
+On the other handl, the naive Bayes classifiers are known to be bad estimators, so the output from their `predict_proba` should not be trusted.
+
+### Gaussian Naive Bayes
+
+`GaussianNB` implements the Gaussian Naive Bayes algorithm for classification, where the likelihood of the features is assumed to be Gaussian and the params sigma and my are found through maximum likelihood
+
+### Multinomial Naive Bayes
+
+`MultinomialNB` - The multinomial Naive Bayes classifier is suitable for classification with discrete features (e.g., word counts for text classification). The multinomial distribution normally requires integer feature counts. However, in practice, fractional counts such as tf-idf may also work.
+
+### Complement Naive Bayes
+
+_`ComplementNB` implements the complement naive Bayes (CNB) algorithm. CNB is an adaptation of the standard multinomial naive Bayes (MNB) algorithm that is particularly suited for imbalanced data sets. Specifically, CNB uses statistics from the complement of each class to compute the model’s weights. The inventors of CNB show empirically that the parameter estimates for CNB are more stable than those for MNB. Further, CNB regularly outperforms MNB (often by a considerable margin) on text classification tasks._
+
+### Bernoulli Naive Bayes
+
+_`BernoulliNB` implements the naive Bayes training and classification algorithms for data that is distributed according to multivariate Bernoulli distributions; i.e., there may be multiple features but each one is assumed to be a binary-valued (Bernoulli, boolean) variable. Therefore, this class requires samples to be represented as binary-valued feature vectors; if handed any other kind of data, a BernoulliNB instance may binarize its input_
+
+The decision rule here differs from the multinomial NB's rule in that it explicitly penalises the non-coccurance of a feature i that is an indicator for class y, where the multinomial variant would simply ignore a non-occuring feature. 
+
+For text classification, word occurance vectors (not word count vectors) may be used to train BNB, and it might even perform better on some datasets, sepecially those with shorter docs. 
+
+### Categorical Naive Bayes
+
+`CategoricalNB` is a Naive Bayes classifier for categorical data. It estimates a categorical distribution for each feature conditioned on the class label. The data must be encoded (like using OrdinalEncoder) so each feature's categories are represented as integers.
+
+### Out-of-core naive Bayes model fitting
+
+We can use `partial_fit` in cases where the full training set might not fit in memory. 
+
+## [Decision Trees](https://scikit-learn.org/stable/modules/tree.html#decision-trees)
+
+_Decision Trees (DTs) are a non-parametric supervised learning method used for classification and regression. The goal is to create a model that predicts the value of a target variable by learning simple decision rules inferred from the data features. A tree can be seen as a piecewise constant approximation._
+
+**Advantages:**
+
+* simple to understand and interpret
+* requires little data prep; other techniques often require data normalisation, dummy vars need to be created and blank values to be removed; some tree algs support missing values
+* the cost of using the tree is logarithmic in the # of data points used to train the tree
+* able to handle both numerical and cat data; however, the sklearn implementation does not support cat vars _for now_
+* able to handle multi-output problems
+* uses a white box model - if a given situation is observable in a model, the explanation for the condition is easlity explained by boolean logic
+* possible to validate a model using stat tests; that makes it possible to acount for the reliability of the model
+* performs well even if its assumptions are somewhat violated by the true model from which the data were generated
+
+**Disadvantages:**
+
+* decision tree learners can create over-complex trees that do not generalise the data well
+* can be unstable because small variations in the data might result in a completely different tree being generated (fixed through ensemble methods)
+* predictions of decision trees are neither smooth nor continuous, but piecewise constant approximations - they are not good at extrapolation
+* the problem of learning an optimal deicision tree is known to be NP-complete
+* there are concepts that are hard to learn because decision trees do not express them easily - such as XOR, parity or multiplexer problems
+* learners create biased trees if some classes dominate, so it's recommended to balance the data prior to fitting
+
+### Complexity
+
+In general, the run time cost to construct a balanced binary tree is `O(n_samples x n_featuers x log(n_samples))` and query time `O(log(n_samples))`. Although the tree construction alg attempts to generated balanced trees, they will not always be balanced. Assuming te subtrees remain approx balanced, the cost at each node consists of searching through O(n_features) to find the feature that offers the largest reduction in the impurity criterion. This has a cost of `O(n_samples x n_featuers x log(n_samples))` and query time `O(log(n_samples))` at each node, leading to a total cost over the entire trees
+`O(n_samples^2 x n_featuers x log(n_samples)) and query time O(log(n_samples))`.
+
+### Tips on practical use
+
+* decision trees tend to overfit on data with a large number of features. Getting the right ratio of samples to features is important since a tree with few samples in high dims is very likely to overfit
+* consider performing dim reduction beforehand to give the tree a better chance of finding features that are discriminative
+* understanding the decition tree structure will help in gaining more insights about how the decision tree makes preds, which is important for understanding the important features in the data
+* visualise the tree as you are training, use max_depth=3 to start with and get a feel of how the tree is fitting, and then increase the depth accordingly
+* the number of samples required to populate the tree doubles for each additional level the tree grows to. Use max_depth to control the size of the tree to prevent overfitting
+* use min_samples_split or min_samples_leaf to ensure that multiple samples inform every decision in the tree by controlling which splits will be considered. A very small number will usually mean the tree will overfit, and a large number will prevent the tree for learning the data. Try min_samples_leaf=5 as an initial value. If the sample size varies greatly, a float number can be used as percentage in these two parameters. While min_samples_split can create arbitrarily small leaves, min_samples_leaf guarantees that each leaf has a minimum size, avoiding low-variance, over-fit leaf nodes in regression problems. For classification with few classes, min_samples_leaf=1 is often the best choice. Note that min_samples_split considers samples directly and independent of sample_weight, if provided. Consider min_weight_fraction_leaf or min_impurity_decrease if accounting for sample weights is required at splits
+* balance the data before training to prevent the tree from being biased towards the dominant classes. This can be done by sampling an equal number of samples from each class, or preferably by normalising the sum of the sample_weights for each class to the same value
+* if the samples are weighted, it will be easier to optimize the tree structure using weight-based pre-pruning criterion such as min_weight_fraction_leaf, which ensure that leaf nodes contain at least a fraction of the overall sum of the sample weights
+* all decision trees use np.float32 arrays internally. If training data is not in this format, a copy of the dataset will be made
+* if the input matrix X is very sparse, it is recommended to convert to sparse csc_matrix before calling fit and sparse csr_matrix before calling predict. Training time can be orders of magnitude faster for a sparse matrix input compared to a dense matrix when features have zero values in most of the samples
+
+### Missing values support
+
+_DecisionTreeClassifier and DecisionTreeRegressor have built-in support for missing values when splitter='best' and criterion is 'gini', 'entropy’, or 'log_loss', for classification or 'squared_error', 'friedman_mse', or 'poisson' for regression. For each potential threshold on the non-missing data, the splitter will evaluate the split with all the missing values going to the left node or the right node._
+
+Decisions are made as follows:
+
+* by default when predicting, the samples with missing values are classified with the class used in the split found during training
+* if the criterion evaluation is the same for both nodes, then the tie for missing value at predict time is broken by going to the right node. The splitter also checks the split where all the missing values go to one child and non-missing values go to the other
+* if no missing values are seen during training for a given feature, then during prediction missing values are mapped to the child with the most samples
+
+## [Semi-supervised learning](https://scikit-learn.org/stable/modules/semi_supervised.html)
+
+_Semi-supervised learning is a situation in which in your training data some of the samples are not labeled. The semi-supervised estimators in sklearn.semi_supervised are able to make use of this additional unlabeled data to better capture the shape of the underlying data distribution and generalize better to new samples. These algorithms can perform well when we have a very small amount of labeled points and a large amount of unlabeled points._
+
+It's important to assign an intifier to unlabeled points, the module sees -1 as unlabaled. 
+
+Semi-supervised learning algorithms make use of at least one of the following assumptions:
+
+* continuity/smoothness assumption
+
+Points that are close to each other are more likely to share a label
+
+* cluster assumption
+
+The data tend to form discrete clusters, and points in the same cluster are more likely to share a label
+
+* manifold assumption
+
+The data lie approximately on a manifold of much lower dimension than the input space
+
+### Self Training
+
+This self-training implementation is based on Yarowsky’s algorithm where a given supervised classifier can function as a semi-supervised classifier, allowing it to learn from unlabeled data.
+
+`SelfTrainingClassifier` can be called with any classifier that implements `predict_proba`, passed as the parameter `base_classifier`. In each iteration, the base classifier predicts labels for the unlabeled samples and adds a subset of these labels to the labeled dataset.
+
+The choice of this subset is determined by the selection criterion. This selection can be done using a threshold on the prediction probabilities, or by choosing the k_best samples according to the prediction probabilities.
+
+The labels used for the final fit as well as the iteration in which each sample was labeled are available as attributes. The optional max_iter parameter specifies how many times the loop is executed at most. The max_iter parameter may be set to None, causing the algorithm to iterate until all samples have labels or no new samples are selected in that iteration.
+
+_When using the self-training classifier, the calibration of the classifier is important._
+
+### Label Propagation
+
+Label propagation denotes a few variations of semi-supervised graph inference algorithms.
+
+A few features available in this model:
+
+* used for classification tasks
+* kernel methods to project data into alternate dimensional spaces
+
+sklearn provides two label propagation models: LabelPropagation and LabelSpreading. Both work by constructing a similarity graph over all items in the input dataset.
+
+LabelPropagation uses hard clamping of input labels, meaning it strictly retains the original label distribution. The clamping factor can be adjusted, allowing for some flexibility in label distribution updates. The algorithm uses the raw similarity matrix without modifications.
+
+LabelSpreading minimises a loss function with regularization, making it more robust to noise. It iterates on a modified graph and normalizes edge weights using the normalized graph Laplacian, a method also used in Spectral clustering.
+
+Both models offer two kernels:
+
+* rbf: produces a fully connected graph, represented by a large, dense matrix. While more accurate, it can result in longer running times due to memory usage and matrix calculations
+* knn: produces a sparse matrix, reducing memory usage and running times significantly
+
+## [Manifold learning](https://scikit-learn.org/stable/modules/manifold.html#manifold-learning)
+
+_Manifold learning is an approach to non-linear dimensionality reduction. Algorithms for this task are based on the idea that the dimensionality of many data sets is only artificially high._
+
+_Manifold Learning can be thought of as an attempt to generalize linear frameworks like PCA to be sensitive to non-linear structure in data. Though supervised variants exist, the typical manifold learning problem is unsupervised: it learns the high-dimensional structure of the data from the data itself, without the use of predetermined classifications._
+
+Below are sklearn's manifold learning implementations and the pictures are of each method applied on
+
+![image](https://github.com/user-attachments/assets/107f979c-9385-4d35-84b0-c53938e76b03)
+
+### Isomap
+
+_One of the earliest approaches to manifold learning is the Isomap algorithm, short for Isometric Mapping. Isomap can be viewed as an extension of Multi-dimensional Scaling (MDS) or Kernel PCA. Isomap seeks a lower-dimensional embedding which maintains geodesic distances between all points._
+
+![image](https://github.com/user-attachments/assets/e7efc15c-7917-46f4-aa1e-2f1eff566ba1)
+
+### Locally Linear Embedding
+
+_LLE seeks a lower-dimensional projection of the data which preserves distances within local neighborhoods. It can be thought of as a series of local Principal Component Analyses which are globally compared to find the best non-linear embedding._
+
+### Modified LLE
+
+_One well-known issue with LLE is the regularization problem. When the number of neighbors is greater than the number of input dimensions, the matrix defining each local neighborhood is rank-deficient. One method to address the regularization problem is to use multiple weight vectors in each neighborhood. This is the essence of modified locally linear embedding (MLLE)._
+
+![image](https://github.com/user-attachments/assets/89300b74-ec7b-4bfd-b63b-f0e2b977e140)
+
+### Heissian Eigenmapping
+
+_This is another method of solving the regularization problem of LLE. It revolves around a hessian-based quadratic form at each neighborhood which is used to recover the locally linear structure. Though other implementations note its poor scaling with data size, sklearn implements some algorithmic improvements which make its cost comparable to that of other LLE variants for small output dimension._
+
+![image](https://github.com/user-attachments/assets/d7696bdd-b0c9-4ac0-8ed2-a9e677d0f49c)
+
+### Spectral Embedding
+
+_This is an approach to calculating a non-linear embedding. Scikit-learn implements Laplacian Eigenmaps, which finds a low dimensional representation of the data using a spectral decomposition of the graph Laplacian. The graph generated can be considered as a discrete approximation of the low dimensional manifold in the high dimensional space. Minimization of a cost function based on the graph ensures that points close to each other on the manifold are mapped close to each other in the low dimensional space, preserving local distances_
+
+### Local Tangent Space Alignment
+
+_Though not technically a variant of LLE, Local tangent space alignment (LTSA) is algorithmically similar enough to LLE that it can be put in this category. Rather than focusing on preserving neighborhood distances as in LLE, LTSA seeks to characterize the local geometry at each neighborhood via its tangent space, and performs a global optimization to align these local tangent spaces to learn the embedding._
+
+### Multi-dimensional Scaling
+
+_MDS seeks a low-dimensional representation of the data in which the distances respect well the distances in the original high-dimensional space. In general, its is a technique used for analyzing similarity or dissimilarity data. It attempts to model similarity or dissimilarity data as distances in a geometric spaces. The data can be ratings of similarity between objects, interaction frequencies of molecules, or trade indices between countries_
+
+![image](https://github.com/user-attachments/assets/7eabce7d-4066-4886-abff-c27ac77d5525)
+
+### t-distributed Stochastic Neighbour Embedding (t-SNE)
+
+t-SNE converts data affinities to probabilities, using Gaussian distributions in the original space and Student's t-distributions in the embedded space. This approach makes t-SNE sensitive to local structures, with advantages such as revealing structure at multiple scales, uncovering multiple manifolds or clusters, and reducing point crowding at the center. Unlike Isomap or LLE, which focus on unfolding a single manifold, t-SNE emphasizes local data structure and can group samples based on local similarities, which is useful for visualizing datasets with multiple clusters.
+
+The algorithm minimizes the Kullback-Leibler (KL) divergence between joint probabilities using gradient descent. Since the KL divergence is not convex, multiple restarts with different initializations may be needed to avoid local minima.
+
+![image](https://github.com/user-attachments/assets/00d3e2fb-7685-4486-9625-11760a49c007)
+
+### Practical tips
+
+* ensure all features are on the same scale to improve algorithm performance, as manifold learning relies on nearest-neighbor search
+* the reconstruction error can guide the selection of the optimal output dimension. It decreases as `n_components` increases, until it matches the dimensionality of the data
+* noisy data can distort the manifold, connecting parts of it that would otherwise be separate. Manifold learning on noisy or incomplete data remains an active research topic
+* identical points or disjoint data groups can lead to singular weight matrices, causing issues with solvers like `arpack`. Use `solver='dense'` to handle singular matrices, though it may be slower with large datasets. Alternatively, resolve the issue by increasing `n_neighbors` for disjoint data or removing identical points
+
+## [Clustering](https://scikit-learn.org/stable/modules/clustering.html)
+
+_Each clustering algorithm comes in two variants: a class, that implements the fit method to learn the clusters on train data, and a function, that, given train data, returns an array of integer labels corresponding to the different clusters. For the class, the labels over the training data can be found in the labels_ attribute._
+
+| Method name                           | Parameters                                                | Scalability                                          | Usecase                                                          | Geometry (metric used)                        |
+|---------------------------------------|-----------------------------------------------------------|-----------------------------------------------------|------------------------------------------------------------------|-----------------------------------------------|
+| K-Means                   | number of clusters                                        | Very large `n_samples`, medium `n_clusters` with MiniBatch code | General-purpose, even cluster size, flat geometry, not too many clusters, inductive | Distances between points                      |
+| Affinity propagation| damping, sample preference                                 | Not scalable with `n_samples`                        | Many clusters, uneven cluster size, non-flat geometry, inductive  | Graph distance (e.g. nearest-neighbor graph)  |
+| Mean-shift            | bandwidth                                                 | Not scalable with `n_samples`                        | Many clusters, uneven cluster size, non-flat geometry, inductive  | Distances between points                      |
+| Spectral clustering | number of clusters                                        | Medium `n_samples`, small `n_clusters`               | Few clusters, even cluster size, non-flat geometry, transductive  | Graph distance (e.g. nearest-neighbor graph)  |
+| Ward hierarchical clustering | number of clusters or distance threshold                  | Large `n_samples` and `n_clusters`                   | Many clusters, possibly connectivity constraints, transductive   | Distances between points                      |
+| Agglomerative clustering | number of clusters or distance threshold, linkage type, distance | Large `n_samples` and `n_clusters`                   | Many clusters, possibly connectivity constraints, non-Euclidean distances, transductive | Any pairwise distance                        |
+| DBSCAN                     | neighborhood size                                         | Very large `n_samples`, medium `n_clusters`         | Non-flat geometry, uneven cluster sizes, outlier removal, transductive | Distances between nearest points              |
+| HDBSCAN                | minimum cluster membership, minimum point neighbors       | Large `n_samples`, medium `n_clusters`              | Non-flat geometry, uneven cluster sizes, outlier removal, transductive, hierarchical, variable cluster density | Distances between nearest points              |
+| OPTICS                     | minimum cluster membership                                | Very large `n_samples`, large `n_clusters`          | Non-flat geometry, uneven cluster sizes, variable cluster density, outlier removal, transductive | Distances between points                      |
+| Gaussian mixtures        | many                                                      | Not scalable                                         | Flat geometry, good for density estimation, inductive           | Mahalanobis distances to centers              |
+| BIRCH                       | branching factor, threshold, optional global clusterer.   | Large `n_clusters` and `n_samples`                  | Large dataset, outlier removal, data reduction, inductive         | Euclidean distance between points             |
+| Bisecting K-Means | number of clusters                                        | Very large `n_samples`, medium `n_clusters`         | General-purpose, even cluster size, flat geometry, no empty clusters, inductive, hierarchical | Distances between points                      |
+
+### K-means
+
+_The KMeans algorithm clusters data by trying to separate samples in n groups of equal variance, minimizing a criterion known as the inertia or within-cluster sum-of-squares (see below). This algorithm requires the number of clusters to be specified. It scales well to large numbers of samples and has been used across a large range of application areas in many different fields._
+
+The K-means algorithm aims to choose centroids that minimise the inertia, or within-cluster sum-of-squares criterion
+
+Inertia can be recognized as a measure of how internally coherent clusters are. It suffers from various drawbacks:
+
+* it makes the assumption that clusters are convex and isotropic, which is not always the case. It responds poorly to elongated clusters, or manifolds with irregular shapes
+* it's not a normalised metric; lower is better and 0 is optimal. In high-dim spaces, Euclidean distances tend to become inflated, so running a dim reduction alg such as PCA prior to k-means can alleviate this problem and speed up computations
+
+_Given enough time, K-means will always converge, however this may be to a local minimum. This is highly dependent on the initialization of the centroids. As a result, the computation is often done several times, with different initializations of the centroids. One method to help address this issue is the k-means++ initialization scheme, which has been implemented in scikit-learn. This initializes the centroids to be (generally) distant from each other, leading to probably better results than random initialization, as shown in the reference._
+
+There is also a mini-batch kmeans, which converges faster than KMeans, but the quality of the results is reduced. In practice this difference in quality can be quite small, as shown in the example and cited reference.
+
+![image](https://github.com/user-attachments/assets/09983f20-a967-4552-9900-174c9ef24539)
+
+### Affinity Propagation
+
+_AffinityPropagation creates clusters by sending messages between pairs of samples until convergence. A dataset is then described using a small number of exemplars, which are identified as those most representative of other samples. The messages sent between pairs represent the suitability for one sample to be the exemplar of the other, which is updated in response to the values from other pairs. This updating happens iteratively until convergence, at which point the final exemplars are chosen, and hence the final clustering is given._
+
+![image](https://github.com/user-attachments/assets/aea86062-1e74-4f63-aa70-f81ba84f3291)
+
+_Affinity Propagation can be interesting as it chooses the number of clusters based on the data provided. For this purpose, the two important parameters are the preference, which controls how many exemplars are used, and the damping factor which damps the responsibility and availability messages to avoid numerical oscillations when updating these messages._
+
+The main disadvantage is the algorithm's complexity, as it has a time complexity of O(N^2 x T), where N is # of samples and T is # of iterations until convergence; and the memory complexity is O(N^2) if a dense similarity matrix is used, but reducible if a sparse one is used. Therefore, AP is most appropriate for small to medium datasets.
+
+[This example](https://scikit-learn.org/stable/auto_examples/applications/plot_stock_market.html#sphx-glr-auto-examples-applications-plot-stock-market-py) shows AP applied to group stocks that behave similarly according to their stock price movement:
+
+![image](https://github.com/user-attachments/assets/d3f29dd0-036b-419f-b2a8-4079b9b7556c)
+
+### Mean Shift
+
+_MeanShift clustering aims to discover blobs in a smooth density of samples. It is a centroid based algorithm, which works by updating candidates for centroids to be the mean of the points within a given region. These candidates are then filtered in a post-processing stage to eliminate near-duplicates to form the final set of centroids._
+
+It is not highly scalable, as it requires multiple nearest neighbor searches during its execution. It is guaranteed to converge, however it will stop iterating when the change in centroids is small.
+
+### Spectral clustering
+
+_SpectralClustering performs a low-dimension embedding of the affinity matrix between samples, followed by clustering, e.g., by KMeans, of the components of the eigenvectors in the low dimensional space. It is especially computationally efficient if the affinity matrix is sparse and the amg solver is used for the eigenvalue problem.It works well for a small number of clusters, but is not advised for many clusters._
+
+### Hierarchical clustering
+
+_Hierarchical clustering is a general family of clustering algorithms that build nested clusters by merging or splitting them successively. This hierarchy of clusters is represented as a tree (or dendrogram). The root of the tree is the unique cluster that gathers all the samples, the leaves being the clusters with only one sample_
+
+The `AgglomerativeClustering` object performs a hierarchical clustering using a bottom up approach: each observation starts in its own cluster, and clusters are successively merged together. The linkage criteria determines the metric used for the merge strategy:
+
+* 'ward' minimises the sum of squared differences within all clusters; similar to the k-means objective function but tackled with an agglomerative hirarchical approach
+* 'maximum' or 'complete linkage' minimises the max distance b/e observations of pairs of clusters
+* 'average linkage' minimises the avg of the distances b/e all observations of pairs of clusters
+* 'single linkage' minimises the distance b/e the closes observations of pairs of clusters
+
+![image](https://github.com/user-attachments/assets/e03c4014-5222-4107-9e30-fcbecdd9612f)
+
+An interesting aspect of AgglomerativeClustering is that connectivity constraints can be added to this algorithm through a connectivity matrix that defines for each sample the neighboring samples following a given structure of the data. These constraint are useful to impose a certain local structure, but they also make the algorithm faster, especially when the number of the samples is high.
+
+### Bisecting K-means
+
+This is a variant of K-Means using divisive hierarchical clustering, where clusters are iteratively split into two until reaching the target number of clusters. It's more efficient than standard K-Means, especially with a high cluster count, as it operates on smaller data subsets per split. While it lacks the "k-means++" initialization, it achieves comparable results with lower computational costs.
+
+This method avoids empty clusters and provides two strategies for splitting: selecting the cluster with the most points (faster and yields balanced clusters) or the cluster with the highest inertia (higher accuracy but costlier).
+
+[This example](https://scikit-learn.org/stable/auto_examples/cluster/plot_bisect_kmeans.html#sphx-glr-auto-examples-cluster-plot-bisect-kmeans-py) shows a comparison between bisecting k-means and k-means. While the regular K-Means algorithm tends to create non-related clusters, clusters from Bisecting K-Means are well ordered and create quite a visible hierarchy.
+
+![image](https://github.com/user-attachments/assets/2a09f060-a249-4f9d-967a-8ccc945c67f3)
+
+### DBSCAN
+
+_The DBSCAN algorithm views clusters as areas of high density separated by areas of low density. Due to this rather generic view, clusters found by DBSCAN can be any shape, as opposed to k-means which assumes that clusters are convex shaped. The central component to the DBSCAN is the concept of core samples, which are samples that are in areas of high density. A cluster is therefore a set of core samples, each close to each other (measured by some distance measure) and a set of non-core samples that are close to a core sample (but are not themselves core samples). There are two parameters to the algorithm, min_samples and eps, which define formally what we mean when we say dense. Higher min_samples or lower eps indicate higher density necessary to form a cluster._
+
+_Any core sample is part of a cluster, by definition. Any sample that is not a core sample, and is at least eps in distance from any core sample, is considered an outlier by the algorithm.
+While the parameter min_samples primarily controls how tolerant the algorithm is towards noise (on noisy and large data sets it may be desirable to increase this parameter), the parameter eps is crucial to choose appropriately for the data set and distance function and usually cannot be left at the default value. It controls the local neighborhood of the points. When chosen too small, most data will not be clustered at all (and labeled as -1 for “noise”). When chosen too large, it causes close clusters to be merged into one cluster, and eventually the entire data set to be returned as a single cluster. Some heuristics for choosing this parameter have been discussed in the literature, for example based on a knee in the nearest neighbor distances plot (as discussed in the references below)._
+
+![image](https://github.com/user-attachments/assets/b365f2eb-839a-4db4-804a-d4d314ba4a4b)
+
+Big circles - core samples
+
+Smaller curcles - non-core samples
+
+Black points - outliers
+
+- DBSCAN is deterministic, producing the same clusters if data is provided in the same order
+- Results can vary with different data orders:
+  - core samples always form the same clusters, but cluster labels depend on the encounter order
+  - non-core samples may be assigned differently if they are close to core samples in different clusters
+- a non-core sample near two core samples from different clusters joins the cluster encountered first in data order
+- uses ball trees and kd-trees for efficient neighborhood searches, avoiding full distance matrix calculations
+- supports custom metrics for distance calculations
+
+### HDBSCAN
+
+DBSCAN assumes that the clustering criterion (i.e. density requirement) is globally homogeneous so it might struggle in capturing clusters with different densities. HDBSCAN alleviates this assumption and explores all possible density scales by building an alternative representation of the clustering problem. 
+
+Here's how it works:
+
+* it calculates a core distance for each point based on a set number of nearest neighbours (`min_samples`)
+* points are connected based on their mutual reachability distance (distance with adjustments for density). This graph helps find clusters by linking close points while considering density variations
+* it builds a hierarchical cluster structure by iteratively removing high-distance edges, revealing clusters at various densities
+* unlike DBSCAN, HDBSCAN doesn’t require a fixed density threshold (epsilon). It only requires min_samples and optionally min_cluster_size, making it adaptable to clusters of different densities
+
+### OPTICS
+
+The OPTICS (Ordering Points To Identify the Clustering Structure) algorithm is similar to DBSCAN but generalises it by allowing a range of `eps` values rather than a fixed one. 
+
+Unlike DBSCAN, OPTICS builds a reachability graph that records each sample's reachability distance and position in cluster order. This reachability information enables flexible cluster detection at varying densities within the same dataset. By setting `max_eps` to infinity, OPTICS can perform DBSCAN-style clustering repeatedly for any `eps` value. The reachability plot, generated from the reachability distances, can be analyzed to extract clusters by identifying steep slopes, with `xi` controlling the slope sensitivity. Additional options include hierarchical clustering views via reachability-plot dendrograms, accessible through the `cluster_hierarchy_` parameter.
+
+**Comparison with DBSCAN**
+
+_The results from OPTICS cluster_optics_dbscan method and DBSCAN are very similar, but not always identical; specifically, labeling of periphery and noise points. This is in part because the first samples of each dense area processed by OPTICS have a large reachability value while being close to other points in their area, and will thus sometimes be marked as noise rather than periphery. This affects adjacent points when they are considered as candidates for being marked as either periphery or noise.
+Note that for any single value of eps, DBSCAN will tend to have a shorter run time than OPTICS; however, for repeated runs at varying eps values, a single run of OPTICS may require less cumulative runtime than DBSCAN. It is also important to note that OPTICS’ output is close to DBSCAN’s only if eps and max_eps are close._
+
+### BIRCH
+
+The BIRCH (Balanced Iterative Reducing and Clustering using Hierarchies) algorithm compresses data into a Clustering Feature Tree (CFT), where each node contains subclusters (CF Subclusters) with summary statistics like sample count, linear sum, squared sum, centroid, and squared norm. This structure enables efficient clustering without needing to store all data in memory. Key parameters include the *threshold* (maximum distance for adding points to subclusters) and branching factor* (limit on subclusters per node). BIRCH can act as a data reduction method, creating subclusters for further processing by a global clusterer, defined by the *n_clusters* parameter.
+
+### [Clustering performance evaluation](https://scikit-learn.org/stable/modules/clustering.html#clustering-performance-evaluation)
+
+* rand index
+* mutual information based scores
+* homogeneity, completeness and V-measure
+* fowlkes-mallows scores
+* silhouette coefficient
+* calinski-barabasz index
+* davies-bouldin index
+* contingency matrix
+* pair confusion matrix
+
+## [Biclustering](https://scikit-learn.org/stable/modules/biclustering.html)
+
+_Biclustering algorithms simultaneously cluster rows and columns of a data matrix. These clusters of rows and columns are known as biclusters. Each determines a submatrix of the original data matrix with some desired properties._
+
+Algorithms differ in how they define biclusters. Some of the common types include:
+
+* constant values, constant rows, or constant columns
+* unusually high or low values
+* submatrices with low variance
+* correlated rows or columns
+
+Algorithms also differ in how rows and columns may be assigned to biclusters, which leads to different bicluster structures. Block diagonal or checkerboard structures occur when rows and columns are divided into partitions.
+
+If each row and each column belongs to exactly one bicluster, then rearranging the rows and columns of the data matrix reveals the biclusters on the diagonal.
+
+### Spectral Co-Clustering
+
+_The SpectralCoclustering algorithm finds biclusters with values higher than those in the corresponding other rows and columns. Each row and each column belongs to exactly one bicluster, so rearranging the rows and columns to make partitions contiguous reveals these high values along the diagonal_
+
+[Here](https://scikit-learn.org/stable/auto_examples/bicluster/plot_spectral_coclustering.html#sphx-glr-auto-examples-bicluster-plot-spectral-coclustering-py) is an example:
+
+<img width="639" alt="image" src="https://github.com/user-attachments/assets/f4eccc50-1caf-4b3c-8efb-0f3a3ae0bff5">
+
+### Spectral Biclustering
+
+_The SpectralBiclustering algorithm assumes that the input data matrix has a hidden checkerboard structure. The rows and columns of a matrix with this structure may be partitioned so that the entries of any bicluster in the Cartesian product of row clusters and column clusters are approximately constant. For instance, if there are two row partitions and three column partitions, each row will belong to three biclusters, and each column will belong to two biclusters.
+The algorithm partitions the rows and columns of a matrix so that a corresponding blockwise-constant checkerboard matrix provides a good approximation to the original matrix._
+
+### Biclustering evaluation
+
+In sklearn, only `consensus_score` is available where the minimum is 0 and occurs when all pairs of biclusters are totally dissimilar. The maximum score, 1 and occurs when both sets are identical.
+
+---
+
+The above order is not the exact as in the docs as I skipped some pages because I already read them recently on stream.
+
+That is all for today!
+
+See you tomorrow :)

From 43999a1a164002c0fb849913fdb84758eb6568d7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 10 Nov 2024 21:33:41 +0900
Subject: [PATCH 492/638] Create 2024-11-10-day-314.md

---
 _posts/2024-11-10-day-314.md | 123 +++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 _posts/2024-11-10-day-314.md

diff --git a/_posts/2024-11-10-day-314.md b/_posts/2024-11-10-day-314.md
new file mode 100644
index 000000000..6250abbb5
--- /dev/null
+++ b/_posts/2024-11-10-day-314.md
@@ -0,0 +1,123 @@
+---
+layout: post
+title: (Day 314) Streamed + new video + more sklearn docs
+categories: [theory,traditional-machine-learning,statistics,applying-knowledge]
+---
+
+# Hello :) Today is Day 314!
+A quick summary of today:
+* covariance estimation
+* novelty and outlier detection
+* made a basic video on my notebook on multicollinearity
+* streamed
+
+## [Covariance estimation](https://scikit-learn.org/stable/modules/covariance.html)
+
+_Many statistical problems require the estimation of a population’s covariance matrix, which can be seen as an estimation of data set scatter plot shape. Most of the time, such an estimation has to be done on a sample whose properties (size, structure, homogeneity) have a large influence on the estimation’s quality. The sklearn.covariance package provides tools for accurately estimating a population’s covariance matrix under various settings.
+We assume that the observations are independent and identically distributed (i.i.d.)._
+
+### Empirical covariance
+
+The covar matrix is known to be well approximated by the classical maximum likelihoodd estimator (or empirical covar), provided the # of observations is large enough compared to # of features. The MLE of a sample is an asymptomatically unbiased estimator of the corresponding popoulation's covar matrix. 
+
+When using `empirical_covariance`'s fit method we need to be careful as the results depend on whether the data is centred. For this there is a given `assume_centered` param: if assume_centered=False, then the test set is supposed to have the same mean vector as the training set. If not, both should be centered by the user, and assume_centered=True should be used.
+
+### Shrunk covariance
+
+The shrunk covar approach is a transformation of the empirical covar matrix that mitigates issues with inversion and eigenvalue estimation, commonly encountered with the Maximum Likelihood Estimator (MLE). This transformation, often applied using a shrinkage coefficient in sklearn, reduces the ratio between the smallest and largest eigenvalues, effectively regularizing the covariance matrix by offsetting eigenvalues to manage the bias-variance trade-off.
+
+**Basic shrinkage**
+
+A convex transformation applies a penalty to MLE, making the covariance estimation more stable.
+
+**Ledoit-Wolf Shrinkage**
+
+This method calculates an optimal shrinkage coefficient to minimize the Mean Squared Error (MSE) of covariance estimates. Notably, when sample size greatly exceeds features, the Ledoit-Wolf shrinkage approaches 1 if the population covariance matrix is isotropic, yielding a multiple of the identity matrix.
+
+**Oracle Approximating Shrinkage (OAS)**
+
+For Gaussian-distributed data, the OAS method achieves lower MSE than Ledoit-Wolf’s estimator. 
+
+### Sparse inverse covariance
+
+_The matrix inverse of the covariance matrix, often called the precision matrix, is proportional to the partial correlation matrix. It gives the partial independence relationship. In other words, if two features are independent conditionally on the others, the corresponding coefficient in the precision matrix will be zero. This is why it makes sense to estimate a sparse precision matrix: the estimation of the covariance matrix is better conditioned by learning independence relations from the data. This is known as covariance selection._
+
+In cases when n_samples is similar or smaller than n_features, this tends to work better than the shrunk covar estimators. However, in the opposite situation, or for very correlated data, they can be numerically unstable. In addition, unlike shrinkage estimators, sparse estimators are able to recover off-diagonal structure.
+
+A sparse inverse covar (`GraphicalLassoCV`) is used in [this case](https://scikit-learn.org/stable/auto_examples/applications/plot_stock_market.html#sphx-glr-auto-examples-applications-plot-stock-market-py) where the similarities between stocks are calculated and then this similarity matrix is passed onto `affinity_propagation` (a clustering algorithm)
+
+### Robust covariance estimation
+
+The two main methods above are sensitive to outliers. In such cases we should use a robust covar estimator. In addition, such algorithms can be used for outlier detection and discard/downweight some observations. 
+
+## [Novelty and Outlier Detection](https://scikit-learn.org/stable/modules/outlier_detection.html)
+
+### Overview of outlier detection methods
+
+![image](https://github.com/user-attachments/assets/e50c2c71-adaa-4ae3-b8fe-800e286dd315)
+
+Local Outlier Factor (LOF) does not show a decision boundary in black as it has no predict method to be applied on new data when it is used for outlier detection.
+
+`ensemble.IsolationForest` and `neighbors.LocalOutlierFactor` perform reasonably well on the data sets considered here. The `svm.OneClassSVM` is known to be sensitive to outliers and thus does not perform very well for outlier detection. That being said, outlier detection in high-dimension, or without any assumptions on the distribution of the inlying data is very challenging. `svm.OneClassSVM` may still be used with outlier detection but requires fine-tuning of its hyperparameter nu to handle outliers and prevent overfitting. `linear_model.SGDOneClassSVM` provides an implementation of a linear One-Class SVM with a linear complexity in the number of samples. This implementation is here used with a kernel approximation technique to obtain results similar to `svm.OneClassSVM` which uses a Gaussian kernel by default. Finally, `covariance.EllipticEnvelope` assumes the data is Gaussian and learns an ellipse.
+
+### Novelty detection
+
+_Consider a data set of n observations from the same distribution described by p features. Consider now that we add one more observation to that data set. Is the new observation so different from the others that we can doubt it is regular? (i.e. does it come from the same distribution?) Or on the contrary, is it so similar to the other that we cannot distinguish it from the original observations? This is the question addressed by the novelty detection tools and methods._
+
+The One-Class SVM is used for this purpose and requires the choice of a kernel and a scalar parameter to define a frontier. The RBF kernel is usually chosen although there exists no exact formula or algorithm to set its bandwidth parameter. This is the default in the scikit-learn implementation. The nu parameter, also known as the margin of the One-Class SVM, corresponds to the probability of finding a new, but regular, observation outside the frontier
+
+![image](https://github.com/user-attachments/assets/47df468f-17fc-4357-a9fc-cd11c19e10fc)
+
+**Scaling up**
+
+An online linear version of the One-Class SVM is implemented by sklearn which scales linearly with # of samples and can be used with a kernel approximation to approximate the solution of a kernelized svm.OneClassSVM whose complexity is at best quadratic in # of samples.
+
+### Outlier detection
+
+_Outlier detection is similar to novelty detection in the sense that the goal is to separate a core of regular observations from some polluting ones, called outliers. Yet, in the case of outlier detection, we don’t have a clean data set representing the population of regular observations that can be used to train any tool._
+
+**Fitting an elliptic envelope**
+
+_One common way of performing outlier detection is to assume that the regular data come from a known distribution (e.g. data are Gaussian distributed). From this assumption, we generally try to define the “shape” of the data, and can define outlying observations as observations which stand far enough from the fit shape._
+
+![image](https://github.com/user-attachments/assets/2d7382db-f730-4584-b2d6-ce721b197af6)
+
+**Isolation forest**
+
+Isolation Forest is a method for detecting outliers in high-dimensional data by isolating observations. It works by building a forest of randomly partitioned trees, where each sample is 'isolated' by splitting features at random values. The path length from the root to a leaf node in each tree represents how isolated a sample is. Anomalies tend to have shorter path lengths on average, as they are easier to isolate. The implementation uses an ensemble of ExtraTreeRegressor trees, with a maximum tree depth proportional to the logarithm of the sample size
+
+![image](https://github.com/user-attachments/assets/a0843a2b-1a64-4154-9686-c4f3e61dc3fd)
+
+**Local Outlier Factor**
+
+This is another efficient method for high dim datasets. It computes a score (called local outlier factor) reflecting the degree of abnormality of the observations. It measures the local density deviation of a given data point with respect to its neighbors. The idea is to detect the samples that have a substantially lower density than their neighbors.
+
+_In practice the local density is obtained from the k-nearest neighbors. The LOF score of an observation is equal to the ratio of the average local density of its k-nearest neighbors, and its own local density: a normal instance is expected to have a local density similar to that of its neighbors, while abnormal data are expected to have much smaller local density._
+
+_The strength of the LOF algorithm is that it takes both local and global properties of datasets into consideration: it can perform well even in datasets where abnormal samples have different underlying densities. The question is not, how isolated the sample is, but how isolated it is with respect to the surrounding neighborhood._
+
+![image](https://github.com/user-attachments/assets/980d948e-69f3-41cb-a75d-66fd9ec94f1c)
+
+### Novelty detection with LOF
+
+To use that, we need to set `LocalOutlierFactor(novelty=True)`. When novelty is set to true we must only use predict, decision_function and score_samples on new unseen data and not on the training samples as this would lead to wrong results.
+
+![image](https://github.com/user-attachments/assets/b473936b-2dab-4d95-85e9-43bb0c20039a)
+
+## [The full story behind multicollinearity](https://youtu.be/4bMSygDuWvI)
+
+The original notebook and post were made on [day 121](https://ivanstudyblog.github.io/day-121/). But given that I am doing youtube streams these days and have a few viewers, I decided to make a video of the notebook. Nothing special it was pretty much me reciting/reading my notebook, but hoping it reaches a larger audience and get feedback in case something is incorrect.
+
+## [Streamed](https://youtu.be/7OmDiEcqHpw)
+
+Covered more of the sklearn docs' user guide.
+
+* density estimation
+* cross validation
+* hyperparam tuning
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 7e2dd2f701f07b9795e2d760d7cd79300c8b404c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 11 Nov 2024 23:55:51 +0900
Subject: [PATCH 493/638] Create 2024-11-11-day-315.md

---
 _posts/2024-11-11-day-315.md | 362 +++++++++++++++++++++++++++++++++++
 1 file changed, 362 insertions(+)
 create mode 100644 _posts/2024-11-11-day-315.md

diff --git a/_posts/2024-11-11-day-315.md b/_posts/2024-11-11-day-315.md
new file mode 100644
index 000000000..eb927068c
--- /dev/null
+++ b/_posts/2024-11-11-day-315.md
@@ -0,0 +1,362 @@
+---
+layout: post
+title: (Day 315) Finally started reading Chip Huyen's Designing ML systems
+categories: [theory,traditional-machine-learning,applying-knowledge,mlops,nlp,data-eng]
+---
+
+# Hello :) Today is Day 315!
+A quick summary of today:
+* [started reading one of the most talked about books in ML](https://learning.oreilly.com/library/view/designing-machine-learning/9781098107956/)
+* basic deployment for the company reviewer project
+
+## Chapter 1 - Overview of ML systems
+
+When people hear ML systems they often think only of the underlying algorithms. However, the algorithm is just a small piece of an ML system in production. A complete ML system also includes the business requirements that drive the project, the interface for user and developer interactions, the data stack, and the processes for developing, monitoring, and updating models. Additionally, it involves the infrastructure that enables the delivery of these components. Below the components of an ML system and indicates the chapters in this book are shown
+
+![image](https://github.com/user-attachments/assets/3cd09d0a-29e1-4d0c-9b8f-4cea34a70dd2)
+
+### When to use ML
+
+_Machine learning is an approach to learn complex patterns from existing data and use these patterns to make predictions on unseen data._
+
+Survey results on the use of ML:
+
+![image](https://github.com/user-attachments/assets/55ed879c-6365-4cf6-93db-72ed71cc9154)
+
+### Understanding ML systems
+
+**ML in research vs production**
+
+|                    | Research                                         | Production                                 |
+|--------------------|--------------------------------------------------|--------------------------------------------|
+| **Requirements**   | State-of-the-art model performance on benchmark datasets | Different stakeholders have different requirements |
+| **Computational priority** | Fast training, high throughput                    | Fast inference, low latency                |
+| **Data**          | Static                                           | Constantly shifting                        |
+| **Fairness**      | Often not a focus                                | Must be considered                         |
+| **Interpretability** | Often not a focus                                | Must be considered                         |
+
+**ML systems vs Traditional software**
+
+* in SWE there's an underlying assumption that code and data are separate and engineers want to keep things modular and separate
+* contrary, ML systems are part code, part data, and part artifacts created from the two
+* in SWE, we focus on testing and versioning the code
+* with ML, we have to test and version data as well (and that's hard)
+* ML models are getting bigger and bigger, requiring more and more space
+* ML models on edge devices is another challenge
+* prod monitoring and debugging is also nontrivial
+
+## Chapter 2 - Introduction to ML systems design
+
+_ML systems design takes a system approach to MLOps, which means that we’ll consider an ML system holistically to ensure that all the components—the business requirements, the data stack, infrastructure, deployment, monitoring, etc.—and their stakeholders can work together to satisfy the specified objectives and requirements._
+
+Before an ML system is developed we must understand why it is needed. If it's built for a business, then there must be some business objectives behind which will need to be translated into ML objectives. 
+
+Once these objectives are defined and everyone is on board, we'll need to set some requirements to guide the development of the system. The book considers 4 requirements:
+
+* reliability
+* scalability
+* maintainability
+* adaptability
+
+### Business and ML objectives
+
+Data scientists might be interested to improve the performance of their model - i.e. increase accuracy from 94% ot 94.2% and they might spend a lot of resources to achieve that. 
+
+However, most companies don't care about this 0.2% unless it moves some business metric. We need to pay more attention to the buisness objectives. The ultimate goal of any project in a business is to increase profits - either directly or indirectly. So ~ for an ML project to succeed within a business, it's crucial to tie the performance of an ML system to the overall business performance. 
+
+When evaluating ML solutions through the business lens, it’s important to be realistic about the expected returns. Due to all the hype surrounding ML, generated both by the media and by practitioners with a vested interest in ML adoption, some companies might have the notion that ML can magically transform their businesses overnight.
+
+Returns on investment in ML depend a lot on the maturity stage of adoption. The longer we've adopted ML, the more efficient our pipes will run, the faster the dev cycle will be, the less engineering time we'll need, and the lower the cloud bills will be.
+
+The belos is based on a survey on how long it takes for a company to bring a model to prod. It was conclded that it is proportional to how long it has used ML
+
+![image](https://github.com/user-attachments/assets/d1271cc0-b65a-43ff-b85d-07cc2f982fa2)
+
+### Requirements for ML systems
+
+_We can’t say that we’ve successfully built an ML system without knowing what requirements the system has to satisfy. The specified requirements for an ML system vary from use case to use case. However, most systems should have these four characteristics: reliability, scalability, maintainability, and adaptability._
+
+**Reliability**
+
+The system should continue to perform the correct function at the desired level of performance even if problems arise (hardware/software faults/human error)
+
+'Correctness' might be difficult to determine because we cannot know if a prediction is wrong if we don't have the gorund truth. With traditional software systems we might get a warning/system crash/runtime error/404, but ML systems can fail silently and end users might not even know it. This topic is explored more in chapter 8.
+
+**Scalability**
+
+An ML system can grow in complexity, in traffic volume, in ML model count. Whichever way it grows, we should be ready and have reasonable ways of dealing with that growth. Again, this is discussed more in detail later in the book
+
+**Maintainability**
+
+There are many people who will work on an ML systems, so its important to strucutre the workloads set up up the infrastructure in such a way that different contributors can work using tools they are comfortable with. 
+
+* code should be documented
+* code, data, and artifacts should be versioned
+* models should be sufficiently reproducible
+* when a problem occurs, different contributors should be able to work together to identify the problem and implement a solution without finger-pointing
+
+**Adaptability**
+
+To adapt to shifting data distributions and business requirements, the system should have some capacity for both discovering aspects for performance improvement and allowing updates without service interruption.
+
+### Iterative process
+
+_Developing an ML system is an iterative and, in most cases, never-ending process. Once a system is put into production, it’ll need to be continually monitored and updated._
+
+Below shows an oversimplified representation of what the iterative process for developing ML systems in prod looks like from the perspective of a DS/MLE
+
+![image](https://github.com/user-attachments/assets/3a236d04-5d52-4b9f-899c-ef800b5150cf)
+
+A brief of each step
+
+**Step 1: Project scoping**
+
+A project starts with scoping the project, laying out goals, objectives, and constraints. Stakeholders should be identified and involved. Resources should be estimated and allocated.
+
+**Step 2: Data engineering**
+
+A vast majority of ML models today learn from data, so developing ML models starts with engineering data. Garbage-in-garbage-out as the saying goes.
+
+**Step 3: ML model development**
+
+With the initial set of training data, we’ll need to extract features and develop initial models leveraging these features. This is the stage that requires the most ML knowledge and is most often covered in ML courses.
+
+**Step 4: Deployment**
+
+After a model is developed, it needs to be made accessible to users.
+
+**Step 5: Monitoring and continual learning**
+
+Once in production, models need to be monitored for performance decay and maintained to be adaptive to changing environments and changing requirements.
+
+**Step 6: Business analysis**
+
+Model performance needs to be evaluated against business goals and analysed to generate business insights. These insights can then be used to eliminate unproductive projects or scope out new projects. This step is closely related to the first step.
+
+### Framing ML problems
+
+If a stakeholder one day sees that our company's customer support is slower than a competitor's ML system, they will come to us (as we are an MLE) and ask us to use ML to improve our customer support as well. Slow customer support is a problem, but it's not an ML problem. An ML problem is defined by inputs, outputs, and the objective function that guides the learning process - and these are not so obvious to our stakeholder, so it's our job, as an MLE, to figure those things out. Upon investigation, we discover that the slow CS comes form the fact that the bottleneck in responding to customer requests lies in routing customer requests to the right department among 4. We can alleviate this bottleneck by developing an ML model to predict to which department a request should go to - this just became a multiclass classification problem. 
+
+* the input is the customer request
+* the output is the department the request should go to
+* the objective function is to minimize the difference between the predicted department and the actual department
+
+There might be multiple ways to frame a problem, and changing the way we frame the problem might make the problem significantly easier or harder. 
+
+### Mind vs Data
+
+_Progress in the last decade shows that the success of an ML system depends largely on the data it was trained on. Instead of focusing on improving ML algorithms, most companies focus on managing and improving their data._
+
+_In theory, you can both pursue architectural designs and leverage large data and computation, but spending time on one often takes time away from another._
+
+_Many people in ML today are in the data-over-mind camp. Professor Richard Sutton, a professor of computing science at the University of Alberta and a distinguished research scientist at DeepMind, wrote a great blog post in which he claimed that researchers who chose to pursue intelligent designs over methods that leverage computation will eventually learn a bitter lesson: “The biggest lesson that can be read from 70 years of AI research is that general methods that leverage computation are ultimately the most effective, and by a large margin.… Seeking an improvement that makes a difference in the shorter term, researchers seek to leverage their human knowledge of the domain, but the only thing that matters in the long run is the leveraging of computation.”_
+
+_When asked how Google Search was doing so well, Peter Norvig, Google’s director of search quality, emphasized the importance of having a large amount of data over intelligent algorithms in their success: “We don’t have better algorithms. We just have more data.”_
+
+The debate isn’t about whether finite data is necessary, but whether it’s sufficient. The term finite here is important, because if we had infinite data, it might be possible for us to look up the answer. Having a lot of data is different from having infinite data.
+
+The data science hierarchy of needs:
+
+![image](https://github.com/user-attachments/assets/f2c15b4e-330f-4170-8448-870475e52675)
+
+Regardless of which camp will prove to be right eventually, no one can deny that data is essential, for now. Both the research and industry trends in the recent decades show the success of ML relies more and more on the quality and quantity of data.
+
+## Chapter 3 - Data engineering fundamentals
+
+### Data sources
+
+* user input data
+* system-generated data
+* internal databases
+* 3rd-party data
+
+### Data formats
+
+* json
+* csv
+* parquet
+* avro
+* protobuf
+* pickle
+
+### Data models
+
+* relational model
+* noSQL
+  * document model
+  * graph model
+ 
+* structured vs. unstructured data
+
+| Structured data                        | Unstructured data                                                                                      |
+|----------------------------------------|--------------------------------------------------------------------------------------------------------|
+| Schema clearly defined                 | Data doesn’t have to follow a schema                                                                   |
+| Easy to search and analyze             | Fast arrival                                                                                           |
+| Can only handle data with a specific schema | Can handle data from any source                                                                          |
+| Schema changes will cause a lot of troubles | No need to worry about schema changes (yet), as the worry is shifted to the downstream applications that use this data |
+| Stored in data warehouses              | Stored in data lakes                                                                                   |
+
+### Data storage engines and processing
+
+**Transactional databases**
+
+Designed to process online transactions and satisfy the low latency, high availability requirements. 
+
+ACID:
+
+* atomicity
+
+To guarantee that all the steps in a transaction are completed successfully as a group. If any step in the transaction fails, all other steps must fail also. For example, if a user’s payment fails, you don’t want to still assign a driver to that user.
+
+* consistency
+
+To guarantee that all the transactions coming through must follow predefined rules. For example, a transaction must be made by a valid user.
+
+* isolation
+
+To guarantee that two transactions happen at the same time as if they were isolated. Two users accessing the same data won’t change it at the same time. For example, you don’t want two users to book the same driver at the same time.
+
+* durability
+
+To guarantee that once a transaction has been committed, it will remain committed even in the case of a system failure. For example, after you’ve ordered a ride and your phone dies, you still want your ride to come.
+
+_However, transactional databases don’t necessarily need to be ACID, and some developers find ACID to be too restrictive. According to Martin Kleppmann, “systems that do not meet the ACID criteria are sometimes called BASE, which stands for Basically Available, Soft state, and Eventual consistency. This is even more vague than the definition of ACID.”_
+
+OLAP vs OLTP
+
+**ETL**
+
+![image](https://github.com/user-attachments/assets/c8c2a9a7-3ea0-45cc-b9ef-99bb56dadc60)
+
+### Modes of Dataflow
+
+Most times, in production there are multiple processes going on, and the question becomes how do we pass data between different processes that don't share memory?
+
+When data is passed from one process to another, we say the data flows from one process to another, which gives us a dataflow. There are 3 main modes:
+
+* data passing through databases
+* data passing through services using requests such as the requests provided by REST and RPC APIs (i.e. POST/GET requests)
+* data passing through a real-time transport like Apache Kafka and Amazon Kinesis
+
+## Chapter 4 - Training data
+
+_Data is messy, complex, unpredictable, and potentially treacherous. If not handled properly, it can easily sink your entire ML operation. But this is precisely the reason why data scientists and ML engineers should learn how to handle data well, saving us time and headache down the road._
+
+### Sampling
+
+**Nonprobability sampling**
+
+As the name suggests, this is when the selection of data isn't based on any probability criteria. Here are some of the criteria for nonprob sampling:
+
+* convenience sampling: samples of data are selected based on their availability; this method is popular because its convenient
+* snowball sampling: future samples are selected based on existing samples; for instance, to scrape legitimate twitter accounts without having access to twitter dbs, we start with a small # of accounts, then scrape all the accounts they follow, and so on
+* judgement sampling: experts decide what samples to include
+* quota sampling: samples are selected based on quotas for certain slices of data without any randomisation; 
+
+The samples selected by nonprobability criteria are not representative of the real-world data and therefore are riddled with selection biases and because of that it's a bad idea to use them for ML. In most cases, data for ML models is still driven by convenience. 
+
+Nonprobability sampling can be a quick and easy way to gather initial data to get a project running, but for reliable models, we might need to use probability-based sampling methods
+
+**Simple random sampling**
+
+In the simplest form, we can sample data after giving each sample a uniform chance of being picked. The advantage of this method is that it's easy to implement, however, rare categories of the data might not appear in the selection
+
+**Stratified sampling**
+
+One drawback of this sampling method is that it isn’t always possible, such as when it’s impossible to divide all samples into groups. This is especially challenging when one sample might belong to multiple groups, as in the case of multilabel tasks. For instance, a sample can be both class A and class B.
+
+**Weighted sampling**
+
+This method allows to leverage domain expertise. For example, if we know that a certain subpopulation of data, such as more recent data, is more valuable to the model and want it to have a higher chance of being selected, we can give it a higher weight.
+
+**Reservoir sampling**
+
+This is especially useful when dealing with streaming data. It is useful when the dataset size is unknown or too large to fit in memory. 
+
+The algorithm ensures that every element in the stream has an equal probability of being selected by maintaining a fixed-size reservoir of k elements. The steps are:
+
+1. fill the reservoir with the first k elements
+2. for each new element n, generate a random integer i between 1 and n
+3. if i is within the range of the reservoir (1 to k), replace the i-th element in the reservoir with the n-th element; otherwise, ignore the new element
+
+This method guarantees that each element has an equal chance of being included in the sample, even if the algorithm stops partway through the data stream.
+
+**Importance sampling**
+
+_Importance sampling is one of the most important sampling methods, not just in ML. It allows us to sample from a distribution when we only have access to another distribution._
+
+Imagine we have to sample x from a distribution P(x), but P(x) is really expensive, slow, or infeasible to sample from. However, there is a distribution Q(x) that is a lot easier to sample from, so we sample x from Q(x) instead and weigh this sample by P(x)/Q(x). Q(x) is called the proposal distribution or the importance distribution. Q(x) can be any distribution as long as Q(x) > 0 whenever P(x) ≠ 0
+
+### Labeling
+
+**Hand labels**
+
+* expensive, especially if subject matter expertise is required
+* poses a threat to data privacy
+* slow
+
+**Natural labels**
+
+* canonical example of tasks with natural labels is recommender systems
+
+**How companies obtain labels**
+
+![image](https://github.com/user-attachments/assets/87d59ae6-c786-4ee3-8e94-0568c0defdb5)
+
+**Handling the lack of labels**
+
+| Method             | How                                               | Ground truths required?                                                                                 |
+|--------------------|---------------------------------------------------|---------------------------------------------------------------------------------------------------------|
+| Weak supervision   | Leverages (often noisy) heuristics to generate labels | No, but a small number of labels are recommended to guide the development of heuristics                |
+| Semi-supervision   | Leverages structural assumptions to generate labels  | Yes, a small number of initial labels as seeds to generate more labels                                 |
+| Transfer learning  | Leverages models pretrained on another task for your new task | No for zero-shot learning. Yes for fine-tuning, though the number of ground truths required is often much smaller than what would be needed if you train the model from scratch |
+| Active learning    | Labels data samples that are most useful to your model | Yes                                                                                                    |
+
+
+**Programmatic labeling**
+
+| Hand labeling         | Programmatic labeling                                      |
+|-----------------------|------------------------------------------------------------|
+| **Expensive**: Especially when subject matter expertise required | **Cost saving**: Expertise can be versioned, shared, and reused across an organization |
+| **Lack of privacy**: Need to ship data to human annotators | **Privacy**: Create LFs using a cleared data subsample and then apply LFs to other data without looking at individual samples |
+| **Slow**: Time required scales linearly with number of labels needed | **Fast**: Easily scale from 1K to 1M samples |
+| **Nonadaptive**: Every change requires relabeling the data | **Adaptive**: When changes happen, just reapply LFs! |
+
+### Class imbalance
+
+* there’s insufficient signal for the model to learn to detect the minority classes
+* it's easier for the model to get stuck in a nonoptimal solution by exploiting a simple heuristic instead of learning anything useful about the underlying pattern of the data
+* leads to asymmetric costs of error
+* can be caused by biases during the sampling process
+* another cause can be due to labeling errors
+
+Need to make sure to use the appropriate evaluation metrics in the presence of imbalanced data.
+
+We can use resampling methods:
+
+![image](https://github.com/user-attachments/assets/8777b675-4184-484b-93c9-bfbb6d685a21)
+
+Algorithm-level methods address class imbalance by adjusting the learning algorithm rather than modifying the training data. These methods often alter the loss function to make the model more sensitive to minority classes.
+
+* cost-sensitive learning: this approach adjusts the individual loss by applying a cost matrix, which specifies different misclassification costs for each class. The cost matrix enables prioritizing misclassification penalties based on class importance, though it requires manual definition for each task
+
+* class-balanced loss: to prevent models from favoring majority classes, class-balanced loss assigns weights inversely proportional to class frequencies, making rare classes more influential in the learning process. Advanced versions, like those based on effective sample numbers, further refine these weights.
+
+* focal loss: focal loss increases focus on harder-to-classify examples by boosting the loss weight for low-confidence predictions, encouraging the model to improve on challenging samples rather than easy ones
+
+### Data augmentation
+
+**Simple label-preserving transformations**
+
+* pertrubation
+* data synthesis
+
+## Company reviewer project
+
+Today my advisor sent me an email to share a link where someone can use the fine-tuned company reviewer chatbit. A reminder - this is a project where based 50 online reviews, an LLM writes a review for that company based on 12 dimensions (i.e. salary, work-life balance, etc.). The data is created by our client and I am using it to fine-tune various models. The best one so far is gpt4o-mini, so I helped my professor understand how to use it so that potentially the customer company can use it as well. It is not final, but for now we have a [streamlit UI](https://company-reviewer-hnu.streamlit.app/) option (which can only be used with an API key from the organisation but the UI itself is public), using the model through the openai playground, and also creating a GPT (which I have not done yet as I do not have GPT Plus). Later this week I will be going to the lab again to continue trying out different open-source model configurations and evaluate them.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f7a9a40d8de39cd43aaf9701b625b3dda3e7cc7d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 12 Nov 2024 22:44:15 +0900
Subject: [PATCH 494/638] Create 2024-11-12-day-316.md

---
 _posts/2024-11-12-day-316.md | 228 +++++++++++++++++++++++++++++++++++
 1 file changed, 228 insertions(+)
 create mode 100644 _posts/2024-11-12-day-316.md

diff --git a/_posts/2024-11-12-day-316.md b/_posts/2024-11-12-day-316.md
new file mode 100644
index 000000000..8365022ee
--- /dev/null
+++ b/_posts/2024-11-12-day-316.md
@@ -0,0 +1,228 @@
+---
+layout: post
+title: (Day 316) Reading more of the infamous Designing ML systems book
+categories: [theory,traditional-machine-learning,applying-knowledge,mlops,data-eng]
+---
+
+# Hello :) Today is Day 316!
+A quick summary of today:
+* read more of Chip Huyen's book
+* streamed
+
+## Designing ML systems - chapter 5 - Feature engineering
+
+### Common feature engineering operations
+
+* handling missing values (MNAR/MAR/MCAR)
+  * delete them
+  * impute them
+ 
+* scaling
+* discretisation (binarisation)
+* encoding cat features
+* feature crossing
+* discrete and continuous positional embeddings
+
+### Data leakage
+
+Common causes for data leakage
+
+* splitting time-correlated data randomly instead of by time
+* scaling before splitting
+* filling in missing data with stats from the test split
+* poor handling of data duplication before splitting
+* group leakage
+* leakage from data generation process
+
+### Engineering good features
+
+* the more features you have, the more opportunities there are for data leakage
+* too many features can cause overfitting
+* too many features can increase memory required to serve a model, which, in turn, might require you to use a more expensive machine/instance to serve your model
+* too many featuers can incrase inference latency when doing online prediction, especially if we need to extract these features from data raw
+* useless features become technical debt; whenever the data pipe changes, all the affected features need to be adjusted accordingly
+
+**Feature importance**
+
+* default model's feature importance method
+* SHAP
+* [InterpretML](https://oreil.ly/oPllN)
+
+**Feature generalisation**
+
+This ensures that an ML model can make reliable predictions on unseen data by using features that generalise well beyond the training set. Not all features have equal generalisation power. For instance, identifiers specific to each instance (like a comment ID) don't generalise, while identifiers linked to broader attributes (like usernames) might.
+
+Measuring feature generalisation relies on intuition, domain knowledge, and statistical understanding. Key factors include coverage—the percentage of data samples containing the feature—and value distribution. High coverage suggests better generalisability, while low coverage, especially if values are missing randomly, can reduce utility. However, if missingness is systematic (i.e. the feature only appears in cases with positive labels), even low-coverage features may add value.
+
+Features should ideally have consistent coverage and distributions across both training and testing splits. Variances here might indicate that the splits don’t share the same distribution, potentially hinting at data leakage. Overlapping distributions also help, as seen in time-based features like HOUR_OF_THE_DAY for traffic models, which often cover the same range across training and test sets. More specific features, like IS_RUSH_HOUR, generalise better but may lose critical detail provided by broader features, highlighting a trade-off between generalisation and specificity.
+
+## Chapter 6 - Model development and offline evaluation
+
+### Model development and training
+
+**6 tips for model selection**
+
+* avoid the state-of-the-art trap
+* start with the simplest models
+* avoid human biases in selecting models
+* evaluate good performance now vs. good performance later
+* evaluate trade-offs
+* understand our model's assumptions (prediction, IID, smoothness, tractability, boundaries, cond. independence, normally distributed)
+
+**Ensembles**
+
+They are popular solution for competitions like Kaggle, and provide competitive performance. However, they a re less favoured in production because ensembles are more complex to deploy and harder to maintain. However, they are still common for tasks where a small performance boost can lead to a huge financial gain, such as predicting click-through rate for ads.
+
+Ensembles consist of methods such as boosting, bagging and stacking, shown in the pictures below, respectively.
+
+![image](https://github.com/user-attachments/assets/900ed86d-0f86-46c9-b9ea-6cc103fb33cb)
+
+![image](https://github.com/user-attachments/assets/a7fc382c-7c14-4ba1-a904-c44610939805)
+
+![image](https://github.com/user-attachments/assets/c9901c9c-fde4-4eab-9d18-6058c42a9da4)
+
+**Experiment tracking and Versioning**
+
+- experiment tracking involves monitoring various aspects of the training process, such as loss curves, performance metrics, and system resource usage
+- versioning focuses on recording all details of each experiment, including code, data, and hyperparameters, to ensure reproducibility and facilitate comparisons
+
+Challenges of data versioning
+
+- handling large datasets can be challenging due to storage and processing limitations
+- defining and managing data differences adds complexity
+- legal concerns: issues such as GDPR introduce additional complications in data versioning, particularly concerning user data privacy
+
+**Debugging models**
+
+- silent model failures: models may fail without clear indicators, making issues hard to detect
+- time-consuming validation: validating bug fixes can be a lengthy process
+- component interplay; the intricate interaction among different ML components adds complexity
+
+**Common causes of failures**
+
+- theoretical constraints
+- implementation errors
+- poor hyperparameter choices
+- data-related issues
+- inadequate feature selection
+
+**Debugging techniques**
+
+1. start with a simple model and gradually add complexity to isolate issues
+2. overfit a small data batch which helps verify model functionality quickly
+3. set a random seed to ensure reproducibility for consistent results
+
+**Distributed training**
+
+- data parallelism: splits data across multiple machines for training, discussing synchronous and asynchronous stochastic gradient descent (SGD) and their respective challenges, along with batch size and learning rate trade-offs
+- model parallelism: trains different model components on separate machines
+  - pipeline parallelism: enhances parallel execution in model parallelism for efficiency
+
+**AutoML**
+
+AutoML aims to automate portions of the machine learning workflow. Two forms are discussed:
+
+* soft AutoML: focuses on hyperparameter tuning
+- hard AutoML: encompasses architecture search and learned optimizers
+
+**Four phases of ML model dev**
+
+1. non-ML solutions: establish a baseline and gain insights into the problem
+2. simple ML models: initial exploration with simpler models
+3. optimization: enhance model performance
+4. complex models: consider complex models if the use case requires it
+
+### Model offline evaluation
+
+To assess the effectiveness of ML models, companies need clear evaluation metrics. Without them, determining a model's accuracy or comparing models becomes challenging, as seen in a case where a company couldn't measure undetected intrusions by their ML system for drone surveillance. Although not having precise evaluation criteria doesn't doom an ML project, it hinders finding the best solution and convincing stakeholders to adopt ML. Collaboration with business teams to develop relevant metrics can bridge this gap. Ideally, evaluation methods used in development (where ground truth labels are available) should align with those in production, though this alignment can be difficult due to the lack of labels in production.
+
+**Baselines**
+
+Evaluation metrics, by themselves, mean little. When evaluating your model, it’s essential to know the baseline we’re evaluating it against. The exact baselines should vary from one use case to another, but here are the five baselines that might be useful across use cases:
+
+* random baseline (if our model were to predict at random)
+* simple heuristic
+* zero rule baseline (predicting the most common class)
+* human baseline
+* existing solutions
+
+**Evaluating methods**
+
+* pertrubation tests
+* invariance tests (i.e. keep the inputs the same but change the sensitive information , like race)
+* directional expectation tests (certain changes to the inputs should cause predictable changes in the outputs)
+* model calibration
+* confidence measurement
+* slice-based evaluation 
+
+Slice-based evaluation involves breaking down data into subsets(slices) and assessing model performance on each slice individually. This approach highlights performance disparities that may not appear in overall metrics, like accuracy or F1 score. For instance, a model might perform well overall but show significant bias against a minority subgroup if its accuracy is much lower on that slice.
+
+Ignoring slice-based metrics can lead to biased models and overlook areas for improvement. Slice-based evaluation also helps in cases where specific subsets are more critical, such as paid users in churn prediction. Without slicing, important trends, as illustrated by Simpson’s paradox, can be concealed, where combined data shows one trend but individual groups reveal the opposite.
+
+To perform effective slice-based evaluation, three main approaches are recommended:
+
+* heuristics-based slicing uses domain knowledge to identify relevant data dimensions
+* error analysis reviews misclassified examples to find common patterns
+* automated slice finders uses algorithms to identify and rank slice candidates
+
+## Chapter 7 - Model deployment and Prediction service
+
+_Production is a spectrum. For some teams, production means generating nice plots in notebooks to show to the business team. For other teams, production means keeping your models up and running for millions of users a day._
+
+### ML deployment myths
+
+* you only deploy one or two ML models at a time
+* if we don't do anything, model performance remains the same
+* you won't need to update your models as much
+* most MLEs don't need to worry about scale
+
+### Batch vs Online prediction
+
+* batch prediction, which uses only batch features
+* online prediction that uses only batch features (i.e. precomputed embeddings)
+* online prediction that uses both batch features and streaming features; also known as streaming prediction
+
+![image](https://github.com/user-attachments/assets/7154d76a-c942-4fec-867e-85c61af64a93)
+
+![image](https://github.com/user-attachments/assets/a80c8d11-d5c1-4198-a25d-3a0e9ce75733)
+
+Imagine we work at DoorDash, we might need to following features to estimate the delivery time:
+
+* batch features - the mean prep time of a particular restaurant in the past
+* streaming features - in the last 10 mins, how many other orders they have, and how many delivery people are available
+
+![image](https://github.com/user-attachments/assets/d47c3e2b-9aee-4261-9c98-e6bdfe214354)
+
+|                                  | Batch prediction (asynchronous)                     | Online prediction (synchronous)                    |
+|----------------------------------|-----------------------------------------------------|----------------------------------------------------|
+| **Frequency**                    | Periodical, such as every four hours                | As soon as requests come                           |
+| **Useful for**                   | Processing accumulated data when you don’t need immediate results (such as recommender systems) | When predictions are needed as soon as a data sample is generated (such as fraud detection) |
+| **Optimized for**                | High throughput                                     | Low latency                                        |
+
+### Model compression
+
+* low-rank factorisation: replace high-dimensional tensors with lower-dimensional tensors
+* knowledge distillation: a small model (student) is trained to mimic a larger model or ensemble of models (teacher)
+* pruning parameters
+* quantization: reduces a model’s size by using fewer bits to represent its parameters
+
+### ML on the Cloud and on the Edge
+
+![image](https://github.com/user-attachments/assets/e43fb070-7a2f-4dda-8593-ef0e579d6709)
+
+As hardware becomes more powerful, ML models will move to online and on the edge
+
+## [Streamed](https://youtu.be/dF0jPc82PqU)
+
+* [tuning the decision threshold for class prediction](https://scikit-learn.org/stable/modules/classification_threshold.html)
+* [validation curves](https://scikit-learn.org/stable/modules/learning_curve.html)
+* [shap package readme](https://github.com/shap/shap)
+* [InterpretML's homepage](https://interpret.ml/docs/)
+
+I will coninue reading the SHAP and interpretML's docs in the next few streams because they seem interesting.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 2e54de76cfcad810508de9dfe6c6fe660c46300c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 14 Nov 2024 00:10:13 +0900
Subject: [PATCH 495/638] Create 2024-11-13-day-317.md

---
 _posts/2024-11-13-day-317.md | 191 +++++++++++++++++++++++++++++++++++
 1 file changed, 191 insertions(+)
 create mode 100644 _posts/2024-11-13-day-317.md

diff --git a/_posts/2024-11-13-day-317.md b/_posts/2024-11-13-day-317.md
new file mode 100644
index 000000000..59b8c23f1
--- /dev/null
+++ b/_posts/2024-11-13-day-317.md
@@ -0,0 +1,191 @@
+---
+layout: post
+title: (Day 317) Streaming + more of Chip Huyen's book
+categories: [theory,traditional-machine-learning,applying-knowledge,mlops,data-eng]
+---
+
+# Hello :) Today is Day 317!
+A quick summary of today:
+* read two more chapters of Chip Huyen's book
+* streamed for 2hrs
+* read a bit of the sklearn docs
+
+## Designing ML systems - Chapter 8 - Data distribution shifts and monitoring
+
+### Causes of ML system failures
+
+**Software system failures**
+
+* dependency failure
+* deployment failure
+* hardware failures
+* downtime or crashing
+
+**ML-specific failures**
+
+* production data differing from train data
+* edge cases
+* degenerate feedback loops
+
+### Data distribution shifts
+
+**Types**
+
+* covariate shift (when P(X) changes but P(Y|X) remains the same; this refers to the first decomposition of the joint distribution)
+* label shift (when P(Y) changes but P(X|Y) remains the same; this refers to the second decomposition of the joint distribution)
+* concept drift (when P(Y|X) changes but P(X) remains the same. This refers to the first decomposition of the joint distribution)
+
+Detecting shifts involves monitoring accuracy metrics, but when ground truth labels are delayed, examining distributions like P(X), P(Y), P(X|Y), and P(Y|X) can help.
+
+Industry tools often compare summary statistics like mean, variance, or skewness to detect shifts, but more sophisticated two-sample tests (i.e. Kolmogorov–Smirnov test, MMD) offer statistical confirmation of distribution differences. Temporal and spatial shifts add complexity, requiring time-series analysis techniques to detect changes across different time windows.
+
+To address shifts, companies may periodically retrain models, use extensive datasets to enhance robustness, or adapt models to new distributions. Feature selection should balance predictive power and stability, as frequently shifting features (like app ranking) can require frequent retraining. Tailoring models to specific markets can also minimize unnecessary updates. Robust systems should be designed for adaptability, and not all performance issues require ML fixes; human errors remain a common cause.
+
+### Monitoring and observability
+
+**ML-specific metrics**
+
+There are generally 4 artifacts to monitor: a model's accuracy-related metrics, predictions, features, and raw inputs. 
+
+![image](https://github.com/user-attachments/assets/e3d0db47-b6d2-4bfa-8513-86c049345d30)
+
+The more transformations an artifact has gone through, the more likely its changes are to be caused by errors in one of those transformations
+
+To monitor ML model performance, tracking user feedback and predictions is crucial. User feedback like clicks, upvotes, and completion rates offers insights into model accuracy. For instance, in a recommendation system, a stable click-through rate but dropping completion rates may indicate performance issues. Feedback can also guide the labeling process, as seen with Google Translate’s upvote/downvote feature.
+
+Prediction monitoring helps detect distribution shifts, signaling input data changes. Consistent abnormal predictions can indicate immediate issues, unlike slower metrics like accuracy.
+
+Feature monitoring validates data consistency but has challenges, such as resource costs and 'alert fatigue' due to frequent, non-critical alerts. Versioning schemas helps differentiate between genuine data shifts and schema mismatches.
+
+Finally, raw input monitoring—typically managed by data platforms—is essential to detect early-stage data issues but is often inaccessible to ML teams due to data processing structures. These metrics collectively provide a comprehensive view of ML model health.
+
+**Monitoring toolbox**
+
+* logs
+* dashboards
+* alerts
+
+**Observability**
+
+_Monitoring centers around metrics, and metrics are usually aggregated. Observability allows more fine-grain metrics, so that you can know not only when a model’s performance degrades but also for what types of inputs or what subgroups of users or over what period of time the model degrades_
+
+In ML, observability encompasses interpretability. Interpretability helps us understand how an ML model works, and observability helps us understand how the entire ML system, which includes the ML model, works. For example, when a model’s performance degrades over the last hour, being able to interpret which feature contributes the most to all the wrong predictions made over the last hour will help with figuring out what went wrong with the system and how to fix it.
+
+## Chapter 9 - Continual learning and test in production
+
+### Continual Learning
+
+The idea is for a model to update itself as new data comes in. However, if the model updates itself with every single new data point, that could lead to catastrophic forgetting, and is also a waste of compute power. 
+
+Companies that employ continual learning in production update their models in micro-batches - i.e. after every 512/1024 examples (the amount is task dependent). And the updated model shouldn't be deployed until it's been evaluated. 
+
+![image](https://github.com/user-attachments/assets/7c777c84-5ea1-4a0b-a0f5-1348fcc77ff9)
+
+**Stateless retraining vs. Stateful training**
+
+Continual learning isn’t about the retraining frequency, but the manner in which the model is retrained. Most companies do stateless retraining—the model is trained from scratch each time. Continual learning means also allowing stateful training—the model continues training on new data. Stateful training is also known as fine-tuning or incremental learning.
+
+![image](https://github.com/user-attachments/assets/e9dcbcb6-f573-4746-ba1d-77249f2e101f)
+
+Stateful training sounds cool, but how does this work if I want to add a new feature or another layer to my model? To answer this, two types of model updates need to be differentiated:
+
+* model iteration: a new feature is added to an existing model architecture or the model architecture is changed
+* data iteration: the model architecture and features remain the same, but you refresh this model with new data
+
+As of today, stateful training is mostly applied for data iteration, as changing your model architecture or adding a new feature still requires training the resulting model from scratch
+
+**Why continual learning?**
+
+CL enables fast, frequent model updates, addressing several challenges in machine learning:
+
+* adaptation to data shifts: when sudden changes occur, such as an unexpected surge in demand for ride-sharing, continual learning allows models to adjust quickly, improving user experience and preventing revenue loss
+* handling rare events: CL helps adapt to infrequent but impactful events, like Black Friday, by learning from fresh data throughout the day, enhancing performance without requiring extensive historical data
+* solving continuous cold start: unlike traditional cold start, CL allows models to adapt to both new users and changing user behaviors. For instance, if a user switches devices, the model can still make relevant recommendations, as seen in TikTok's success with personalized content within minutes of app usage
+
+CL expands on batch learning by offering real-time adaptability for various complex use cases. With advancing MLOps tools, it could soon be as accessible and practical as batch learning, making it a compelling choice for modern ML applications.
+
+**CL challenges**
+
+* fresh data access challenge
+* evaluation challenge
+* algorithm challenge (affects only matrix-based and tree-based models that want to be updated very fast - like hourly)
+
+**4 stages  of CL**
+
+1. Manual, stateless retraining
+2. Automated retraining
+3. Automated, stateful training
+4. Continual learning
+
+**How often to update your models?**
+
+* experiment with different time windows: train models on data from different periods and test them on current data. Comparing performance reveals how much fresher data improves results
+* fine-grained experiments: adjust timeframes (months, weeks, days) to determine optimal update frequency. For instance, Facebook shifted ad CTR prediction retraining from weekly to daily due to a measurable 1% gain
+
+### Test in production
+
+**Shadown deployment**
+
+1. deploy the candidate model in parallel with the existing model
+2. for each incoming request, route it to both models to make predictions, but only serve the existing model's prediction to the user
+3. log the predictions from the new model for analysis purposes
+
+Once the new model's results have been found satisfactory, we can reaplce the existing model
+
+**A/B testing**
+
+1. deploy the candidate model alongside the existing model
+2. a percentage of traffic is routed to the new model for predictions; the rest is routed to the existing model for predictions. It’s common for both variants to serve prediction traffic at the same time. However, there are cases where one model’s predictions might affect another model’s predictions— i.e., in ride-sharing’s dynamic pricing, a model’s predicted prices might influence the number of available drivers and riders, which, in turn, influence the other model’s predictions. In those cases, you might have to run your variants alternatively, e.g., serve model A one day and then serve model B the next day
+3. monitor and analyze the predictions and user feedback, if any, from both models to determine whether the difference in the two models’ performance is statistically significant
+
+Randomisation is essential: traffic must be evenly randomized between models to avoid biases like device differences (e.g., mobile vs. desktop) that can skew results. 
+
+Sufficient sample size is crucial for statistical confidence. The test’s outcome should be validated through hypothesis testing, typically with two-sample tests to determine statistical significance. However, even statistically significant results aren’t foolproof, as repeated tests may yield different results. Non-significant results can indicate minimal model differences, making either model acceptable. For more on A/B testing, multiple variant testing (A/B/C or A/B/C/D), and related statistics, readers are advised to explore additional literature.
+
+**Canary release**
+
+Canary release is a technique to reduce the risk of introducing a new software version in production by slowly rolling out the change to a small subset of users before rolling it out to the entire infrastructure and making it available to everybody
+
+1. deploy the candidate model alongside the existing model; the candidate is called the canary
+2. a portion of the traffic is routed to the candidate
+3. if its performance is satisfactory, increase the traffic to the candidate. If not, abort the canary and route all the traffic back to the existing model
+4. stop when either the canary serves all the traffic or when the canary is aborted
+
+**Interleaving experiments**
+
+Imagine there are 2 recommender systems -  A and B, and we want to evaluate which one is better. Each time, a model recommends 10 items users might like. With A/B testing, you’d divide your users into two groups: one group is exposed to A and the other group is exposed to B. Each user will be exposed to the recommendations made by one model.
+
+Instead, the idea behind interleaving experiments is to expose the user to recommendations from both models and see which one they prefer. 
+
+![image](https://github.com/user-attachments/assets/6bc0a3c3-600a-4fe4-93be-9b5edd0e32eb)
+
+**Bandits**
+
+Bandit algorithms, originating from gambling, are used to balance exploration (testing new options) and exploitation (leveraging known best options). In model evaluation, a multi-armed bandit approach routes traffic based on models’ current performance, unlike stateless A/B testing, which simply divides traffic randomly. This approach requires online prediction capability, rapid feedback loops, and mechanisms to update performance based on feedback, making bandits data-efficient but complex to implement.
+
+Basic bandit strategies include ε-greedy (favoring the top model most of the time), Thompson Sampling (selects based on probability of optimality), and Upper Confidence Bound (adds an 'exploration bonus' for uncertainty). Contextual bandits extend these principles, adjusting recommendations by learning from user feedback, which is challenging due to partial feedback and model dependencies.
+
+Although powerful, bandit algorithms are rare outside big tech due to complexity. Good evaluation pipelines are essential to ensure reliable model deployment, reducing failure risks by defining tests, thresholds, and automation for continuous evaluation, similar to CI/CD in software development.
+
+## Stream
+
+* [first stream](https://www.youtube.com/watch?v=iOXWnaqhSxk&t=18s)
+  * did some db setup for Zach Wilson's data eng camp starting this Friday
+  * [read SHAP's intro docs page](https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html)
+
+* [second stream](https://www.youtube.com/watch?v=oLEm7e24sMo)
+  * [covered InterpretML's glassbox models section](https://interpret.ml/docs/glassbox.html)
+ 
+After stream I read:
+
+* [Pipelines and composite estimators](https://scikit-learn.org/stable/modules/compose.html)
+* [Feature extraction](https://scikit-learn.org/stable/modules/feature_extraction.html#)
+* [Preprocessing data](https://scikit-learn.org/stable/modules/preprocessing.html#)
+* [Imputation of missing values](https://scikit-learn.org/stable/modules/impute.html#)
+* [Random projection](https://scikit-learn.org/stable/modules/random_projection.html)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 01563375c5da437f9cde121c9fec73bbd14fad46 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 15 Nov 2024 02:11:27 +0900
Subject: [PATCH 496/638] Create 2024-11-14-day-318.md

---
 _posts/2024-11-14-day-318.md | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 _posts/2024-11-14-day-318.md

diff --git a/_posts/2024-11-14-day-318.md b/_posts/2024-11-14-day-318.md
new file mode 100644
index 000000000..b87857c7d
--- /dev/null
+++ b/_posts/2024-11-14-day-318.md
@@ -0,0 +1,42 @@
+---
+layout: post
+title: (Day 318) I'm a scikit-learn professional
+categories: [theory,traditional-machine-learning,applying-knowledge,mlops,nlp]
+---
+
+# Hello :) Today is Day 318!
+A quick summary of today:
+
+## Practical notebook attemp
+
+I am writing this at 2am as I was up just continuing what I started on stream - a practical notebook encouraging learners to think of improving models in terms of improving some business metric. I made something that is cool but I think I need to change the idea :D I was going the route of optimising Expected Loss (for loans) but I think I will go down the route of pretending that obtaining official FICO scores is too expensive and a stakeholder wants to use our current data and current scores to try to create a model that replicates the FICO scores for which we pay.
+
+## scikit-learn professional certification - completed
+
+Anyway ~ the big news ~
+
+<img width="976" alt="image" src="https://github.com/user-attachments/assets/018c47f4-4a98-4620-9cf0-13f5487fe000">
+
+I passed the professional scikit-learn exam 🥳🥳🥳
+
+I did not think I will do it today, but I decided to try and luckily I passed haha
+
+## Comapny reviewer chatbot
+
+Today I was in the lab working on the company reviewer chatbot. Here is an updated table
+
+<img width="426" alt="image" src="https://github.com/user-attachments/assets/4d6ac25c-7477-42db-90e0-cdc8a45c0ea4">
+
+Tomorrow I will do the few-shot versions.
+
+## [Streamed]
+
+I streamed for a bit today and also had my 1st chat with a viewer - about tutorials/practical exercises/courses I could recommend. Shoutout to 'Benjamin Franklin' (the person's username)
+
+---
+
+This post is kind of short and the structure is not that great, so apologies for that, but it is 2am and my eyes are almost closed.
+
+That is all for today!
+
+See you tomorrow :)

From 7f6e23bc0413568443082d7d2186925411e3ed1a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 15 Nov 2024 02:12:14 +0900
Subject: [PATCH 497/638] Update 2024-11-14-day-318.md

---
 _posts/2024-11-14-day-318.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/_posts/2024-11-14-day-318.md b/_posts/2024-11-14-day-318.md
index b87857c7d..e7333c043 100644
--- a/_posts/2024-11-14-day-318.md
+++ b/_posts/2024-11-14-day-318.md
@@ -6,6 +6,9 @@ categories: [theory,traditional-machine-learning,applying-knowledge,mlops,nlp]
 
 # Hello :) Today is Day 318!
 A quick summary of today:
+* streamed
+* took the sklearn level 2 certification
+* worked on the company reviewer llm
 
 ## Practical notebook attemp
 

From 3a871ef13f4b26aee4f2653e72a3d6ab2b722a53 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 16 Nov 2024 00:19:17 +0900
Subject: [PATCH 498/638] Create 2024-11-15-day-319.md

---
 _posts/2024-11-15-day-319.md | 120 +++++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 _posts/2024-11-15-day-319.md

diff --git a/_posts/2024-11-15-day-319.md b/_posts/2024-11-15-day-319.md
new file mode 100644
index 000000000..ba259a6ef
--- /dev/null
+++ b/_posts/2024-11-15-day-319.md
@@ -0,0 +1,120 @@
+---
+layout: post
+title: (Day 319) Creating a practical ML notebook to evaluate a model based on some business statistic
+categories: [traditional-machine-learning,applying-knowledge,mlops,nlp]
+---
+
+# Hello :) Today is Day 319!
+A quick summary of today:
+* creating a practical ML notebook
+* streaming
+* more company reviewer LLMs
+
+## [Practical notebook](https://www.kaggle.com/code/divakaivan12/bank-ml-model-based-on-business-metrics)
+
+I uploaded the whole notebook on Kaggle for feedback but I think I got a good thing to start with. At the moment, the code is in the notebook but what I'd to do is create a sklearn MOOC style exercise. I encourage you to check the full Kaggle notebook but below is just the start and end of it.
+
+### Goal of this notebook
+
+*'Most companies don’t care about the fancy ML metrics. They don’t care about increasing a model’s accuracy from 94% to 94.2% unless it moves some business metrics.' - [Chip Huyen, Designing Machine Learning Systems](https://www.oreilly.com/library/view/designing-machine-learning/9781098107956/)*
+
+Chip Huyen (among many other practitioners) highlights that ML models hold value only when they drive meaningful benefits for the business. The goal of this notebook is to encourage you to develop a model with a focus on enhancing a relevant business metric, rather than just optimizing traditional ML metrics.
+
+### Problem Setting
+
+If you are an adult, chances are you have heard about credit scores (or FICO scores). Credit scores are a numerical representation of an individual's creditworthiness. They are used by lenders to assess how likely someone is to repay borrowed money or manage financial responsibilities. A credit score, usually ranging from 300 to 850, is based on various factors, including payment history, total debt, length of credit history, types of credit accounts, and recent credit inquiries. Higher scores typically indicate lower risk, making it easier to secure loans, credit cards, or favorable interest rates. Conversely, lower scores can make credit access challenging and more expensive.
+
+Experian is one of the most famous and reliable provider of these FICO scores, and banks typically purchase these scores to assess the credit risk of potential borrowers and make lending decisions.
+
+Imagine we are a seasoned bank with lots of data on our customers and a big loan portfolio. A stakeholder comes to you and says that given the bank needs to cut costs. Given the bank's long history and data availability, the stakeholder believes that we can cut costs by not purchasing FICO scores from providers but calculate them ourselves.
+
+In the provided dataset there is information about 7500 loans, each with information about the borrower, the loan, and the respective FICO score which we have from Experian. We will assume that the FICO scores are the actual truth (for transparency - I generated them using a custom PD model. If you want to learn how to do that check the Udemy course below). Given that information, the stakeholder would like you to create a model that estimates the FICO scores with the least difference between your model's output and the actual FICO scores.
+
+The stakeholder would also like to know how the bank's expecte loss will be affected in case your model's FICO scores differ from the original FICO scores. Don't worry, help on calculating this will be provided. 
+
+*If you want to learn more about credit risk modelling, I recommend this course on [Udemy](https://www.udemy.com/course/credit-risk-modeling-in-python/)*
+
+<img width="932" alt="image" src="https://github.com/user-attachments/assets/811d9861-9000-4d71-ab96-5edd92eaed85">
+
+---
+
+**And here is the 'final' report to the stakeholders:**
+
+---
+
+This report compares the performance of the current and challenger models used to determine the required loan reserve. Key metrics, including the accuracy of FICO score estimations, expected loss, and reserve ratio, are analyzed to assess improvements made by the challenger model.
+
+
+
+Here are the corrected markdown tables:
+
+
+### Key Metrics
+
+| Metric                                      | Model 1 (Current) | Model 2 (Challenger) |
+|---------------------------------------------|--------------------|-----------------------|
+| **% of loans for which the FICO score is overshot by more than 5%** | 67.23%            | 0.03%                |
+| **% of loans outside the required 5% margin** | 64.20%            | 0.013333             |
+| **Expected Loss**                           | $9,909,298.07     | $7,871,616.41        |
+| **Model Reserve Ratio**                     | 10.83%            | 8.61%                |
+
+*Regulators require the bank to keep a minimum of 8% of our loan portfolio as reserves.*
+
+---
+
+### FICO Score Range Comparison
+
+This table shows the percentage difference in FICO score predictions by range. Values outside the ±5% range are areas of concern, with Model 2 demonstrating significant improvement over Model 1 in accuracy.
+
+| FICO Score Range       | Model 1 Difference (%) | Model 2 Difference (%) |
+|------------------------|------------------------|------------------------|
+| **Poor (300-579)**     | -100.00               | -0.61                 |
+| **Fair (580-669)**     | 90.74                | 0.03                  |
+| **Good (670-739)**     | -100.00              | 0.83                  |
+| **Very Good (740-799)**| -100.00              | 4.41                  |
+| **Exceptional (800-850)** | NaN                | NaN                   |
+
+---
+
+
+### Analysis
+
+1. **Accuracy Improvement**: Model 2 significantly reduces the percentage of loans overshot by more than 5%, bringing it down to just 0.03% compared to Model 1's 67.23%. Additionally, Model 2 maintains a high level of accuracy across all FICO score ranges, with no difference exceeding ±5%.
+
+
+
+2. **Expected Loss Reduction**: Model 2 decreases the expected loss from $9.91M to $7.87M, an improvement that could yield substantial financial benefits.
+
+
+
+3. **Reserve Ratio Compliance**: Model 2's reserve ratio of 8.61% meets the regulatory requirement of keeping at least 8% in reserves and is closer to this minimum requirement than Model 1's 10.83%, optimizing capital allocation. 
+
+
+
+### Conclusion
+
+The challenger model (Model 2) demonstrates clear advantages in prediction accuracy, financial efficiency, and regulatory compliance. Transitioning to Model 2 is recommended to optimize reserve allocation while maintaining adherence to regulatory requirements.
+
+## [Streamed](https://youtu.be/tHEIi09SOOw)
+
+I started watching the end projects of [CS 329S Demo Day 2022](https://youtu.be/AZNTqytOhXk) - Stanford's ML system design course taught by Chip Huyen. 
+
+## I was again in the lab today - fine-tuning LLMs on the company review dataset
+
+This is the updated evaluation table 
+
+<img width="424" alt="image" src="https://github.com/user-attachments/assets/d18ee8e0-f687-445c-84d7-deac4ac1361c">
+
+I need to do the last 4. For one of the Qwen2.5 models I had to reduce the max input (to 10K) so that it can train and my PC does not run out of memory. 
+
+I tried following a prompt tuning guide but the model training time was 9 hours ?? Not sure if that's normal of maybe I did something wrong. It is something I need to figure out. 
+
+## Interview with the product designers of [skore](https://github.com/probabl-ai/skore/)
+
+One of the product designers invited me to do an interview 'to understand more about how you could use our new open source tool skore, and how it would answer your needs'. We scheduled it for next Thursday so I am really excited.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From c2d0d0e096509da4275587d98f33fcf08b689c4f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 16 Nov 2024 23:34:43 +0900
Subject: [PATCH 499/638] Create 2024-11-16-day-320.md

---
 _posts/2024-11-16-day-320.md | 169 +++++++++++++++++++++++++++++++++++
 1 file changed, 169 insertions(+)
 create mode 100644 _posts/2024-11-16-day-320.md

diff --git a/_posts/2024-11-16-day-320.md b/_posts/2024-11-16-day-320.md
new file mode 100644
index 000000000..95099fae8
--- /dev/null
+++ b/_posts/2024-11-16-day-320.md
@@ -0,0 +1,169 @@
+---
+layout: post
+title: (Day 320) Day 1 of Zach Wilson's DE bootcamp - Data Modelling
+categories: [data-eng,applying-knowledge,theory]
+---
+
+# Hello :) Today is Day 320!
+A quick summary of today:
+* watched the lecture + covered the lab 
+
+The videos are available on youtube as well, but I am going to do it on his dataexpert platform to get credits ^^ And also I can submit assignments on there as well
+
+<img width="955" alt="image" src="https://github.com/user-attachments/assets/771e207e-1cdd-4fb5-9a44-67a7114258fe">
+
+I decided to take notes while watching the lecture/lab:
+
+![Dimensional data modelling day 1 lecture P1](https://github.com/user-attachments/assets/9b1e969f-e891-4ce3-a029-fedb16134364)
+![Dimensional data modelling day 1 lecture P2](https://github.com/user-attachments/assets/a09828a9-8f85-4bb4-9989-3b88fea8082d)
+![Dimensional data modelling day 1 lecture P3](https://github.com/user-attachments/assets/a9cdfae1-f6e3-4cf2-804b-9fa23c2abce5)
+![Dimensional data modelling day 1 lecture P4](https://github.com/user-attachments/assets/e7164aa5-720f-4eda-8ab6-7f905099adda)
+![Dimensional data modelling day 1 lecture P5](https://github.com/user-attachments/assets/282a1961-0547-4012-895d-4f46e9d49f25)
+
+And here is my code from the lab: (I am using DataGrip)
+
+```sql
+select * from player_seasons;
+
+-- create a struct
+create type season_stats as (
+    season integer,
+    gp integer,
+    pts real,
+    reb real,
+    ast real
+);
+
+create type scoring_class as enum ('star', 'good', 'average', 'bad');
+
+create table players (
+    player_name text,
+    height text,
+    college text,
+    country text,
+    draft_year text,
+    draft_round text,
+    draft_number text,
+    season_stats season_stats[],
+    scoring_class scoring_class,
+    years_since_last_season integer,
+    current_season integer,
+    primary key(player_name, current_season)
+);
+
+select min(season) from player_seasons;
+-- 1996 - this is the 1st year in the data
+-- used for the cumulative table
+
+-- this is like a 'pipeline' where we can change the seasons
+-- as if it's yesterday and today, and insert new data into players
+insert into players
+with yesterday as (
+    select * from players
+             where current_season = 1994
+),
+    today as (
+        select * from player_seasons
+                 where season = 1995
+    )
+
+select
+    coalesce(t.player_name, y.player_name) as player_name,
+    coalesce(t.height, y.height) as height,
+    coalesce(t.college, y.college) as college,
+    coalesce(t.country, y.country) as country,
+    coalesce(t.draft_year, y.draft_year) as draft_year,
+    coalesce(t.draft_round, y.draft_round) as draft_round,
+    coalesce(t.draft_number, y.draft_number) as draft_number,
+    case
+        when y.season_stats is null then array[row(
+            t.season,
+            t.gp,
+            t.pts,
+            t.reb,
+            t.ast
+            )::season_stats] --:: casting it
+        when t.season is not null then y.season_stats || array[row(
+            t.season,
+            t.gp,
+            t.pts,
+            t.reb,
+            t.ast
+            )::season_stats] -- || concat
+        else y.season_stats
+        -- if a player has retired just keep his stats
+        -- don't add empty values
+    end as season_stats,
+    case
+        when t.season is not null then
+            case when t.pts > 20 then 'star'
+                when t.pts > 15 then 'good'
+                when t.pts > 10 then 'average'
+                else 'bad'
+            end::scoring_class
+        else y.scoring_class
+    end as scoring_class,
+    case
+        when t.season is not null then 0
+        else y.years_since_last_season + 1
+    end as years_since_last_season,
+    coalesce(t.season, y.current_season+1) as current_season
+-- using the coalesce line is much nicer
+--     case
+--         when t.season is not null then t.season
+--         else y.current_season + 1
+--     end
+from today t full outer join yesterday y
+    on t.player_name = y.player_name;
+
+-- turn the 1 line about MJ to 3 lines because
+-- at that point in our data he has stats for 3 seasons
+-- and season_stats was filled with 3 items
+select player_name,
+       unnest(season_stats) as season_stats
+       from players
+where current_season = 2001
+and player_name = 'Michael Jordan';
+
+-- show how to explode so we get the columns as well
+-- going back to the old schema
+-- unnest keeps the temporal pieces together
+with unnested as (
+    select player_name,
+         unnest(season_stats) as season_stats
+    from players
+    where current_season = 2001
+-- it is all sorted
+--     and player_name = 'Michael Jordan'
+)
+select player_name,
+       (season_stats::season_stats).*
+from unnested;
+
+select * from players
+where current_season = 2001;
+
+-- check which player has the biggest improvement from their debut to their most recent season
+-- there is no group by yet it feels like we grouped by so it is fast
+-- the order by is the slowest part
+select
+
+    player_name,
+    (season_stats[cardinality(season_stats)]::season_stats).pts/
+    case when (season_stats[1]::season_stats).pts = 0 then 1 else (season_stats[1]::season_stats).pts end
+from players
+where current_season = 2001
+order by 2 desc;
+```
+
+After watching the lecture+lab once, I decided to rewatch it just because the 1st time I kept pausing to take notes/write code meself.
+
+Definitely amazing, learned a lot, and am excited for the next lecture + lab coming out Monday Korean time.
+
+---
+
+Today I wanted to go to the lab but all building doors were locked 😆 so I could not even enter the building. Usually there is 1 open door but not today :/ I will try again tomorrow.
+
+That is all for today!
+
+See you tomorrow :)

From 0ed316b9595042afc77481e83bebc4c3af8f0eeb Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 18 Nov 2024 00:11:25 +0900
Subject: [PATCH 500/638] Create 2024-11-17-day-321.md

---
 _posts/2024-11-17-day-321.md | 287 +++++++++++++++++++++++++++++++++++
 1 file changed, 287 insertions(+)
 create mode 100644 _posts/2024-11-17-day-321.md

diff --git a/_posts/2024-11-17-day-321.md b/_posts/2024-11-17-day-321.md
new file mode 100644
index 000000000..8907760c1
--- /dev/null
+++ b/_posts/2024-11-17-day-321.md
@@ -0,0 +1,287 @@
+---
+layout: post
+title: (Day 321) Slowly changing dimensions
+categories: [data-eng,applying-knowledge,theory,mlop]
+---
+
+# Hello :) Today is Day 321!
+A quick summary of today:
+* learned about slowly changing dimensions
+* finished Chip Huyen's book
+
+Today I mostly hung out in Zach Wilson's discord and chated with people about the homework and lectures.
+
+Today, just a lecture about SDC was released and here are my notes from it:
+
+![Slowly changing dims and idempotency day 2 P1](https://github.com/user-attachments/assets/42deed0f-e58e-48ac-bea6-9c7a81095708)
+![Slowly changing dims and idempotency day 2 P2](https://github.com/user-attachments/assets/e5143522-0f85-4251-ab30-e381333f523b)
+![Slowly changing dims and idempotency day 2 P3](https://github.com/user-attachments/assets/4c2262f5-7e44-48e8-aef2-cd75a6e332c1)
+
+And I started doing the [homework](https://github.com/DataExpert-io/data-engineer-handbook/blob/main/bootcamp/materials/1-dimensional-data-modeling/homework/homework.md)
+
+Here is my solution (not sure if its correct but looks good 😆) for 1 and 2
+
+1. **DDL for `actors` table:** Create a DDL for an `actors` table with the following fields:
+    - `films`: An array of `struct` with the following fields:
+		- film: The name of the film.
+		- votes: The number of votes the film received.
+		- rating: The rating of the film.
+		- filmid: A unique identifier for each film.
+
+    - `quality_class`: This field represents an actor's performance quality, determined by the average rating of movies of their most recent year. It's categorized as follows:
+		- `star`: Average rating > 8.
+		- `good`: Average rating > 7 and ≤ 8.
+		- `average`: Average rating > 6 and ≤ 7.
+		- `bad`: Average rating ≤ 6.
+    - `is_active`: A BOOLEAN field that indicates whether an actor is currently active in the film industry (i.e., making films this year).
+    
+2. **Cumulative table generation query:** Write a query that populates the `actors` table one year at a time.
+
+```sql
+create type films as (
+    film text,
+    votes integer,
+    rating real,
+    filmid text
+);
+
+create type quality_class as enum ('star', 'good', 'average', 'bad');
+
+create table actors (
+    actorid text,
+    actor text,
+    films films[],
+    quality_class quality_class,
+    current_year integer,
+    is_active boolean,
+    primary key (actorid, current_year)
+);
+
+-- select min(year) from actor_films; -- 1970
+
+DO $$
+DECLARE
+    y_year INT; -- Previous year
+    t_year INT; -- Target year
+BEGIN
+    FOR y_year IN 1969..2020 LOOP
+        t_year := y_year + 1;
+        insert into actors
+        with yesterday as (
+            select *
+            from actors
+            where current_year = y_year
+        ),
+        today_aggregated as (
+            select
+                actorid,
+                max(actor) as actor,
+                array_agg(row(film, votes, rating, filmid)::films) as films,
+                max(year) as year,
+                avg(rating) as average_rating
+            from actor_films
+            where year = t_year
+            group by actorid
+        )
+        select
+            coalesce(t.actorid, y.actorid) as actorid,
+            coalesce(t.actor, y.actor) as actor,
+            case
+                when y.films is null then t.films
+                when t.films is not null then y.films || t.films
+                else y.films
+            end as films,
+            case
+                when t.films is not null then
+                    case when t.average_rating > 8 then 'star'
+                        when t.average_rating > 7 then 'good'
+                        when t.average_rating > 6 then 'average'
+                        else 'bad'
+                    end::quality_class
+                else y.quality_class
+            end as quality_class,
+            coalesce(t.year, y.current_year + 1) as current_year,
+            case when t.films is not null then true else false end as is_active
+        from today_aggregated t
+        full outer join yesterday y
+            on t.actorid = y.actorid;
+    END LOOP;
+END $$;
+```
+
+## Designing ML systems Chapter 10: Infrastructure and Tooling for MLOps
+
+Different layers of infra for ML
+
+![image](https://github.com/user-attachments/assets/69808827-5ac1-4614-866f-28d07411cbe6)
+
+### Storage and Compute
+
+**Storage Layer:**
+
+The storage layer is where data is collected and stored, ranging from simple HDDs or SSDs to cloud-based solutions like Amazon S3 and Snowflake. While earlier companies managed their own storage, the last decade has seen a shift to commoditized, cloud-based storage. The affordability of cloud storage enables companies to retain vast amounts of data without prohibitive costs.
+
+**Compute Layer:**
+The compute layer powers ML workloads by providing the resources to process data. It can range from a single CPU/GPU core to cloud-managed instances like AWS EC2 or GCP units. Compute units are characterized by memory (i.e. GB) and speed (i.e. FLOPS). Utilization, or how effectively compute resources are used, depends on memory bandwidth and data input speeds.
+
+Benchmarks like MLPerf help evaluate compute performance by measuring tasks such as training a ResNet-50 model or generating predictions using BERT. For simplicity, people often focus on the number of cores and memory when selecting instances.
+
+**Cloud vs. On-Premises:**
+Cloud computing's elasticity allows companies to scale workloads dynamically, reducing operational overhead. This flexibility suits bursty ML workloads, where experiments demand high compute power temporarily. However, cloud costs are significant, often prompting mature companies to repatriate workloads to private data centers to cut expenses. For example, Dropbox saved $75M by optimizing its infrastructure.
+
+**Multicloud Strategy:**
+To avoid vendor lock-in, many organizations adopt multicloud strategies, spreading workloads across providers like AWS, GCP, and Azure. This approach leverages cost-effective services but introduces complexity, often resulting from acquisitions or independent decision-making within organizations.
+
+Cloud and compute strategies should balance cost, scalability, and operational complexity based on a company's size and workload patterns.
+
+### Development Environment
+
+The development (dev) environment is where ML engineers write code, run experiments, and interact with production systems. It comprises tools like IDEs, versioning systems, and CI/CD pipelines. Despite its importance, many companies underinvest in their dev environments, often solving development processes in ad-hoc ways. Improvements in the dev environment directly enhance engineering productivity.
+
+**Key Components of a Dev Environment**
+
+Integrated Development Environment (IDE):
+
+IDEs include applications like VS Code or browser-based tools like AWS Cloud9.
+Notebooks, such as Jupyter and Google Colab, are favored by data scientists for their stateful execution, supporting exploratory data analysis and iterative experiments.
+Notebooks, while powerful, can complicate reproducibility due to out-of-order cell execution.
+
+Versioning:
+
+Version control is critical for managing code, data, and model artifacts in ML projects.
+
+Tools include Git (code), DVC (data), Weights & Biases, and MLflow (experiment tracking).
+Emerging platforms, like Claypot AI, aim to centralize ML workflow versioning.
+
+CI/CD Pipelines:
+
+Continuous Integration and Continuous Deployment (CI/CD) ensure code changes are tested before moving to production.
+
+Common tools include GitHub Actions and CircleCI.
+
+**Standardization of Dev Environments**
+
+Standardizing dev environments minimizes compatibility issues and boosts productivity.
+
+* ensure consistency in tools, packages, and virtual environments (e.g. conda)
+* maintain strict package versioning in requirements.txt to prevent dependency conflicts
+* use the same programming language version across the team
+* consider cloud dev environments for standardized setups and easier IT support
+
+Cloud environments offer added benefits:
+
+* cost-efficiency with auto-shutdown features (like GitHub Codespaces)
+* remote access through SSH
+* enhanced security by centralizing sensitive data
+* reduced gap between dev and production environments
+
+**Transitioning from Dev to Production: Containers**
+
+Containers streamline deployment by ensuring consistent environments. And to manage containers: use Kubernetes (K8s) for multi-host, scalable deployments, enabling auto-scaling and high availability.
+
+### Workflows
+
+Simple ML workflow DAG
+
+![image](https://github.com/user-attachments/assets/d7db0d39-3cc4-445f-bd0f-5f4fee1f0414)
+
+After a workflow is defined, the tasks in this workflow are scheduled and orchestrated
+
+![image](https://github.com/user-attachments/assets/d7aef7cc-898b-4e36-b30a-ad1c648b8a98)
+
+### ML Platform
+
+Machine learning (ML) platforms are transforming how companies manage and deploy ML solutions. A manager from a major streaming company shared a firsthand account of how his ML platform team was formed. Initially focused on recommender systems, they developed tools like feature management and model monitoring. Eventually, these tools were repurposed for broader ML applications, leading to the creation of a dedicated ML platform team. This mirrors a broader industry trend: companies are increasingly leveraging shared ML infrastructure to unify workflows across diverse applications.
+
+**Key ML Platform Components**
+
+Model deployment
+
+Deployment tools make ML models accessible for predictions—either online (real-time) or batch (precomputed). The market offers various options, from cloud-native tools like AWS SageMaker and GCP Vertex AI to open-source alternatives like MLflow Models and Seldon.
+
+Model store
+
+Beyond simply storing models, a robust model store tracks a wide range of artifacts: model definitions, parameters, dependencies, and training data. This ensures reproducibility, simplifies debugging, and supports long-term maintenance. MLflow is a popular open-source solution, though its artifact management capabilities are evolving.
+
+Feature store
+
+* feature management: centralized sharing and discovery of features across teams
+* feature computation: efficient computation and storage of features
+* feature consistency: ensuring consistent feature definitions between training and production
+
+### Build vs. Buy Considerations
+
+Build versus buy decisions depend on factors such as:
+
+* company stage: early-stage companies often buy vendor solutions to focus on core offerings, while mature companies may build in-house solutions as vendor costs rise
+* competitive focus: companies prioritize in-house development for areas tied to their competitive advantage, while others prefer vendor solutions, especially in non-tech sectors. Tech companies often favor modular, customizable services
+* tool maturity: if mature vendor tools are unavailable, teams may need to build their own solutions, potentially leveraging open-source options
+
+## Chapter 11: The Human Side of Machine Learning
+
+### User Experience
+
+Ensuring user experience consistency is crucial when integrating machine learning (ML) into applications or websites. Users expect predictable interfaces; for example, moving a frequently used button in a browser can lead to confusion. However, ML predictions are inherently probabilistic, meaning they can vary over time or contexts, potentially disrupting user expectations. A case study from Booking.com illustrates this challenge: the platform used ML to recommend filters to users but found that inconsistent suggestions frustrated users. To address this, they implemented rules to maintain consistency, such as retaining previously applied filters while allowing new recommendations when the user changes contexts. This highlights the balance between consistency and accuracy in ML systems.
+
+Another challenge with ML systems is handling "mostly correct" predictions. Large language models like GPT-3 can generate outputs for various tasks with minimal training, such as producing React code for a website. While these predictions can accelerate workflows, they are often imperfect. Users familiar with the domain can make corrections, but non-experts may struggle. To mitigate this, developers can provide multiple predictions for the same input and render them in user-friendly ways, such as displaying visualized web pages for evaluation. This approach, known as “human-in-the-loop” AI, allows users to select or refine outputs, ensuring a better overall experience.
+
+Lastly, handling latency issues in ML systems is critical for smooth user experiences. Some queries may take longer for models to process, especially in sequential tasks like language or time-series models. To address this, many companies implement backup systems that use simpler models, heuristics, or cached predictions to deliver quicker responses when the primary model is delayed. Alternatively, predictive models can determine which system to use based on the expected processing time. This approach balances the speed–accuracy trade-off, ensuring timely responses while maintaining acceptable prediction quality.
+
+By addressing these challenges—ensuring consistency, managing imperfect predictions, and mitigating latency—developers can leverage ML to enhance user experiences effectively.
+
+### Team structure
+
+**Cross-functional Teams Collaboration**  
+
+Subject matter experts (SMEs) like doctors, lawyers, and farmers play a vital role in the lifecycle of ML systems, beyond the initial data labeling phase. Their expertise is critical in problem formulation, feature engineering, error analysis, and system evaluation. However, collaboration between SMEs and ML engineers poses challenges, such as translating nuanced domain knowledge into code or versioning and enabling SMEs to contribute without needing engineering skills. To address this, companies are developing no-code/low-code platforms to empower SMEs to participate in tasks like dataset creation and quality assurance without extensive technical knowledge.
+
+**End-to-End Data Scientists**  
+
+In ML projects, operational expertise is as important as ML expertise, creating the need for effective collaboration between teams or for data scientists to own the entire process. Two approaches are common:  
+
+1. separate Ops and ML teams: the ML team develops models, and an Ops team handles deployment and maintenance. This simplifies hiring but introduces drawbacks like communication overhead, debugging difficulties, and limited visibility across the entire workflow. 
+2. data scientists owning the process: data scientists manage both modeling and production, fostering accountability but increasing their workload with operational tasks. They risk becoming "grumpy unicorns," burdened with tasks like managing infrastructure, which detracts from their core focus
+
+![image](https://github.com/user-attachments/assets/05be14a7-2d1c-4300-a95b-3a7ddc322c16)
+
+
+A balance can be struck by creating tools that abstract complex infrastructure tasks, allowing data scientists to manage projects end-to-end without deep expertise in areas like containerization or distributed systems. Companies like Netflix and Stitch Fix have demonstrated the effectiveness of tools that automate infrastructure management, enabling data scientists to focus on data and modeling while leveraging robust operational capabilities.
+
+Ultimately, successful ML systems require seamless collaboration between diverse expertise areas, supported by tools that simplify complex workflows, empowering teams to work efficiently and effectively.
+
+### Responsible AI
+
+- discovering sources of bias  
+  - biases can originate in training data, where unrepresentative datasets marginalize underrepresented groups.  
+  - human annotation can introduce subjective biases, compromising label quality.  
+  - feature engineering may rely on variables correlated with sensitive attributes, causing disparate impacts.  
+  - model objectives and evaluation often prioritize majority groups or overlook subgroup performance, leading to unfair outcomes.  
+
+- recognizing data-driven limitations  
+  - ml systems often fail to consider real-world socioeconomic and cultural factors.  
+  - addressing these limitations requires collaboration across disciplines to integrate the experiences of those impacted by the systems.  
+
+- managing trade-offs  
+  - privacy versus accuracy: techniques like differential privacy can reduce accuracy, disproportionately affecting minority groups.  
+  - compactness versus fairness: model compression techniques can harm underrepresented groups more, with varying impacts depending on the approach used.  
+  - trade-offs require careful evaluation to minimize unintended consequences and ensure fairness.  
+
+- acting early  
+  - addressing ethical issues early in the development lifecycle saves costs and prevents long-term risks.  
+  - late-stage fixes for bias or ethical concerns are significantly more expensive and challenging.  
+
+- using model cards for transparency  
+  - model cards document critical details about an ai system, including its design, intended uses, and limitations.  
+  - automated tools can generate and update model cards, reducing manual effort and ensuring up-to-date transparency.  
+
+- establishing bias mitigation processes  
+  - create internal tools for bias detection and mitigation, drawing inspiration from organizations like google and ibm.  
+  - consider using third-party audits to ensure independent evaluations.  
+
+Responsible ai development emphasizes proactive bias mitigation, thoughtful trade-offs, and transparent communication to maximize societal benefits while minimising harm.  
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From bd5f13b060078479f835c4eec27594e4878b4715 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 19 Nov 2024 00:56:46 +0900
Subject: [PATCH 501/638] Create 2024-11-18-day-322.md

---
 _posts/2024-11-18-day-322.md | 413 +++++++++++++++++++++++++++++++++++
 1 file changed, 413 insertions(+)
 create mode 100644 _posts/2024-11-18-day-322.md

diff --git a/_posts/2024-11-18-day-322.md b/_posts/2024-11-18-day-322.md
new file mode 100644
index 000000000..d2064a384
--- /dev/null
+++ b/_posts/2024-11-18-day-322.md
@@ -0,0 +1,413 @@
+---
+layout: post
+title: (Day 322) Data Eng camp homework 1 completed (maybe)
+categories: [data-eng,applying-knowledge]
+---
+
+# Hello :) Today is Day 322!
+A quick summary of today:
+* watched lab 2 - SCD modelling
+* finished the week 1 homework
+* trained a failing prompt tuned llm
+* streamed
+
+It is 1am and I am very sleepy so this will be short and sweet ~
+
+## [Data Modeling - Building Slowly Changing Dimensions (SCDs) - Day 2 Lab](https://youtu.be/nyu-8Si21ec)
+
+The lab 2 video was released today so I watched and took some notes.
+
+Here is my code from the lab:
+
+```sql
+create table players_scd (
+    player_name text,
+    scoring_class scoring_class,
+    is_active boolean,
+    -- for type 2 scd we need the two more cols
+    start_season integer,
+    end_season integer,
+    current_season integer,
+    primary key (player_name, start_season)
+);
+
+-- we want to calculate how many seasons they have been in a current dim
+-- we look at what was the dim before - we use lag
+-- the big expensive parts are the windows
+-- scale matters and something to always consider
+insert into players_scd
+with with_previous as (
+    select player_name,
+          current_season,
+          scoring_class,
+          is_active,
+          lag(scoring_class, 1)
+          over (partition by player_name order by current_season)                   as previous_scoring_class,
+          lag(is_active, 1) over (partition by player_name order by current_season) as previous_is_active
+   from players
+   where current_season <= 2022
+            ),
+    with_indicators as (
+        select *,
+               case
+                   when scoring_class <> previous_scoring_class then 1
+                   when is_active <> previous_is_active then 1
+                   else 0
+                end as change_indicator
+        from with_previous
+),
+    with_streaks as (
+        select *,
+            sum(change_indicator)
+                over (partition by player_name order by current_season) as streak_identifier
+        from with_indicators
+    )
+
+select player_name,
+       scoring_class,
+       is_active,
+       min(current_season) as start_season,
+       max(current_season) as end_season,
+       2022 as current_season
+from with_streaks
+group by player_name, streak_identifier, is_active, scoring_class
+order by player_name, streak_identifier
+
+
+create type scd_type as (
+    scoring_class scoring_class,
+    is_active boolean,
+    start_season integer,
+    end_season integer
+);
+
+-- see if things have changed
+-- this is incremental, the previous is all at once
+-- the prev adds all but last year, and this adds the last just for the sake of an example
+-- this is processing probably around 20x less data
+-- we assume scoring_class and is_active are never null
+with last_season_scd as (
+    select * from players_scd
+    where current_season = 2022
+    and end_season = 2022
+),
+    historical_scd as (
+        select
+            player_name,
+            scoring_class,
+            is_active,
+            start_season,
+            end_season
+        from players_scd
+        where current_season = 2022
+        and end_season < 2022
+    ),
+    this_seasons_data as (
+        select * from players
+        where current_season = 2023
+    ),
+    unchanged_records as (
+        select ts.player_name,
+               ts.scoring_class,
+               ts.is_active,
+               ls.start_season,
+               ts.current_season as end_season
+        from this_seasons_data ts
+            join last_season_scd ls
+            on ts.player_name = ls.player_name
+        where ts.scoring_class = ls.scoring_class
+        and ts.is_active = ls.is_active
+    ),
+    changed_records as (
+        select ts.player_name,
+               unnest(array[
+                   row(
+                       ls.scoring_class,
+                       ls.is_active,
+                       ls.start_season,
+                       ls.end_season
+                       )::scd_type,
+                   row( -- new record is just 1 value
+                       ts.scoring_class,
+                       ts.is_active,
+                       ts.current_season,
+                       ts.current_season
+                       )::scd_type
+                   ]) as records
+        from this_seasons_data ts
+            left join last_season_scd ls
+            on ts.player_name = ls.player_name
+        where (ts.scoring_class <> ls.scoring_class
+            or ts.is_active <> ls.is_active)
+    ),
+    unnested_chaged_records as (
+        select player_name,
+               (records::scd_type).scoring_class,
+               (records::scd_type).is_active,
+               (records::scd_type).start_season,
+               (records::scd_type).end_season
+        from changed_records
+    ),
+    new_records as (
+        select
+            ts.player_name,
+            ts.scoring_class,
+            ts.is_active,
+            ts.current_season as start_season,
+            ts.current_season as end_season
+        from this_seasons_data ts
+        left join last_season_scd ls
+            on ts.player_name = ls.player_name
+        where ls.player_name is null
+    )
+
+select * from historical_scd
+union all
+select * from unchanged_records
+union all
+select* from unnested_chaged_records
+union all
+select * from new_records;
+
+select * from players_scd
+```
+
+And then based on this I completed all the homework:
+
+```sql
+-- 1. DDL for actors table
+create type films as (
+    film text,
+    votes integer,
+    rating real,
+    filmid text
+);
+
+create type quality_class as enum ('star', 'good', 'average', 'bad');
+
+create table actors (
+    actorid text,
+    actor text,
+    films films[],
+    quality_class quality_class,
+    current_year integer,
+    is_active boolean,
+    primary key (actorid, current_year)
+);
+
+-- 2. Cumulative table generation query
+DO $$
+DECLARE
+    y_year INT; -- Previous year
+    t_year INT; -- Target year
+BEGIN
+    FOR y_year IN 1969..2020 LOOP
+        t_year := y_year + 1;
+        insert into actors
+        with yesterday as (
+            select *
+            from actors
+            where current_year = y_year
+        ),
+        today_aggregated as (
+            select
+                actorid,
+                max(actor) as actor,
+                array_agg(row(film, votes, rating, filmid)::films) as films,
+                max(year) as year,
+                avg(rating) as average_rating
+            from actor_films
+            where year = t_year
+            group by actorid
+        )
+        select
+            coalesce(t.actorid, y.actorid) as actorid,
+            coalesce(t.actor, y.actor) as actor,
+            case
+                when y.films is null then t.films
+                when t.films is not null then y.films || t.films
+                else y.films
+            end as films,
+            case
+                when t.films is not null then
+                    case when t.average_rating > 8 then 'star'
+                        when t.average_rating > 7 then 'good'
+                        when t.average_rating > 6 then 'average'
+                        else 'bad'
+                    end::quality_class
+                else y.quality_class
+            end as quality_class,
+            coalesce(t.year, y.current_year + 1) as current_year,
+            case when t.films is not null then true else false end as is_active
+        from today_aggregated t
+        full outer join yesterday y
+            on t.actorid = y.actorid;
+    END LOOP;
+END $$;
+
+-- 3. DDL for actors_history_scd table
+create table actors_history_scd (
+    actorid text,
+    quality_class quality_class,
+    is_active boolean,
+    start_date integer,
+    end_date integer,
+    current_year integer,
+    primary key (actorid, start_date)
+);
+
+-- 4. Backfill query for actors_history_scd
+insert into actors_history_scd
+with with_previous as (
+    select
+        actorid,
+        current_year,
+        quality_class,
+        is_active,
+        lag(quality_class, 1)
+            over (partition by actorid order by current_year) as previous_quality_class,
+        lag(is_active, 1)
+            over (partition by actorid order by current_year) as previous_is_active
+    from actors
+    where current_year <= 2020
+),
+    with_indicators as (
+        select *,
+            case
+                when quality_class <> previous_quality_class then 1
+                when is_active <> previous_is_active then 1
+                else 0
+            end as change_indicator
+        from with_previous
+    ),
+    with_streaks as (
+        select *,
+            sum(change_indicator)
+                over (partition by actorid order by current_year) as streak_identifier
+        from with_indicators
+    )
+
+select actorid,
+       quality_class,
+       is_active,
+       min(current_year) as start_date,
+       max(current_year) as end_date,
+       2020 as current_year
+from with_streaks
+group by actorid, streak_identifier, is_active, quality_class
+order by actorid, streak_identifier;
+
+-- 5. Incremental query for actors_history_scd
+create type scd_actors_type as (
+    quality_class quality_class,
+    is_active boolean,
+    start_date integer,
+    end_date integer
+);
+
+with last_year_scd as (
+    select * from actors_history_scd
+    where current_year = 2020
+    and end_date = 2020
+),
+    historical_scd as (
+        select
+            actorid,
+            quality_class,
+            is_active,
+            start_date,
+            end_date
+        from actors_history_scd
+        where current_year = 2020
+        and end_date < 2020
+    ),
+    this_year_data as (
+        select * from actors
+        where current_year = 2021
+    ),
+    unchanged_records as (
+        select ts.actorid,
+               ts.quality_class,
+               ts.is_active,
+               ls.start_date,
+               ts.current_year as end_date
+        from this_year_data ts
+            join last_year_scd ls
+            on ts.actorid = ls.actorid
+        where ts.quality_class = ls.quality_class
+        and ts.is_active = ls.is_active
+    ),
+    changed_records as (
+        select ts.actorid,
+               unnest(array[
+                   row(
+                       ls.quality_class,
+                       ls.is_active,
+                       ls.start_date,
+                       ls.end_date
+                       )::scd_actors_type,
+                   row( -- new record is just 1 value
+                       ts.quality_class,
+                       ts.is_active,
+                       ts.current_year,
+                       ts.current_year
+                       )::scd_actors_type
+                   ]) as records
+        from this_year_data ts
+            left join last_year_scd ls
+            on ts.actorid = ls.actorid
+        where (ts.quality_class <> ls.quality_class
+            or ts.is_active <> ls.is_active)
+    ),
+    unnested_chaged_records as (
+        select actorid,
+               (records::scd_actors_type).quality_class,
+               (records::scd_actors_type).is_active,
+               (records::scd_actors_type).start_date,
+               (records::scd_actors_type).end_date
+        from changed_records
+    ),
+    new_records as (
+        select
+            ts.actorid,
+            ts.quality_class,
+            ts.is_active,
+            ts.current_year as start_date,
+            ts.current_year as end_date
+        from this_year_data ts
+        left join last_year_scd ls
+            on ts.actorid = ls.actorid
+        where ls.actorid is null
+    ),
+    all_records as (
+        select * from (
+        select * from historical_scd
+        union all
+        select * from unchanged_records
+        union all
+        select * from unnested_chaged_records
+        union all
+        select * from new_records
+        ) a
+    )
+select *
+from all_records
+```
+
+I hung out in the discord looking at other people's questions and problems ~ 
+
+## I was in the lab today
+
+Today I figured out how to do prompt tuning and I used llama-3.2-1b and ran the training which took ~4 hours, but at the end it seems the model did not learn anything. I am suspecting because of the training arguments I passed. Before I left, I started a 9hr training and left it to go overnight so I will check the results tomorrow. 
+
+## [Streamed](https://www.youtube.com/watch?v=UmZOvOZDfGs)
+
+I watched 2 talks on how Riot Games does data engineering, and uses data to enhance player experience which were very nice (at least the 2nd one, the 1st one was a bit too vague and short). 
+
+Then I had a quick look at sktime - a library for time-series forecasting based on sklearn
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From de4a4b8b5b76617de47013e22d15018a8647c94d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 20 Nov 2024 00:15:59 +0900
Subject: [PATCH 502/638] Create 2024-11-19-day-323.md

---
 _posts/2024-11-19-day-323.md | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 _posts/2024-11-19-day-323.md

diff --git a/_posts/2024-11-19-day-323.md b/_posts/2024-11-19-day-323.md
new file mode 100644
index 000000000..70429238b
--- /dev/null
+++ b/_posts/2024-11-19-day-323.md
@@ -0,0 +1,36 @@
+---
+layout: post
+title: (Day 323) Predicting subway demand + Additive Dimensions
+categories: [data-eng,applying-knowledge,theory,traditional-machine-learning]
+---
+# Hello :) Today is Day 323!
+A quick summary of today:
+* new lecture on Additive dimensions and Graph Modelling
+* prompt tuning is taking soo long
+* trying to make a good model for predicting subway passengers 
+
+## Additive dimensions and Graph Modelling
+
+Today [Lecture 3](https://www.youtube.com/watch?v=XWaJFretFec) from Zach Wilson's camp was released, and here are my notes.
+
+![Additive dims and Graph data modelling day 3 P1](https://github.com/user-attachments/assets/9910e79b-71dc-4ca1-b925-1b9812d73a04)
+![Additive dims and Graph data modelling day 3 P2](https://github.com/user-attachments/assets/7c034863-b9b0-4251-869c-54ccc075fadc)
+![Additive dims and Graph data modelling day 3 P3](https://github.com/user-attachments/assets/be52df3b-e5db-471b-addd-d29bfff61a68)
+![Additive dims and Graph data modelling day 3 P4](https://github.com/user-attachments/assets/a3cd41e3-a124-4db4-a51e-1456b57046a1)
+
+## Prompt tuning
+
+The LLM I left training last night ... well it did not succeed. I wonder if the training epochs are too little - reason being the loss goes down consistently but is nowhere close to what I have seen with other models. But I am running 10 train epochs and its taking 9 hours... Am I going to train a model for 24 hours if I increase the epochs ? 😆  I left a model training tonight as well and we'll see in the morning but again I suspect the train epochs won't be enough. Of course the learning rate could be adjusted, and many other params but given my limited GPU resources... I have to make some educated guesses based on what I'm seeing. 
+
+## Predicting subway demand
+
+Today I saw that my university shared some small competition about predicting the passenger for every hour of a set of stations. I cannot join as the reward is a scholarship, and I am not eligible for a scholarship award (given I am already receiving a 100% one 😆). But I still wanted to check out the dataset and see how good of a model I can make. I spent most of the time preparing the data, hyperparam tuning, and cross validation with different split methods. [Here is a link to a colab notebook](https://colab.research.google.com/drive/1zgyiGfNwFXc5nwWaVLedMZG8VfXWKGy0?usp=sharing). I uploaded it so I can share it. However, at the moment I think the mean absolute error (the metric they say they will evaluate the results by) is a bit high ~ around 100, but there is some inconsistency because of the splitter I was using. I am writing this a bit early today (early meaning midnight haha) and I will continue running some code after I post this so hopefully I get something good tomorrow.
+
+Talking about time-series, given that subway passenger prediction is a great time-series prediction problem - [this week's probabl stream will be about time-series predictions](https://www.youtube.com/live/EnhyJx8l2LE?si=oHEREpc91cLP_aFw) and Vincent had asked if we had any questions/suggestions on what to show on stream so I asked to see how he would approach the problem using the library he will show. Then he requested if possible to find and put a dataset he could use (and try to make some kind of a showcase out of it) - [so here it is](https://github.com/divakaivan/seoul_subway_data). I found it from Seoul's official subway webpage and just translated some of the Korean to English and uploaded it as parquet as the csv sizes were >25mb. 
+
+
+--- 
+
+That is all for today!
+
+See you tomorrow :)

From acde63ffb17c4c1db6a10473223f75decd49ac4e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 20 Nov 2024 14:36:41 +0900
Subject: [PATCH 503/638] Update about.md

---
 _pages/about.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_pages/about.md b/_pages/about.md
index c24b5d56f..df5ab1e2c 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -12,6 +12,6 @@ On Day 100 I knew it was not over, so I continued.
 
 After a few days I decided to go for the full year.
 
-My studies have become a cover of math, stats, data engineering, classical machine learning, modern deep learning architectures, AI assistants, MLOps systems. So far this journey has been about the technical side of things, but crucially also about thinking as DataEng/DataScientist/MachineLearningEng/DataAnalyst and developing the soft skills needed for any of these roles. Also about learning how I learn, but others too which I will summarise by the end of this 1-year journey. 
+My studies have covered a range of topics, including mathematics, statistics, data engineering, classical machine learning, modern deep learning architectures, AI assistants, and MLOps systems. This journey has not only been about mastering the technical aspects but also about adopting the mindset of a Data Engineer, Data Scientist, Machine Learning Engineer, and Data Analyst, while developing the soft skills essential for these roles. Additionally, it has been a process of learning how I learn—and understanding how others learn—which I plan to summarize at the end of this 1-year journey.
 
-**These days I stream part of my daily studied on [my youtube channel](https://www.youtube.com/@ilearnthings123) - Monday to Thursday 10AM GMT+1/6PM KST**
+**These days I stream part of my daily studied on [my youtube channel](https://www.youtube.com/@ilearnthings123) - Monday to Thursday**

From 4b2ea61e247ceea2c910909b7b7438cd18618a52 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 20 Nov 2024 14:38:05 +0900
Subject: [PATCH 504/638] Update about.md

---
 _pages/about.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/about.md b/_pages/about.md
index df5ab1e2c..19a1b9366 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -14,4 +14,4 @@ After a few days I decided to go for the full year.
 
 My studies have covered a range of topics, including mathematics, statistics, data engineering, classical machine learning, modern deep learning architectures, AI assistants, and MLOps systems. This journey has not only been about mastering the technical aspects but also about adopting the mindset of a Data Engineer, Data Scientist, Machine Learning Engineer, and Data Analyst, while developing the soft skills essential for these roles. Additionally, it has been a process of learning how I learn—and understanding how others learn—which I plan to summarize at the end of this 1-year journey.
 
-**These days I stream part of my daily studied on [my youtube channel](https://www.youtube.com/@ilearnthings123) - Monday to Thursday**
+**These days I stream part of my daily studies on [my youtube channel](https://www.youtube.com/@ilearnthings123) - Monday to Thursday**

From 0cfaf34f9ba3238b8993675331664b90a6efdae9 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 21 Nov 2024 01:33:00 +0900
Subject: [PATCH 505/638] Create 2024-11-20-day-324.md

---
 _posts/2024-11-20-day-324.md | 298 +++++++++++++++++++++++++++++++++++
 1 file changed, 298 insertions(+)
 create mode 100644 _posts/2024-11-20-day-324.md

diff --git a/_posts/2024-11-20-day-324.md b/_posts/2024-11-20-day-324.md
new file mode 100644
index 000000000..54683c1c3
--- /dev/null
+++ b/_posts/2024-11-20-day-324.md
@@ -0,0 +1,298 @@
+---
+layout: post
+title: (Day 324) I secured a free spot at Zach Wilson's Jan 2025 data engineering bootcamp
+categories: [data-eng,applying-knowledge]
+---
+
+# Hello :) Today is Day 324!
+A quick summary of today:
+* 121 chat with Zach Wilson (trans-continental video chat)
+* first homework submitted
+* small company reviewer update
+* debugging sql
+* graph data modelling in postgres
+
+## I got a free spot at Zach Wilson's 2025 data engineering bootcamp 🤯
+
+This morning, at 6.30am, I went into a 30-min call with Zach. Why 6.30am well he is in Utah, US and I am in South Korea and as he is a busy person his schedule is fixed so I picked the latest possible time for me. Thankfully, I was not sleepy during the chat, he is very energetic and I got some of his energy 😆. 
+
+We had a chat about bootcamps, how the bootcamp is going, other bootcamps, what I'm doing, and also a bit about this blog. He said a lot of positive things about this blog and I appreciate that because this blog does not really reach a big audience and I am using it on my CV as well ~ so it was nice hearing a compliment about this blog journey :)
+
+Anyway ~ at the end, because I am so active on his free bootcamp's discord (helping others, having discussions about the material, etc.) he said he can offer me a free spot at his paid bootcamp in January 2025. I mean... it's like a dream. Learning from Zach Wilson's best content... really just speechless. There is also so much content on his platform and I will be able to learn sooo much from his bootcamp as well. Super excited for January 2025!!! I'm over the moon 🥳🥳🥳🥳🥳
+
+As there is other content on the platform, I will watch some of it before the bootcamp, HOWEVER, I will not talk about it at all - just like I did for Stanford's XCS ML for graphs course - it is behind a paywall and nothing can go out to the public. If I am watching some video about pyspark (for example) - I will just share 'I watched a video about pyspark' as I do not want to break the platform's policies. 
+
+## First homework for the free bootcamp submitted
+
+<img width="1083" alt="image" src="https://github.com/user-attachments/assets/9b811757-75e5-4c45-9253-ea20c1305151">
+
+Here is some feedback from the AI grader:
+
+<img width="742" alt="image" src="https://github.com/user-attachments/assets/e07a17b3-d163-4d72-994f-c3b906a47413">
+
+## Company reviewr LLM update
+
+<img width="421" alt="image" src="https://github.com/user-attachments/assets/22346000-f43e-499c-8814-fdc71254e129">
+
+All the above models use 15 train epochs. I decided to use the best of them and train it for 20 epochs (llama3.2-3b) but the result was bad, I guess it starts to overfit, and maybe 15 is the sweet spot, but maybe 10 epochs is worth exploring as well (of course other hyperparams need to be considered as well). Also I got an email today from openAI that I got access to their o1-preview and o1-mini models so I will be fine-tuning those as well soon-ish. 
+
+The prompt tuning failed again, so I looked online and found some example and I decided to completely 'send it' by using 50 training epochs. And the train time is 24hours 😆. If it overfits - fine, if it doesn't... well we go back to the drawing board.
+
+## Debugging SQL
+
+```sql
+insert into actors_test
+WITH years AS (
+    SELECT *
+    FROM GENERATE_SERIES(1970, 2020) AS year
+),first_year as(
+    select actor,actorid,min(year) as first_release_year
+    from public.actor_films 
+    group by actor,actorid
+),actor_all_film_release_years as(
+    select * from first_year fy
+    join years y 
+    on fy.first_release_year<=y.year
+), yearly_released_films as (
+    select afy.actor,
+    afy.actorid,
+    afy.year as current_year,
+    ARRAY_REMOVE(
+        ARRAY_AGG(
+            case when af.year is not null then 
+            row(af.film,
+                af.votes,
+                af.rating,
+                af.filmid)::films
+            end
+        )over(partition by afy.actorid 
+                  order by af.year
+         ),null) as films,
+
+    ROUND(
+        avg( case when af.year is not null then
+            af.rating
+           end
+       ) over(partition by afy.actorid 
+                  order by af.year)::NUMERIC,2) as avg_rating_year,
+
+    case when af.year is not null 
+        then True else False end as is_active
+    from actor_all_film_release_years afy 
+    left join actor_films af on
+    afy.actorid=af.actorid
+    and afy.year=af.year
+    order by afy.actorid,afy.year
+
+)
+select
+    DISTINCT
+    actorid,
+    actor,
+    films,
+    case when avg_rating_year >8 then 'star'
+    when avg_rating_year>7 and avg_rating_year<=8 then 'good'
+    when avg_rating_year >6 and avg_rating_year<=7  then 'average'
+    else 'bad' end::quality_class as quality_class,
+    current_year,
+    is_active
+
+from yearly_released_films
+order by actorid
+```
+
+So ... this query is used to populate an actors table in an 'efficient' way. The result being efficient bit is not the important part of this particular writing - it's about the query.
+
+Another student messaged me with a similar query and saying they get duplicate rows in some cases. I think debugging cases like this really helps me to understand the errors and what to expect from some queries. Tldr, the above query works, but the new thing compared to the query that *does not work* are the DISTINCT in the final select, and what took me the most time to grasp and understand - in the partitions using `order by af.year` vs `coalesce(afy.year,af.year)`. And I think it is because of the left join. When we do the left join, the af.year column is often not matched because afy.year is a generated series (from the start), and it may not align exactly with the actual years in af.year. Since afy.year is derived from the GENERATE_SERIES and is never NULL, the COALESCE(afy.year, af.year) always goes to afy.year. This results in incorrect ordering in the PARTITION BY clause because the actual release years (af.year) are ignored. Consequently, the calculated aggregates and film arrays do not correctly follow the chronological order of film releases.
+
+## [Lab 3 - graph data modelling](https://www.youtube.com/watch?v=8J1uOJI2HFE)
+
+I covered lab 3 today related to modelling data as nodes(or vertices, as they are called in DE, I guess) and edges.
+
+Here is my code from the lab
+
+```sql
+-- build a graph data model see which NBA players play with each other
+-- and which players play for what team at each time
+
+create type vertex_type as enum ('player', 'team', 'game');
+
+create table vertices (
+    identifier text,
+    type vertex_type,
+    properties json, -- postgres doesnt have MAP
+    primary key (identifier, type)
+);
+
+create type edge_type as
+    enum ('plays_against', 'shares_team', 'plays_in', 'plays_on');
+
+create table edges (
+    subject_identifier text,
+    subject_type vertex_type,
+    object_identifier text,
+    object_type vertex_type,
+    edge_type edge_type,
+    properties json,
+    -- specific id cols might be included and used for the PK
+    -- for this example this PK is fine
+    primary key (subject_identifier,
+                subject_type,
+                object_identifier,
+                object_type,
+                edge_type)
+);
+
+insert into vertices
+select
+    game_id as identifier,
+    'game'::vertex_type as type,
+    json_build_object(
+        'pts_home', pts_home,
+        'pts_away', pts_away,
+        'winning_team', case when home_team_wins = 1 then home_team_id else visitor_team_id end
+    ) as properties
+from games;
+
+insert into vertices
+with players_agg as (
+    select
+        player_id as identifier,
+        max(player_name) as player_name,
+        count(1) as number_of_games,
+        sum(pts) as total_points,
+        array_agg(distinct team_id) as teams
+    from game_details
+    group by 1
+)
+select identifier, 'player'::vertex_type,
+        json_build_object(
+            'player_name', player_name,
+            'number_of_games', number_of_games,
+            'total_points', total_points,
+            'teams', teams
+       )
+from players_agg;
+
+insert into vertices
+with teams_deduped as (
+    select *, row_number() over (partition by team_id) as row_num
+    from teams
+)
+select
+    team_id as identifier,
+    'team'::vertex_type as type,
+    json_build_object(
+        'abbreviation', abbreviation,
+        'nickname', nickname,
+        'city', city,
+        'arena', arena,
+        'year_founded', yearfounded
+    )
+from teams_deduped
+where row_num = 1;
+
+select type, count(1)
+from vertices
+group by 1
+-- 30 team, 9384 game, 1496 player
+
+insert into edges
+with deduped as (
+    select *, row_number() over (partition by player_id, game_id) as row_num
+    from game_details
+)
+select
+    player_id as subject_identifier,
+    'player'::vertex_type as subject_type,
+    game_id as object_identifier,
+    'game'::vertex_type as object_type,
+    'plays_in'::edge_type as edge_type,
+    json_build_object(
+        'start_position', start_position,
+        'pts', pts, -- this is treated as a str so later we need to convert it to int if we want to use it
+        'team_id', team_id,
+        'team_abbreviation', team_abbreviation
+    ) as properties
+from deduped
+where row_num = 1;
+
+select
+    v.properties->>'player_name',
+    max(cast(e.properties->>'pts' as integer))
+from vertices v
+    join edges e
+on e.subject_identifier = v.identifier
+and e.subject_type = v.type
+group by 1
+order by 2 desc;
+
+insert into edges
+with deduped as (
+    select *, row_number() over (partition by player_id, game_id) as row_num
+    from game_details
+),
+    filtered as (
+        select * from deduped
+                 where row_num = 1
+    ),
+    aggregated as (
+        select
+            f1.player_id as subject_player_id,
+            f2.player_id as object_player_id,
+            case when f1.team_abbreviation = f2.team_abbreviation
+                then 'shares_team'::edge_type
+                else 'plays_against'::edge_type
+            end as edge_type,
+            max(f1.player_name) as subject_player_name, -- maybe they changed their name
+            max(f2.player_name) as object_player_name,
+            count(1) as num_games,
+            sum(f1.pts) as subject_points,
+            sum(f2.pts) as object_points
+        from filtered f1 join filtered f2
+        on f1.game_id = f2.game_id
+        and f1.player_name <> f2.player_name
+        where f1.player_name > f2.player_name -- remove double edges
+        group by f1.player_id,
+            f2.player_id,
+            case when f1.team_abbreviation = f2.team_abbreviation
+                then 'shares_team'::edge_type
+                else 'plays_against'::edge_type
+            end
+    )
+select
+    subject_player_id as subject_identifier,
+    'player'::vertex_type as subject_type,
+    object_player_id as object_identifier,
+    'player'::vertex_type as object_type,
+    edge_type as edge_type,
+    json_build_object(
+        'num_games', num_games,
+        'subject_points', subject_points,
+        'object_points', object_points
+    )
+from aggregated;
+
+-- we can calculate avg points, points when X plays with Y
+-- or points when X plays vs Y, etc.
+select
+    v.properties->>'player_name',
+    e.object_identifier,
+    cast(v.properties->>'number_of_games' as real) /
+    case when cast(v.properties->>'total_points' as real) = 0 then 1
+        else cast(v.properties->>'total_points' as real) end,
+    e.properties->>'subject_points',
+    e.properties->>'num_games'
+
+from vertices v join edges e
+    on v.identifier = e.subject_identifier
+    and v.type = e.subject_type
+where e.object_type = 'player'::vertex_type
+```
+
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 193fc6d42ba68d37e157d2ff11ab883750b65b44 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 22 Nov 2024 01:31:08 +0900
Subject: [PATCH 506/638] Create 2024-11-21-day-325.md

---
 _posts/2024-11-21-day-325.md | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 _posts/2024-11-21-day-325.md

diff --git a/_posts/2024-11-21-day-325.md b/_posts/2024-11-21-day-325.md
new file mode 100644
index 000000000..6aa12de89
--- /dev/null
+++ b/_posts/2024-11-21-day-325.md
@@ -0,0 +1,30 @@
+---
+layout: post
+title: (Day 325) Dimension modelling 
+categories: [data-eng,applying-knowledge,nlp,theory]
+---
+
+# Hello :) Today is Day 325!
+A quick summary of today:
+* started watching the paid content on dataexpert.io (but cannot share as it's against the rules)
+* another failed prompt tuning attempt
+
+## Dimension modelling lectures + labs on the dataexpert.io platform
+
+I started watching the pre-recorded videos offered on the platform to start preparing for the Jan bootcamp. 
+
+As this is all behind paywall, nothing is available publicaly I can just share the topic and that I took notes. I think going forward I should do something alongside covering this content so that these posts are not just a few sentences long :/ 
+
+The platform is great ~ even for the exercises there is a built-in sql editor.
+
+I am yet to read the infamous Designing Data-Intensive Applications book so I think reading it before Jan would be extra helpful, and I can also share summaries of what I read while also covering the content on dataexpert.io.
+
+## 24 hour prompt tuning
+
+The 24hr company reviewer prompt tuning model ... failed :/ I checked it at around 6pm when it finished and it started at loss ~ 13, and flat-lined around 5-6 ... And this is a recurring thing I see with other failing training sessions - the end result is always around loss 5-6. So I am just starting to really wonder what to do. We have time, and having a successful prompt tuned model is not essential as the LoRA fine-tuned gpt4o-mini and llama3.2 are satisfactory, but for the sake of wider exploration I think it's worth doing some more exploration. I adjusted the learning rate and some other hyperparams and put another version to train which will take ~26 hours. (note that the GPU we have is a 48GB A6000) 
+
+---
+
+I guess ~ that is all for today!
+
+See you tomorrow :)

From 033cabe2f6ca2da12bd87ea64508903b50b176a2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 22 Nov 2024 22:48:34 +0900
Subject: [PATCH 507/638] Create 2024-11-22-day-326.md

---
 _posts/2024-11-22-day-326.md | 84 ++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 _posts/2024-11-22-day-326.md

diff --git a/_posts/2024-11-22-day-326.md b/_posts/2024-11-22-day-326.md
new file mode 100644
index 000000000..93ee16812
--- /dev/null
+++ b/_posts/2024-11-22-day-326.md
@@ -0,0 +1,84 @@
+---
+layout: post
+title: (Day 326) Finally started the Designing Data-Intensive Applications book
+categories: [data-eng,theory]
+---
+
+# Hello :) Today is Day 326!
+A quick summary of today:
+* Designing Data-Intensive Applications's audiobook
+* fact data modelling
+* some sql practice
+
+## Started listening to [Designing Data-Intensive Applications: The Big Ideas Behind Reliable, Scalable, and Maintainable Systems](https://learning.oreilly.com/videos/designing-data-intensive-applications/9781663728289/)
+
+I was not aware there were audiobooks on the O'Reilly platform.
+
+<img width="934" alt="image" src="https://github.com/user-attachments/assets/d86c35d0-2d81-4117-aae7-96ba9a95f3bb">
+
+## Chapter 1: Reliable, Scalable, and Maintainable Applications
+
+![image](https://github.com/user-attachments/assets/05ceedfa-544c-48b1-bd73-859409a73072)
+
+### Thinking About Data Systems
+
+The boundaries between traditional categories of data systems (databases, queues, caches) are becoming blurred as new tools optimised for diverse use cases have emerged. For instance, some datastores function as message queues (like redis), and some queues provide database-like durability (Kafka). Modern applications often require multiple tools stitched together via application code to meet complex data processing and storage needs, creating composite data systems.
+
+Designing such systems raises challenges around ensuring correctness, consistency, performance, scalability, and a good API. Various factors, including team expertise, legacy dependencies, and organisational risks, influence design decisions. 
+
+Key concerns for data systems include:
+
+1. reliability: ensuring correct functioning under adverse conditions
+2. scalability: managing growth in data, traffic, or complexity
+3. maintainability: enabling productive work on the system over time
+
+### Reliability
+
+Reliability refers to a system's ability to continue functioning correctly even when faults occur. Fault-tolerant or resilient systems anticipate faults—components deviating from specifications—and prevent them from causing failures, which occur when the system fails to deliver expected services. Key fault types include:
+
+* Hardware faults: common in large-scale systems, mitigated through redundancy (e.g., RAID, backup power, multi-machine setups). Modern demands often require software fault-tolerance to complement hardware redundancy
+* Software errors: often systematic, correlated faults such as bugs, resource exhaustion, or cascading failures. Mitigated by rigorous testing, process isolation, self-monitoring, and allowing systems to crash and restart
+* Human errors: a leading cause of outages. Mitigations include user-friendly design, sandbox environments, automated testing, detailed monitoring, and fast recovery mechanisms
+
+While reliability is essential for user trust and business continuity, trade-offs may occur for prototypes or low-margin services. However, even in non-critical systems, reliability remains a core responsibility to users
+
+### Scalability
+
+Scalability is a system's ability to handle increased load, requiring tailored approaches rather than a one-size-fits-all solution. Load growth can stem from more users, increased data, or changing usage patterns. Key considerations include defining load params (e.g.requests per second or database read/write ratios) and choosing architectures that address specific scalability needs. 
+
+Performance metrics like response time and latency are critical. Response time varies and is best described using percentiles (e.g., p50, p95) to understand typical and extreme cases. High percentiles, or tail latencies, significantly affect user experience, especially for multi-step backend processes. Systems must balance optimising for common use cases while addressing critical outliers (e.g. celebrity tweets on Twitter).
+
+To manage increasing load:
+
+1. scaling up (vertical): moving to a more powerful machine
+2. scaling out (horizontal): distributing load across multiple machines (shared-nothing architecture)
+
+Stateful distributed systems are complex but increasingly necessary for high loads. Elastic systems adjust resources dynamically, while manual scaling offers simplicity. Effective scalability designs rely on understanding specific load patterns, balancing performance, and iterative adaptability.
+
+### Maintainability
+
+Maintainability is crucial, as most software costs stem from ongoing maintenance rather than initial development. To avoid creating cumbersome legacy systems, software should prioritize operability, simplicity, and evolvability:  
+
+1. Operability: focuses on making it easy for operations teams to maintain system health, troubleshoot issues, apply updates, and anticipate problems. Systems should enable visibility, automation, good documentation, self-healing capabilities, and predictable behavior
+
+2. Simplicity: reduces unnecessary complexity, which can hinder understanding, increase maintenance costs, and risk bugs. Simplification often involves removing accidental complexity through effective abstractions that hide implementation details, making systems easier to maintain and reuse
+
+3. Evolvability: ensures systems can adapt to changing requirements, such as new features or regulatory demands. Evolvable systems leverage simplicity and clear abstractions to remain flexible and support larger-scale modifications, fostering long-term agility
+
+These principles interconnect and aim to minimise maintenance challenges and support the system’s longevity.
+
+## Fact data modelling
+
+Today I continued with the content on dataexpert.io - my preparation for the Jan '25 bootcamp. I covered lecture 1 only which is related to fact data modelling.
+
+(Again, I cannot and should not share more info as this content is 100% behind a paywall and none of it is available publicly)
+
+## Doing some SQL practice
+
+On the dataexpert.io platform there is a [/question](https://www.dataexpert.io/questions) page which has some practice SQL questions and built-in query editor. Today I finished all the easy ones and some of the medium ones. It's all free so why not take advantage :) 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 8dca56de6f39f333a9b9389629a02bed76f4a4d8 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 24 Nov 2024 01:34:08 +0900
Subject: [PATCH 508/638] Create 2024-11-23-day-327.md

---
 _posts/2024-11-23-day-327.md | 159 +++++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 _posts/2024-11-23-day-327.md

diff --git a/_posts/2024-11-23-day-327.md b/_posts/2024-11-23-day-327.md
new file mode 100644
index 000000000..e96881ece
--- /dev/null
+++ b/_posts/2024-11-23-day-327.md
@@ -0,0 +1,159 @@
+---
+layout: post
+title: (Day 327) More of the Designing Data-Intensive Applications book
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 327!
+A quick summary of today:
+* Designing Data-Intensive Applications: Chapter 2
+* some more sql practice
+
+## Designing Data-Intensive Applications: Chapter 2. Data Models and Query Languages
+
+I read and listened at the same time the book + audio version (this might be a viable learning strategy 😆), so here is a summary of ch. 2 ~
+
+### Relational Model vs. Document Model: A Comparative Overview  
+
+#### Relational model  
+
+- origin: introduced by edgar codd in 1970, gaining widespread adoption through rdbms implementations by the 1980s  
+- structure: organizes data in tables (rows and columns) with relationships defined via foreign keys  
+- strengths:  
+  - excellent support for joins and complex queries  
+  - enforces strict schemas, ensuring data consistency  
+  - broad adoption and a mature ecosystem  
+- use cases: ideal for transaction processing, batch processing, and general-purpose web applications  
+- challenges:  
+  - impedance mismatch with object-oriented programming  
+  - scalability concerns for massive datasets or high write loads  
+
+
+#### Document model  
+
+- origin: emerged during the nosql movement in the 2010s to address scalability and flexibility challenges  
+- structure: stores data in json, xml, or binary formats, allowing nested and flexible schemas  
+- strengths:  
+  - schema flexibility supports dynamic, evolving data structures  
+  - improved performance for read-heavy workloads due to data locality  
+  - simplifies application code for hierarchical or document-like data  
+- use cases: frequently used in web apps, content management systems, and applications with deeply nested data  
+- challenges:  
+  - limited support for joins and many-to-many relationships  
+  - lack of schema enforcement can result in inconsistencies  
+  - performance degrades with large, frequently updated documents  
+
+
+#### Key Concepts
+
+- object-relational mismatch: discrepancy between object-oriented programming and relational models, often addressed with ORM frameworks
+- normalization: ensures consistency and reduces redundancy but can complicate queries
+- joins: relational databases excel here, while document databases often struggle
+- schema flexibility: document databases allow schema-less or schema-on-read approaches, accommodating rapidly changing data needs
+
+#### Convergence of models 
+
+- relational databases adopting document features: many RDBMS now support JSON/XML, enabling hybrid approaches
+- document databases adding relational capabilities: some incorporate joins and relational features
+- future outlook: the convergence trend suggests databases will increasingly handle both structured and unstructured data seamlessly
+
+Choosing the right model should align with the application's needs, balancing scalability, flexibility, and query complexity.  
+
+Leveraging multiple database types to utilize their strengths in different parts of an application ensures optimal performance and flexibility.  
+
+### Query languages for data
+
+When relational databases emerged, SQL introduced a declarative approach to querying data, in contrast to imperative methods used by systems like IMS and CODASYL.  
+
+**Key differences between Declarative and Imperative queries**  
+
+- **Imperative programming** involves specifying the exact steps to achieve a result. For example, filtering a list in JavaScript would require loops and conditionals
+- **Declarative programming**, as seen in SQL, focuses on describing the desired outcome without detailing how to achieve it. The database handles the execution details
+
+Declarative queries offer significant benefits:  
+
+- conciseness and ease of understanding
+- independence from implementation details, enabling performance optimizations
+- support for parallel execution, leveraging modern multi-core processors effectively
+
+#### Declarative Queries Beyond Databases  
+
+Declarative principles extend beyond databases to web technologies like CSS. Styling a web page declaratively with CSS is simpler and more efficient compared to manipulating the DOM imperatively with JavaScript. For example:  
+
+- a CSS rule applies and updates styles dynamically as elements change 
+- imperative JavaScript styling can result in errors, such as failing to update styles when conditions change
+
+#### MapReduce and mixed paradigms  
+
+MapReduce represents a blend of declarative and imperative approaches. It is widely used for distributed data processing, including in NoSQL databases like MongoDB.  
+
+- **Declarative elements**: filters (e.g. `family: "Sharks"`) specify which data to process
+- **Imperative elements**: custom JavaScript functions define the map and reduce operations  
+
+Example MapReduce Workflow:  
+
+1. **Mapping**: emits key-value pairs for relevant data points
+2. **Reducing**: aggregates values for each key, such as summing the counts of observed shark sightings by month
+
+Challenges with MapReduce include complexity in coordinating map and reduce functions and limited query optimization opportunities. To address this, MongoDB introduced the Aggregation Pipeline, a declarative alternative that simplifies query construction while maintaining flexibility.  
+
+The progression from MapReduce to declarative aggregation pipelines demonstrates how declarative languages often evolve to balance expressiveness and usability.
+
+### Graph-like data models
+
+Graph databases excel at handling complex many-to-many relationships, making them ideal for scenarios like social networks, web links, or road networks where relational databases struggle.
+
+#### Key Features
+
+- structure: data is represented as vertices (nodes) connected by edges (relationships), mimicking real-world connections
+- schema flexibility: graph databases allow for dynamic connections without rigid schema constraints, unlike relational databases
+- efficient traversal: they support seamless navigation through data, making them efficient for analyzing interconnected relationships
+
+#### Graph Database Models
+
+1. Property graph model:  
+   - stores properties as key-value pairs on vertices and edges
+   - examples: Neo4j, Titan, InfiniteGraph
+2. Triple-store model:  
+   - stores data as subject-predicate-object triples, akin to RDF
+   - examples: Datomic, AllegroGraph
+
+#### Query Languages
+
+- Cypher: intuitive, arrow-based syntax for property graphs (Neo4j)
+- SPARQL: concise querying for triple-stores, heavily used in RDF 
+- Datalog: rule-based, logic-oriented querying for complex derivations
+
+#### Comparison with CODASYL
+
+Graph databases improve upon the network model (CODASYL) by:
+
+- offering schema flexibility for connections
+- enabling direct vertex access via unique IDs
+- supporting unordered relationships, simplifying data handling
+- utilizing declarative query languages for easier queries
+
+#### RDF and the Semantic Web
+- RDF triples underpin the semantic web, aiming for a machine-readable internet
+- despite potential for data integration across sites, its adoption remains limited
+
+Overall, graph databases shine when dealing with highly interconnected data that demands efficient traversal and dynamic schema adaptability, making them invaluable for modern, relationship-centric applications.
+
+## [Zach Wilson Q&A stream](https://youtu.be/IG1_nTJGH-8)
+
+I saw that Zach did a live Q&A stream. I did not join as it was 5am or something like that for me haha but thankfully the vod is on youtube. My main takeaway is related to showcasing DE projects and the questions we need to answer when building one:
+
+1. whats the impact of the project? how does it change lives?
+2. what is the robustness of the project? how are you going to improve it?
+3. what is the quality of the project? are we doing quality checks?tests? is it written with code that’s readable? is it documented?
+4. proof - is it visible?
+
+## Completed the medium practice SQL questions as well
+
+[Here](https://www.dataexpert.io/questions) is the link to the practice Qs. Today I finished all medium ones as well and only the hard ones are left. The main things of the medium Qs were window functions, group bys, CTEs and joins. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 731ce1748ec65b90ed76439a99adb054e29b7f57 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 25 Nov 2024 02:00:34 +0900
Subject: [PATCH 509/638] Create 2024-11-24-day-328.md

---
 _posts/2024-11-24-day-328.md | 289 +++++++++++++++++++++++++++++++++++
 1 file changed, 289 insertions(+)
 create mode 100644 _posts/2024-11-24-day-328.md

diff --git a/_posts/2024-11-24-day-328.md b/_posts/2024-11-24-day-328.md
new file mode 100644
index 000000000..32844165a
--- /dev/null
+++ b/_posts/2024-11-24-day-328.md
@@ -0,0 +1,289 @@
+---
+layout: post
+title: (Day 328) Week 2 - Fact Data Modeling
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 328!
+A quick summary of today:
+* learned a tonne about fact data modelling 😆
+
+## [Week 2 Fact data modelling](https://www.youtube.com/watch?v=DQefW9sNmw0&t=1067s)
+
+Today Zach Wilson released the week 2 content in one 4 hour video. 
+
+There are in total 3 lectures and 3 labs. I watched the lecture versions in the dataexpert.io platform which are from the v3 version and updated so unfortunately I cannot share my notes from them. In order to share something here, I followed along the 3 labs.
+
+The videos were all amazing, and here is my code with some small comments from the 3 labs
+
+### Lab 1 - making a fact table
+
+```sql
+-- fact data modelling lab 1
+
+-- check if game_id, team_id, player_id are unique
+-- we find there is (almost) 2 of every record
+select
+    game_id, team_id, player_id, count(1)
+from game_details
+group by game_id, team_id, player_id
+having count(1) > 1;
+
+-- Thing to remember when doing fact data modelling
+-- are the cols we are giving even useful?
+-- don't put any derived values in the fact table, let the analyst do w/e they want
+-- when logging we might get double data
+-- so it's common to dedupe as the 1st step
+insert into fct_game_details
+with deduped as (
+    select
+        g.game_date_est,
+        g.season,
+        g.home_team_id,
+        gd.*,
+        row_number() over (partition by gd.game_id, team_id, player_id order by g.game_date_est) as row_num
+    from game_details gd join games g on gd.game_id = g.game_id
+)
+select
+    -- all these cols are a fundamental nature of the fact
+    -- we might want to put all IDs first then dim them m
+    game_date_est as dim_game_date,
+    season as dim_season,
+    team_id as dim_team_id,
+    player_id as dim_player_id,
+    player_name as dim_player_name,
+    start_position as dim_start_position,
+    team_id = home_team_id as dim_is_playing_at_home,
+    coalesce(position('DNP' in comment), 0) > 0 as dim_did_not_play,
+    coalesce(position('DND' in comment), 0) > 0 as dim_did_not_dress,
+    coalesce(position('NWT' in comment), 0) > 0 as dim_not_with_team,
+--     comment -- we can remove it if we are confident that we have parsed all options
+    cast(split_part(min, ':', 1) as real)
+        + cast(split_part(min, ':', 2) as real)/60 as m_minutes,
+    fgm as m_fgm,
+    fga as m_fga,
+    fg3m as m_fg3m,
+    fg3a as m_fg3a,
+    ftm as m_ftm,
+    fta as m_fta,
+    oreb as m_oreb,
+    dreb as m_dreb,
+    reb as m_reb,
+    ast as m_ast,
+    stl as m_stl,
+    blk as m_blk,
+    "TO" as m_turnovers,
+    pf as m_pf,
+    pts as m_pts,
+    plus_minus as m_plus_minus
+from deduped
+where row_num = 1;
+
+create table fct_game_details (
+    -- dim_ are cols that you should group by and filter on
+    dim_game_date date,
+    dim_season integer,
+    dim_team_id integer,
+    dim_player_id integer,
+    dim_player_name text,
+    dim_start_position text,
+    dim_is_playing_at_home boolean,
+    dim_did_not_play boolean,
+    dim_did_not_dress boolean,
+    dim_not_with_team boolean,
+    -- measures -> m_
+    -- m_ are cols that you should agg and to math
+    m_minutes real,
+    m_fgm integer,
+    m_fga integer,
+    m_fg3m integer,
+    m_fg3a integer,
+    m_ftm integer,
+    m_fta integer,
+    m_oreb integer,
+    m_dreb integer,
+    m_reb integer,
+    m_ast integer,
+    m_stl integer,
+    m_blk integer,
+    m_turnovers integer,
+    m_pf integer,
+    m_pts integer,
+    m_plus_minus integer,
+    -- the PK helps create indexes which is good if we do filtering on cols in the PK
+    primary key (dim_game_date, dim_team_id, dim_player_id)
+);
+
+-- we may have lost the team info but we can easily bring them in
+select t.*, gd.*
+from fct_game_details gd join teams t
+on t.team_id = gd.dim_team_id;
+
+-- lets find the players that bailed out on the most games
+-- fact data modelling is about making tables that are easy to query to create cool things
+select dim_player_name,
+       count(1) as num_games,
+       count(case when dim_not_with_team then 1 end) as bailed_num,
+       cast(count(case when dim_not_with_team then 1 end) as real)/count(1) as bail_pct
+from fct_game_details
+group by 1
+order by 4 desc;
+```
+
+### Lab 2 - Learning about DateList 🤯
+
+```sql
+-- week 2 lab 2
+
+create table users_cumulated (
+    user_id text,
+    -- lists of dates in the past where the user was active
+    dates_active date[],
+    -- current date for the user
+    date date,
+    primary key (user_id, date)
+);
+
+insert into users_cumulated
+with yesterday as (
+    select *
+    from users_cumulated
+    where date = date('2023-01-30')
+), today as (
+    select
+        cast(user_id as text) as user_id,
+        date(cast(event_time as timestamp)) as date_active
+    from events
+    where
+        date(cast(event_time as timestamp)) = date('2023-01-31')
+        and user_id is not null -- deal with null user_ids in this data
+    group by user_id, date(cast(event_time as timestamp))
+)
+select
+    coalesce(t.user_id, y.user_id) as user_id,
+    case
+        when y.dates_active is null then array[t.date_active]
+        when t.date_active is null then y.dates_active -- we don't want to keep adding a big array of nulls
+        else array[t.date_active] || y.dates_active
+    end as dates_active,
+    -- today's date_active might not be date if the user doesn't exist yet
+    -- so we add 1 to yesterday's
+    coalesce(t.date_active, y.date + interval '1 day') as date
+from today t full outer join yesterday y
+on t.user_id = y.user_id;
+
+
+-- generate a datelist for 30 days
+with users as (
+    select * from users_cumulated
+    where date = date('2023-01-31')
+),
+    series as (
+        select * from generate_series(date('2023-01-01'), date('2023-01-31'), interval '1 day') as series_date
+    ),
+    place_holder_ints as (
+        select
+            case
+                when dates_active @> array[date(series_date)]
+                    -- date - series_date is # of days b/e current date and series date
+-- if we cast a power of 2 number as bits and turn it into binary
+-- then we can get a history of 1s and 0s active/inactive
+                    then cast(pow(2, 32 - (date - date(series_date))) as bigint)
+                    else 0
+                end as placeholder_int_value,
+            *
+        from users cross join series -- we got the 31 days for each user
+    )
+select
+    user_id,
+    -- these are extremely efficient operations
+    -- bit_count() can give us how many times the user is active
+    bit_count(cast(cast(sum(placeholder_int_value) as bigint) as bit(32))) > 0 as dim_is_monthly_active,
+    -- let's check a user is active in the last 7 days
+    bit_count(cast('11111110000000000000000000000000' as bit(32)) & --bit-wise and
+        cast(cast(sum(placeholder_int_value) as bigint) as bit(32))) > 0 as dim_is_weekly_active,
+    -- daily is the same but with the 1st one only 1
+    bit_count(cast('10000000000000000000000000000000' as bit(32)) & --bit-wise and
+        cast(cast(sum(placeholder_int_value) as bigint) as bit(32))) > 0 as dim_is_daily_active
+from place_holder_ints
+group by user_id;
+```
+
+### Lab 3 - Reducing fact data
+
+```sql
+-- week 2 lab 3
+
+create table array_metrics (
+    user_id numeric,
+    month_start date,
+    metric_name text,
+    metric_array real[],
+    primary key (user_id, month_start, metric_name)
+);
+
+insert into array_metrics
+with daily_aggregate as (
+    select
+        user_id,
+        date(event_time) as date,
+        count(1) as num_site_hits
+    from events
+    where date(event_time) = date('2023-01-31')
+    and user_id is not null
+    group by user_id, date(event_time)
+),
+    yesterday_array as (
+        select *
+        from array_metrics
+        where month_start = date('2023-01-01')
+    )
+
+select
+    coalesce(da.user_id, ya.user_id) as user_id,
+    coalesce(ya.month_start, date_trunc('month', da.date)) as month_start,
+    'site_hits' as metric_name,
+    case
+        -- this is reverse order from day 2's lab
+        -- we use coalesce and put a 0 if we are not fine with having null
+        when ya.metric_array is not null then ya.metric_array || array[coalesce(da.num_site_hits, 0)]
+        -- deal with cases where a new user shows up after the 1st of the month, so we add 0s at the start and then append the array
+        -- we add the coalesce because that array cannot accept null values, but also if either of them is null then we just don't fill
+        -- because we don't need to fill because that means it's the 1st day of the month
+        when ya.metric_array is null then array_fill(0, array[coalesce(date - date(date_trunc('month', date)), 0)]) || array[coalesce(da.num_site_hits, 0)]
+    end as metric_array
+from daily_aggregate da full outer join yesterday_array ya
+on da.user_id = ya.user_id
+on conflict (user_id, month_start, metric_name)
+do
+    update set metric_array = excluded.metric_array;
+
+
+-- we can do N day analysis
+-- going from monthly array metrics to daily aggregates
+-- but it's very fast as it's the minimal set of data we need
+with agg as (
+    select metric_name,
+        month_start,
+        array [
+            sum(metric_array[1]),
+            sum(metric_array[2]),
+            sum(metric_array[3]),
+            sum(metric_array[4])
+        ] as summed_array
+     from array_metrics
+     group by metric_name, month_start
+)
+select metric_name,
+       month_start + cast(cast(index-1 as text) || 'day' as interval),
+       elem as value
+from agg
+    cross join unnest(agg.summed_array)
+        with ordinality as a(elem, index)
+```
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 730ab009abf85bd4041ba7999c6a3eed43329ea5 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 26 Nov 2024 00:50:51 +0900
Subject: [PATCH 510/638] Create 2024-11-25-day-329.md

---
 _posts/2024-11-25-day-329.md | 352 +++++++++++++++++++++++++++++++++++
 1 file changed, 352 insertions(+)
 create mode 100644 _posts/2024-11-25-day-329.md

diff --git a/_posts/2024-11-25-day-329.md b/_posts/2024-11-25-day-329.md
new file mode 100644
index 000000000..445fbd68e
--- /dev/null
+++ b/_posts/2024-11-25-day-329.md
@@ -0,0 +1,352 @@
+---
+layout: post
+title: (Day 329) Fact Data Modeling homework
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 329!
+A quick summary of today:
+* read Designing data-intensive applications - chapter 3
+* watched a bit more of the dataexpert.io paid content
+* did the homework (but am still not 100% confident on the validity of my solution)
+
+## Designing data-intensive applications - chapter 3 - storage and retrieval
+
+### Data structures that power your db
+
+#### basics of databases
+- key-value stores are simple database structures similar to dictionaries with db_set to store key-value pairs and db_get to retrieve them  
+- appending key-value pairs to a file is easy for writing but slows down reading as the file size increases  
+
+#### Indexing and optimization
+- indexes improve read speeds but add overhead to writes and should be chosen based on query patterns  
+- hash index uses a hash map for fast lookups, requiring enough memory to store the map  
+- segmented files divide data into manageable pieces, with compaction and merging reducing redundancy and optimizing storage  
+
+#### Advanced index structures
+- sstables store sorted key-value pairs for efficient merging, minimal in-memory index needs, and compression support  
+- data is first written to an in-memory balanced tree (memtable) and then persisted as sstable segments  
+- lsm-trees are based on sstables, offering high write throughput with sequential writes and crash safety using write-ahead logs  
+- b-trees organize data into balanced structures for predictable o(log n) lookups, using write-ahead logs to prevent corruption during updates  
+
+#### Trade-offs between lsm-trees and b-trees
+- lsm-trees are faster for writes but compaction can interfere with performance  
+- b-trees provide predictable reads and are better for transactional consistency but write slower  
+
+#### Specialized indexes
+- secondary indexes enable queries on non-unique values, useful for relational joins  
+- clustered and covering indexes store data within or alongside the index for faster reads but increase write overhead  
+- multi-column indexes support multi-field queries using concatenated or multi-dimensional approaches  
+- full-text search handles fuzzy queries with techniques like edit distance and automata  
+
+#### In-memory databases
+- store the entire dataset in ram for speed, with durability through snapshots, logging, or replication  
+- performance comes from optimized data structures and avoiding disk i/o  
+- anti-caching extends in-memory models to handle datasets larger than available memory, similar to virtual memory systems  
+
+### Transaction processing or analytics?
+
+<img width="806" alt="image" src="https://github.com/user-attachments/assets/17c6fe74-616b-41d9-9c3a-3e5c59432013">
+
+#### Transaction processing (OLTP)  
+- focuses on low-latency reads and writes for real-time updates  
+- typical operations: looking up small records by key and modifying them based on user input  
+- primarily used by end users for current data states (e.g., sales or orders)  
+- dataset size: gigabytes to terabytes  
+
+#### Analytics (OLAP)  
+- scans large datasets to calculate aggregates like sums or averages  
+- queries analyze historical events and support decision-making  
+- mainly used by analysts, with datasets ranging from terabytes to petabytes  
+
+#### Divergence of OLTP and OLAP  
+- oltp systems prioritize availability and low latency but struggle with resource-heavy analytic queries  
+- separate data warehouses emerged for analytics to avoid impacting oltp performance  
+
+#### Data warehousing  
+- uses extract-transform-load (etl) to consolidate and clean data from various oltp systems  
+- optimized for analytic patterns like bulk reads and aggregations  
+- schema types:  
+  - **star schema**: central fact table linked to dimension tables for simplicity  
+  - **snowflake schema**: normalized dimensions for complex hierarchies  
+
+#### Tools and systems  
+- modern data warehouses include commercial systems (e.g., teradata, redshift) and open-source platforms (e.g., hive, presto)  
+- some databases support both oltp and olap but often separate their engines for efficiency  
+
+### Column-oriented storage
+
+### Column-Oriented Storage: Optimizing Data Warehouses
+
+Column-oriented storage is crucial for efficiently handling massive datasets, common in data warehouses with trillions of rows and petabytes of data. Unlike traditional row-oriented storage, which stores all attributes of a row together, columnar storage organizes data by columns. This structure brings significant benefits for analytics, where queries typically focus on a subset of columns.
+
+It allows queries to load only the relevant columns, reducing unnecessary disk I/O. It also facilitates compression, as columns often contain repetitive values. For instance, bitmap encoding compresses data by representing values as compact bitmaps, while run-length encoding further optimizes storage, especially for sorted columns.
+
+Sorting plays a key role in column stores. Organizing data by frequently queried columns, like dates or product IDs, improves query performance and enhances compression. Some systems replicate datasets sorted differently across machines to optimize for diverse query patterns.
+
+However, columnar storage poses challenges for writes. Updating compressed, sorted columns is computationally expensive. Systems like Vertica address this by temporarily storing writes in memory and periodically merging them with on-disk data.
+
+## Data accessibility and Airflow
+
+I covered 2 lectures + 2 labs on the dataexper.io platform related to the above two topics. As usual, I cannot share anything about it or from. I can just share that it's amazing 😆
+
+## Fact Data Modelling homework
+
+After watching all the lectures + labs yesterday from Zach Wilson's free youtube bootcamp related to fact data modelling I decided to try to do the homework today.
+
+Surprisingly it was more straightforward than expected. Maybe the fact that I did hw1 and all the sql practice got me rolling. Here are my solutions *at the moment*. I am not 100% sure they are correct but I am getting similar results with 1-2 other people on the community discord.
+
+```sql
+-- 1. A query to deduplicate game_details from Day 1 so there's no duplicates
+
+with deduped_game_details as (
+    select *,
+           row_number() over (partition by game_id, team_id, player_id) as row_num
+    from game_details
+)
+select
+    *
+from deduped_game_details;
+```
+
+---
+
+```sql
+-- 2. A DDL for a user_devices_cumulated table
+
+create table user_devices_cumulated (
+    user_id numeric,
+    device_activity date[],
+    browser_type text,
+    date date,
+    primary key (user_id, browser_type, date)
+);
+```
+
+---
+
+```sql
+-- 3. A cumulative query to generate device_activity_datelist from events
+
+insert into user_devices_cumulated
+with yesterday as (
+    select *
+    from user_devices_cumulated
+    where date = date('2023-01-30')
+), today as (
+    select
+        user_id,
+        browser_type,
+        date(cast(event_time as timestamp)) as date_active
+    from events left join devices
+        on events.device_id = devices.device_id
+    where
+        date(cast(event_time as timestamp)) = date('2023-01-31')
+        and user_id is not null
+        and devices.browser_type is not null
+    group by user_id, date(cast(event_time as timestamp)), browser_type
+)
+select
+    coalesce(t.user_id, y.user_id) as user_id,
+    case
+        when y.device_activity is null then array[t.date_active]
+        when t.date_active is null then y.device_activity
+        else array[t.date_active] || y.device_activity
+    end as device_activity,
+    coalesce(y.browser_type, t.browser_type) as browser_type,
+    coalesce(t.date_active, y.date + interval '1 day') as date
+from today t full outer join yesterday y
+on t.user_id = y.user_id and t.browser_type = y.browser_type;
+
+select count(1)
+from user_devices_cumulated
+```
+
+---
+
+```sql
+-- 4. A datelist_int generation query.
+-- Convert the device_activity_datelist column into a datelist_int column
+
+with users as (
+    select * from user_devices_cumulated
+    where date = date('2023-01-31')
+),
+    series as (
+        select * from generate_series(date('2023-01-01'), date('2023-01-31'), interval '1 day') as series_date
+    ),
+    place_holder_ints as (
+        select
+            case
+                when device_activity @> array[date(series_date)]
+                    then cast(pow(2, 31 - (date - date(series_date))) as bigint)
+                    else 0
+                end as placeholder_int_value,
+            *
+        from users cross join series
+    )
+SELECT user_id,
+       browser_type,
+       cast(cast(sum(placeholder_int_value) as bigint) as bit(32)) as datelist_int_binary,
+       bit_count(cast(cast(sum(placeholder_int_value) as bigint) as bit(32))) as cnt_active_dates,
+       cast(sum(placeholder_int_value) as bigint) as datelist_int,
+       date
+FROM place_holder_ints
+where user_id = '444502572952128450'
+GROUP BY user_id, browser_type, date
+
+-- 31,30,29,28,27,25,24,19,18,17,16,15,14,13,12,11,7,6,5,4,3,2
+-- select * from user_devices_cumulated
+-- where user_id = '444502572952128450'
+-- and date = date('2023-01-31')
+```
+
+---
+
+```sql
+-- 5. A DDL for hosts_cumulated table
+-- a host_activity_datelist which logs to see which dates each host is experiencing any activity
+
+create table hosts_cumulated (
+    host text,
+    host_activity_datelist date[],
+    today_date date,
+    PRIMARY KEY (host, today_date)
+);
+```
+
+---
+
+```sql
+-- 6. The incremental query to generate host_activity_datelist
+
+insert into hosts_cumulated
+with yesterday as (
+    select *
+    from hosts_cumulated
+    where today_date = date('2023-01-30')
+),
+    today as (
+        select
+            host,
+            cast(cast(event_time as timestamp) as date) as today_date
+        from events
+        where cast(cast(event_time as timestamp) as date) = date('2023-01-31')
+        and host is not null
+        group by host, cast(cast(event_time as timestamp) as date)
+    )
+select
+    coalesce(y.host, t.host) as host,
+    case
+        when y.host_activity_datelist is null then array[t.today_date]
+        when t.today_date is not null then array[t.today_date] || y.host_activity_datelist
+        else y.host_activity_datelist
+    end as host_activity_datelist,
+    coalesce(t.today_date, y.today_date + interval '1 day') as today_date
+from yesterday y full outer join today t
+on y.host = t.host;
+```
+
+---
+
+```sql
+-- 7. A monthly, reduced fact table DDL host_activity_reduced
+
+create table host_activity_reduced (
+    month date,
+    host text,
+    hit_metric text,
+    hit_array integer[],
+    unique_visitors_metric text,
+    unique_visitors_array numeric[],
+    primary key (month, host, hit_metric, unique_visitors_metric)
+);
+```
+
+---
+
+```sql
+-- 8. An incremental query that loads host_activity_reduced
+
+insert into host_activity_reduced
+with daily_agg as (
+    select
+        host,
+        date(event_time) as date,
+        count(1) as num_site_hits,
+        COUNT(DISTINCT user_id) as unique_visitors
+    from events
+    where date(event_time) = date('2023-01-31')
+        and user_id is not null
+    group by host, date(event_time)
+),
+    yesterday_array as (
+        select *
+        from host_activity_reduced
+        where month = date('2023-01-01') -- hardcoded
+    )
+select
+    coalesce(ya.month, date_trunc('month', da.date)) as month,
+    coalesce(da.host, ya.host) as host,
+    'site_hits' as hit_metric,
+    case
+        when ya.hit_array is not null then ya.hit_array || array[coalesce(da.num_site_hits, 0)]
+        when ya.hit_array is null then array_fill(0, array[coalesce(date - date(date_trunc('month', date)), 0)]) || array[coalesce(da.num_site_hits, 0)]
+    end as hit_array,
+    'unique_visitors' as unique_visitors_metric,
+    case
+        when ya.unique_visitors_array is not null then ya.unique_visitors_array || array[coalesce(da.unique_visitors, 0)]
+        when ya.unique_visitors_array is null then array_fill(0, array[coalesce(date - date(date_trunc('month', date)), 0)]) || array[coalesce(da.unique_visitors, 0)]
+    end as unique_visitors_array
+from daily_agg da full outer join yesterday_array ya
+on da.host = ya.host
+on conflict (month, host, hit_metric, unique_visitors_metric)
+do
+    update set hit_array = excluded.hit_array,
+               unique_visitors_array = excluded.unique_visitors_array;
+
+-- not part of the exercise just for fun
+with agg as (
+    select
+        host,
+        month,
+        array [
+            sum(hit_array[1]),
+            sum(hit_array[2]),
+            sum(hit_array[3])
+        ] as avg_hit_array,
+        array [
+            avg(unique_visitors_array[1]),
+            avg(unique_visitors_array[2]),
+            avg(unique_visitors_array[3])
+        ] as avg_unique_visitors_array
+    from host_activity_reduced
+    group by host, month
+),
+daily_records as (
+    select
+        host,
+        month,
+        generate_series(1, array_length(avg_hit_array, 1)) as day_offset,
+        avg_hit_array,
+        avg_unique_visitors_array
+    from agg
+),
+unnested as (
+    select
+        dr.host,
+        dr.month + (dr.day_offset - 1) * interval '1 day' as date,
+        dr.avg_hit_array[dr.day_offset] as hit_value,
+        dr.avg_unique_visitors_array[dr.day_offset] as unique_visitor_value
+    from daily_records dr
+)
+select *
+from unnested
+order by host, date;
+```
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 8ca5cf8b8782d3526aee3137afd41eb054bd35e1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 27 Nov 2024 01:57:37 +0900
Subject: [PATCH 511/638] Create 2024-11-26-day-330.md

---
 _posts/2024-11-26-day-330.md | 161 +++++++++++++++++++++++++++++++++++
 1 file changed, 161 insertions(+)
 create mode 100644 _posts/2024-11-26-day-330.md

diff --git a/_posts/2024-11-26-day-330.md b/_posts/2024-11-26-day-330.md
new file mode 100644
index 000000000..b3ac33bb0
--- /dev/null
+++ b/_posts/2024-11-26-day-330.md
@@ -0,0 +1,161 @@
+---
+layout: post
+title: (Day 330) Who are the top 10 NBA players by consecutive 20+ point seasons?
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 330!
+A quick summary of today:
+* applying some of my SCD knowledge
+* learning more on the dataexpert.io platform
+* got A on the 2nd homework for the free DE bootcamp
+* read/listened to chapter 4 of Designing Data-Intensive Applications
+
+## [Doing Hard SQL questions](https://www.dataexpert.io/questions)
+
+Today I decided to try to finish the Qs on the dataexpert.io platform. All the Qs are completely free and there's also a built-in sql editor which is amazing.
+
+I just want to mention the very last Q and that it feels so good knowing how to solving it. And I learned how to solve it thanks to week 1 of Zach Wilson's free DE bootcamp - related to slowly changing dimensions
+
+Here is the question:
+
+```
+Who are the top 10 NBA players by consecutive 20+ point seasons?
+ 
+Using the table bootcamp.nba_player_seasons, find the most consecutive seasons a player has scored 20+ points and find the top 10 players! Make sure to handle ties correctly (you might have more than 10 records!). Sort the output data set by consecutive_seasons descending and player_name ascending!
+
+These are the tables to query for this question:
+bootcamp.nba_player_seasons
+player_name string
+age int
+height string
+weight int
+college string
+country string
+draft_year string
+draft_round string
+draft_number string
+gp double
+pts double
+reb double
+ast double
+netrtg double
+oreb_pct double
+dreb_pct double
+usg_pct double
+ts_pct double
+ast_pct double
+season int
+Your answer should include these columns:
+player_name varchar
+consecutive_seasons integer
+```
+
+And here is my solution:
+
+```sql
+with player_20pt_seasons as (
+    select
+        player_name,
+        season,
+        case when pts >= 20 then 1 else 0 end as has_20pts
+    from bootcamp.nba_player_seasons
+),
+prev_season as (
+    select
+        player_name,
+        season,
+        has_20pts,
+        lag(has_20pts, 1) over (partition by player_name order by season) as previous_season_has_20pts
+    from player_20pt_seasons
+),
+indicators as (
+    select *,
+        case
+            when has_20pts <> previous_season_has_20pts then 1
+            else 0
+        end as change_ind
+    from prev_season
+),
+streaks as (
+    select *,
+        sum(change_ind) over (partition by player_name order by season) as streak_identifier
+    from indicators
+),
+almost_final as (
+    select 
+        player_name,
+        has_20pts,
+        min(season) as start_season,
+        max(season) as end_season
+    from streaks
+    group by player_name, streak_identifier, has_20pts
+    order by player_name, streak_identifier
+),
+ranked_results as (
+    select
+        player_name,
+        end_season + 1 - start_season as consecutive_seasons,
+        rank() over (order by end_season + 1 - start_season desc) as rank
+    from almost_final
+    where has_20pts = 1
+)
+select
+    player_name,
+    consecutive_seasons
+from ranked_results
+where rank <= 10
+order by rank, player_name
+```
+
+## Got A on the 2nd homework on fact data modelling
+
+<img width="1425" alt="image" src="https://github.com/user-attachments/assets/5670c691-23d9-4cf4-87a6-dee05e5805c6">
+
+Submitted it today and got an immediate answer since it's AI graded ^^
+
+## I keep watching videos on the dataexpert.io platform
+
+Just mentioning them and that they are related to Airflow/dbt/Spark and are awesome 😆 but cannot share anything as they are completely non-public content.
+
+## Designing Data-Intensive Applications - Chapter 4 - Encoding and Evolution
+
+### Formats for encoding data
+
+Binary encoding formats like Thrift, Protocol Buffers, and Avro provide efficient and flexible alternatives to textual formats like JSON, XML, and CSV, especially when handling schema evolution—the inevitable changes in data structure over time.
+
+Textual formats, while popular, have notable shortcomings. JSON struggles with ambiguity in encoding numbers, often leading to precision issues with large numbers. XML and JSON offer optional and complex schema support, which is frequently ignored, while CSV lacks schema support entirely, making data interpretation more error-prone and labor-intensive. Additionally, handling binary data in these formats necessitates Base64 encoding, inflating data size and increasing complexity.
+
+In contrast, binary formats are more compact and enforce schema use, which doubles as documentation. They enable schema evolution by allowing compatibility checks and ensuring consistency in data interpretation. Thrift and Protocol Buffers achieve this through field tags—unique numbers assigned to fields in the schema. These tags allow for adding or removing optional fields without breaking compatibility. However, care must be taken when changing data types or field tags, as these can introduce risks such as data truncation.
+
+Avro, unlike Thrift and Protocol Buffers, doesn't use field tags. Instead, it employs the writer's schema and reader's schema, which allow encoding and decoding with differing but compatible schemas. This approach supports adding or removing fields with default values, converting compatible data types, and dynamic schema generation from sources like relational databases. Avro’s flexibility extends to typed languages, offering optional code generation while remaining highly usable in dynamic environments.
+
+The use of schemas in binary formats ensures compact data encoding, consistent documentation, and robust compatibility checks. These features provide stronger data guarantees and better tooling, making binary formats an appealing choice for managing evolving data structures in modern applications.
+
+### Modes of Dataflow
+
+#### Dataflow through dbs
+
+- databases act as a shared communication medium, requiring backward compatibility for decoding older data and forward compatibility for concurrent applications using different schema versions  
+- preserving unknown fields ensures data integrity when older applications access newer data. formats like avro handle this effectively, though application-level precautions may still be necessary  
+- since data outlives code, databases must manage schema evolution without rewriting existing data. archival storage, such as data dumps, can use the latest schema for consistency, especially in analytics  
+
+#### Dataflow through services: REST and RPC
+
+- services enable client-server communication, often using http-based rest apis for simplicity, resource identification, and microservices integration  
+- soap, a more complex xml-based protocol, has declined in popularity due to its tooling requirements and interoperability issues  
+- rpc (remote procedure call) frameworks like grpc address network complexities with features like futures and streams. however, restful apis remain preferred for their ease of use and platform support  
+- ensuring backward compatibility for requests and forward compatibility for responses is vital for service evolvability. api versioning is critical, especially for inter-organizational communication, often requiring multiple active api versions  
+
+#### Message-passing Dataflow
+
+- asynchronous message-passing systems, relying on a message broker, facilitate decoupled, one-way communication with features like buffering and message redelivery  
+- responses in message-passing can be achieved through separate channels, supporting use cases where direct replies are unnecessary  
+- distributed actor frameworks use message-passing to scale applications across nodes, with frameworks like akka and orleans handling message encoding compatibility during rolling upgrades  
+- these systems balance flexibility and reliability, making them well-suited for evolving, distributed applications
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From fc8427683375c140e8fb686ebdee6a713cb3f1c4 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 28 Nov 2024 00:11:47 +0900
Subject: [PATCH 512/638] Create 2024-11-27-day-331.md

---
 _posts/2024-11-27-day-331.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 _posts/2024-11-27-day-331.md

diff --git a/_posts/2024-11-27-day-331.md b/_posts/2024-11-27-day-331.md
new file mode 100644
index 000000000..d6d017896
--- /dev/null
+++ b/_posts/2024-11-27-day-331.md
@@ -0,0 +1,32 @@
+---
+layout: post
+title: (Day 331) More SQL practice before the Jan bootcamp
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 331!
+A quick summary of today:
+* watching more of the pre-recorded content on dataexpert.io
+* doing more SQL exercises
+
+## Spark + analytical partterns content
+
+Today I covered lectures + labs on the above two topics. As always, I cannot share my notes or anything related as it's content 100% behind the wall on dataexpert.io. 
+
+## More SQL practice
+
+After finishing the SQL practice Qs on the dataexpert platform yesterday, I started looking for other websites with SQL practice questions. I considered the obvious ones - hackerrank and datacamp. But then I asked a person who I know works as a DE and is experienced with SQL about their opinion on a good website (the person is George from the Zach Wilson community server). George suggested [interviewquery](https://www.interviewquery.com/questions) so I signed up for a month (using the blackfriday discount 😆) and started from easy to hard Qs again.
+
+Today I solved:
+
+<img width="273" alt="image" src="https://github.com/user-attachments/assets/4660e0e3-7dec-453f-a880-e2f6f552dd0c">
+
+(somehow I set my weekly goal as 25 questions 😆)
+
+21 Qs. I was always selecting next Q after finishing one and somehow I solved a medium Q but I checked and there are like 40 more easy Qs (60 in total), and then I will move into Medium (which seem to be the most) and finally Hard.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From a6daa4d62a8569abadc03c6054c6b7b12f8c8ec4 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 29 Nov 2024 00:19:35 +0900
Subject: [PATCH 513/638] Create 2024-11-28-day-332.md

---
 _posts/2024-11-28-day-332.md | 170 +++++++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)
 create mode 100644 _posts/2024-11-28-day-332.md

diff --git a/_posts/2024-11-28-day-332.md b/_posts/2024-11-28-day-332.md
new file mode 100644
index 000000000..183446cb6
--- /dev/null
+++ b/_posts/2024-11-28-day-332.md
@@ -0,0 +1,170 @@
+---
+layout: post
+title: (Day 332) Have to do projects + network
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 332!
+A quick summary of today:
+* read and listen to Designing Data-Intensive Applications - Chapter 5 - Replication
+* listened to a data careers podcast by Joe Reis
+* more SQL practice
+
+## Designing Data-Intensive Applications - Chapter 5 - Replication
+
+_Replication means keeping a copy of the same data on multiple machines that are connected via a network._
+
+### Leaders and Followers
+
+<img width="820" alt="image" src="https://github.com/user-attachments/assets/63e3dfc3-5e32-4c34-95a9-f997f6b6db93">
+
+#### Leader-based replication overview
+In leader-based replication, one replica (the leader) handles all write operations and updates followers through a replication log. Followers can only serve read requests. This setup is common in relational databases like PostgreSQL, MySQL, and SQL Server, as well as non-relational systems like MongoDB and Kafka.
+
+#### Sync vs. Async replication
+
+- sync: ensures followers have up-to-date data, improving durability but increasing the risk of delays if followers are unresponsive
+- async: allows faster writes by not waiting for follower confirmation, but data loss is possible if the leader fails before replication completes
+
+#### Setting up followers
+
+Followers are initialized using a snapshot of the leader's data, followed by replaying all changes since the snapshot. This process ensures followers eventually catch up without downtime.
+
+#### Handling Failures
+- follower failures: resolved through 'catch-up recovery' where the follower syncs missed changes after reconnecting
+- leader failures: require failover, promoting a follower to leader and reconfiguring clients. Challenges include avoiding data loss and preventing "split brain" (two active leaders)
+
+#### Replication Methods
+
+1. statement-based: sends SQL statements to followers but can fail with nondeterministic or dependent operations
+2. Write-Ahead Log (WAL) shipping: transmits low-level storage changes but ties replication to storage engine internals
+3. logical log replication: uses higher-level, row-based logs, decoupling replication from storage formats and supporting more flexible upgrades
+
+Leader-based replication balances performance, durability, and consistency but requires careful handling of failures and trade-offs between synchronous and asynchronous modes.
+
+### Problems with Replication Lag
+
+Replication in distributed systems is crucial for fault tolerance, scalability, and reducing latency. However, when using asynchronous replication, **replication lag** can introduce inconsistencies. Here are three key issues and potential solutions:
+
+#### Reading your own writes
+
+Problem: Users may not see their updates immediately after submitting them, leading to frustration. 
+
+Solution:
+- read recent writes from the leader while routing other reads to followers
+- use timestamps to ensure replicas serving reads reflect the user's most recent updates
+- implement cross-device consistency by centralizing metadata and routing all user requests to the same datacenter
+
+#### Monotonic reads
+
+Problem: Users may observe inconsistent states (e.g. seeing newer data followed by older data).  
+
+Solution:
+
+- ensure users consistently read from the same replica
+- reroute to a fallback replica only if the primary replica fails
+
+#### Consistent prefix reads
+
+Problem: Causal relationships between updates may appear out of order.  
+
+Solution:
+
+- write causally related updates to the same partition
+- use algorithms that track and maintain causal dependencies
+
+#### General recommendations  
+
+If replication lag severely impacts user experience:  
+
+- design the system to offer stronger guarantees, like **read-after-write consistency**
+- simplify application logic by leveraging transactional guarantees when possible
+
+While asynchronous replication is scalable, pretending it behaves like synchronous replication can lead to subtle, hard-to-debug issues.
+
+### Multi-Leader Replication
+
+This enables multiple nodes to accept writes, addressing some limitations of single-leader systems but introducing complexities. in this setup, each leader acts as a follower to others, forwarding changes to maintain consistency. this approach suits certain use cases but comes with challenges
+
+- multi-datacenter operation: having leaders in different datacenters improves performance and fault tolerance. writes are processed locally and replicated asynchronously, minimizing inter-datacenter latency. however, handling concurrent changes in different datacenters can lead to conflicts
+
+- offline operation: systems like mobile apps rely on asynchronous synchronization, allowing local writes while disconnected. conflicts must be resolved when syncing
+
+- collaborative editing: real-time applications, like google docs, use techniques resembling multi-leader replication. conflict resolution is crucial when multiple users edit the same content simultaneously
+
+- conflict resolution: conflicts occur when concurrent changes affect the same data. solutions include:
+  - prioritizing changes by timestamps or replica ids
+  - merging conflicting values
+  - recording conflicts for later resolution, either automatically or through user input
+
+- topologies: multi-leader systems use communication paths like all-to-all, circular, or star configurations. each has trade-offs in fault tolerance and replication speed
+
+Multi-leader replication offers flexibility for distributed systems but requires careful handling of conflicts and topology design to ensure reliability and consistency.
+
+### Leaderless replication
+
+#### A high-availability strategy
+
+Leaderless replication, also known as Dynamo-style replication, allows any replica in a distributed system to accept write requests. Unlike leader-based replication, where a single leader dictates write order, this approach prioritizes high availability and resilience to node failures. However, it requires careful handling of concurrent writes and the potential for reading stale data.
+
+#### How leaderless replication handles node outages
+
+Leaderless systems operate without a designated leader, enabling any replica to process writes. When nodes are offline—for example, during maintenance—leaderless replication remains operational. Clients send writes to all replicas, and the system deems a write successful even if some replicas are unreachable.
+
+This flexibility, however, introduces challenges. An unavailable node may miss updates while offline, leading to stale data when it rejoins the network. To address this, read requests often query multiple replicas, comparing version numbers to identify and repair outdated values, a process called read repair.
+
+#### Quorums: balancing consistency and availability
+
+Leaderless systems rely on quorum-based mechanisms to ensure consistency amidst node failures. Quorums are defined using three parameters:
+
+- n: total number of replicas
+- w: minimum number of writes needed for a successful write
+- r: minimum number of reads needed for a successful read
+
+The relationship w + r > n ensures that at least one of the read replicas contains the most recent data. This relationship balances consistency and availability, with common configurations like n = 3, w = 2, r = 2. Adjusting 'w' or 'r' can prioritize performance or consistency. For instance, reducing 'w' or 'r' improves speed but increases the risk of stale reads
+
+#### Limitations of quorum consistency
+
+While quorums reduce the chances of stale data, they do not eliminate them entirely. Key challenges include:
+
+- sloppy quorums: during network disruptions, writes may temporarily store on nodes outside the primary replica set, delaying updates
+- concurrent writes: simultaneous writes to the same key by multiple clients can lead to conflicting version
+- edge cases: timing issues during node failures, write retries, or network delays may also result in stale values
+
+Leaderless systems are thus best suited for scenarios where eventual consistency is acceptable, ensuring data convergence over time.
+
+#### Resolving concurrent writes in leaderless systems
+
+Concurrent writes are a major challenge, as they can arrive at replicas in different orders due to network delays. Common resolution strategies include:
+
+- Last Write Wins (LWW): using timestamps to determine the latest write, though this can lead to data loss
+- merging values: for instance, merging shopping cart updates from multiple replicas by taking their union. However, this requires handling deletions carefully, often using tombstones, markers indicating removed items
+
+#### Version vectors for conflict management
+
+To manage concurrent writes without a central leader, leaderless systems use version vectors. These track version numbers for each key across all replicas, helping differentiate between overwrites and concurrent writes. Advanced techniques like dotted version vectors, as seen in Riak, are particularly effective.
+
+By using version vectors, systems can safely read from one replica and write to another, with no risk of data loss when conflicts are correctly resolved. For automation, Conflict-Free Replicated Data Types (CRDTs) are often employed to merge data without requiring application logic.
+
+## [Jobs in data with Joe Reis podcast](https://open.spotify.com/episode/4AqFYdfZZXjATSUpQkMJcN?si=81TlysvDSE6A7RO28WFCfA&context=spotify%3Ashow%3A3mcKitYGS4VMG2eHd2PfDN&nd=1&dlsi=d4e55b7f2eef496a&flow_ctx=d8fc5088-51da-4049-bc11-a5d69e70281b%3A1732804720)
+
+I found this podcast episode that got released yesterday(?) where Joe Reis talks with a guest on careers in data and the data job market. My main takeaways:
+
+* do projects to learn rather than thinking of doing them just to be parts of your cv
+* network
+
+## More SQL practice on interviewquery.com
+
+<img width="282" alt="image" src="https://github.com/user-attachments/assets/0c47968a-d9f2-439d-8e28-cc6d29563a99">
+
+Today I finished the last 15 or so easy Qs (total is 40) and moved onto the medium ones of which I did 5
+
+<img width="1120" alt="image" src="https://github.com/user-attachments/assets/ade579c9-9bb4-4b43-bbc5-942ac6ec4ce0">
+
+So far it's just practicing group bys, CTEs, window functions, and general logic for problems. But the medium ones is where I will most likely learn a tonne.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 16de22f6ab521a837cc219ab9dd854d1f2cb145c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 30 Nov 2024 00:49:56 +0900
Subject: [PATCH 514/638] Create 2024-11-29-day-333.md

---
 _posts/2024-11-29-day-333.md | 385 +++++++++++++++++++++++++++++++++++
 1 file changed, 385 insertions(+)
 create mode 100644 _posts/2024-11-29-day-333.md

diff --git a/_posts/2024-11-29-day-333.md b/_posts/2024-11-29-day-333.md
new file mode 100644
index 000000000..b2b474e04
--- /dev/null
+++ b/_posts/2024-11-29-day-333.md
@@ -0,0 +1,385 @@
+---
+layout: post
+title: (Day 333) Learning about Data Quality
+categories: [data-eng,theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 333!
+A quick summary of today:
+* watched some pydata talks
+* started reading the DQ Fundamentals book
+* some more SQL practice
+
+## [Mojtaba Farmanbar - Uncertainty quantification: How much can you trust your machine learning model?](https://youtu.be/sKGtZyVyMM0)
+
+Mojtaba Farmanbar discussed uncertainty quantification in ML models and how to make models more trustworthy for human interaction. He talked about the importance of quantifying uncertainty in situations where the model's decisions are crucial, such as financial crime detection and medical diagnosis. Data uncertainty can arise from issues like noisy data, disagreement between labellers, natural variability in data collection, or overlapping feature spaces. Model uncertainty can be due to limited data, model architecture choices, or assumptions made within the model.
+
+Two methods for measuring uncertainty:
+
+* intrinsic methods, like Bayesian approaches and quantile regression, inherently provide uncertainty quantification
+* post-hoc methods, like calibration and conformal prediction, are applied after training and don't restrict model choice
+
+He also mentions calibrations and that a well-calibrated model with a score of 0.8 for a positive class implies an 80% chance of the prediction being correct.
+
+He introduces conformal prediction as a framework to transform uncertainty into a rigorous form. Instead of point estimates, conformal prediction provides prediction sets (for classification) or prediction intervals (for regression). This approach offers coverage guarantee, ensuring that the prediction set or interval contains the true outcome with a specified probability. It's also model-agnostic and distribution-free.
+
+## [Vincent D. Warmerdam - Run a benchmark they said. It will be fun they said](https://youtu.be/-86xSqag6VA)
+
+Inspired by scikit-learn discussions about the "table vectorizer," Vincent sought to evaluate its universality across task types (classification, regression, time series).
+
+**Findings:**
+
+ - default ensemble models (e.g. CatBoost, LightGBM, XGBoost) showed similar accuracy but CatBoost had longer training times
+ - memory usage was comparable across models
+
+**Hyperparameter optimisation insights:**
+
+   - avoid overfitting to noise or random seeds by leveraging visual tools like HiPlot
+   - prefer random search over grid search for efficiency
+   - use cross-validation carefully, as fold choices can skew performance perceptions
+
+**Lessons learned:**
+
+   - focus on real-world problems, not just benchmarks
+   - prioritise data quality over excessive reliance on metrics
+   - embrace noise and use robust visualisation and resampling techniques
+
+## Data Quality Fundamentals - Chapter 1 - Why Data Quality Deserves Attention—Now
+
+### What is data quality?
+
+To book defines it as the health of data at any stage in its life cycle. Data quality can be impacted at any stage of the data pipeline, before ingestion, in production, or even during analysis.
+
+DQ might not be so popular with companies compared to ML, analytics, or DS, so most times DQ falls onto the shoulders of data analysts or engineers to handle it.
+
+Data quality issues (or, data downtime) are practically unavoidable given the rate of growth and data consumption of most companies. But by understanding how we define data quality, it becomes much easier to measure and prevent it from causing issues downstream.
+
+### Framing the current moment
+
+DQ has become a more popular topic since 2019 and teams have started using DevOps practices in the data world - culmunating in the term DataOps. 
+
+#### **Understanding the 'rise of data downtime'**
+
+Data downtime refers to periods of time where data is missing, inaccurate, or otherwise erroneous, and it manifests in stale dashboards, inaccurate reports, and even poor decision making.
+
+**Migration to the cloud**
+
+In many ways, the cloud makes data easier to manage, more accessible to a wider variety of users, and far faster to process.
+
+Not long after data warehouses moved to the cloud, so too did data lakes (a place to transform and store unstructured data), giving data teams even greater flexibility when it comes to managing their data assets. As companies and their data moved to the cloud, analytics-based decision making (and the need for high-quality data) became a greater priority for businesses.
+
+**More data sources**
+
+Nowadays, companies use anywhere from dozens to hundreds of internal and external data sources to produce analytics and ML models. Any one of these sources can change in unexpected ways and without notice, compromising the data the company uses to make decisions.
+
+**Increasingly complex data pipelines**
+
+Data pipelines have become increasingly complex with multiple stages of processing and nontrivial dependencies between various data assets as a result of more advanced (and disparate) tooling, more data sources, and increasing diligence afforded to data by executive leadership
+
+**More specialised data teams**
+
+As companies increasingly rely on data to drive smart decision making, they are hiring more and more data analysts, data scientists, and data engineers to build and maintain the data pipelines, analytics, and ML models that power their services and products, as well as their business operations. With all of these different users touching the data, miscommunication or insufficient coordination is inevitable and will cause these complex systems to break as changes are made.
+
+**Decentralised data teams**
+
+For instance, a 200-person company may support a team of 3 data engineers and 10 data analysts, with analysts distributed across functional teams to better support the needs of the business. These analysts will report to operational teams or centralized data teams but own specific data sets and reporting functions. Multiple domains will generate and leverage data, leading to the inevitability that data sets used by multiple teams become duplicated, go missing, or go stale over time. 
+
+#### Other industry trends contributing to the current moment
+
+1. Data mesh:  
+
+The data mesh is a decentralized, domain-oriented data architecture inspired by microservices. It treats data as a product, with each domain managing its own pipelines while adhering to universal governance standards. Organizations like Intuit and JPMorgan Chase have adopted this paradigm to improve data ownership, visibility, and quality. Success depends on reliable data through rigorous testing, monitoring, and observability.  
+
+![image](https://github.com/user-attachments/assets/78deb4de-1b22-4dc0-a0c1-408f7948b63d)
+
+2. Streaming Data:  
+
+Streaming data enables real-time insights by processing continuous data flows, unlike traditional batch data processing. This shift increases the speed of insights but introduces new challenges for ensuring data quality in motion.  
+
+3. Rise of the data lakehouse:  
+
+Data lakehouses combine the structured data capabilities of warehouses with the unstructured flexibility of data lakes. This model supports growing business needs and simplifies complex pipelines. Providers like Redshift Spectrum and Databricks Lakehouse blur the lines between lakes and warehouses, enabling a hybrid approach but adding complexity to data quality management.  
+
+## Data Quality Fundamentals - Chapter 2 - Assembling the Building Blocks of a Reliable Data System
+
+### Understanding the difference between Operational and Analytical data
+
+**Operational data**
+
+Data produced operationally—that is, by the day-to-day ongoing operations at your organization. Inventory snapshots at moments in time, customer impressions, and transaction records are all examples of operational data.
+
+**Analytical data**
+
+Data used analytically. This is to say, it’s the type of data behind data-driven business decisions. Marketing churn, clickthrough rates, and impressions by global region are all examples of analytical data categories.
+
+### What makes them different?
+
+![image](https://github.com/user-attachments/assets/50095c30-921e-4f2d-bcfc-01c27edefdc2)
+
+Almost always, operational data appears upstream from analytical data in your data pipelines. This is because analytical data can and often does contain aggregations or augmentations of operational datastores.
+
+### Data warehouses vs. Data lakes
+
+Generally, data warehouses store data in a structured (row-column) format. Such data is highly transformed (the result of defined preprocessing procedures) and present only because it has a determinate reason for being there—at least, in theory.
+
+Data lakes by contrast store anything—structured data, semi-structured data, unstructured data. Unlike warehouses, data lakes need not have highly specified procedures through which data enters—you can dump any format you like into a lake and directly access it. The result is a system that is typically higher volume and often more complex in terms of its governance and data.
+
+#### Data warehouses
+
+Data warehouses require 'schema on write' access, meaning we set the structure of the data at the instant it enters the warehouse. Further transformations of this data must make its new structure explicit at every step.
+
+The common ones are: Amazon redshift, Google BigQuery, and Snowflake
+
+#### Data lakes
+
+Data lakes also rely on high-quality analytical data to deliver optimal results. Unlike data warehouses, data lake architectures permit 'schema on read' access. This means we infer the structure of the data when we’re ready to use it.
+
+Very early data lakes were built primarily on Apache Hadoop MapReduce and HDFS, leveraging Apache Hive to query their data with a SQL engine. In the early 2010s, Apache Spark made data lakes much more tenable, providing a generalized framework for distributed computations across large data sets in the data lake.
+
+Some common features of data lakes include:
+
+* decoupled storage and compute
+* customization and interoperability
+* largely built on open source technologies
+* ability to handle unstructured or weakly structured data
+* support for sophisticated non-SQL programming models
+
+#### Data lakehouses?
+
+Data lakehouses first came onto the scene when cloud warehouse providers began adding features that offer lake-style benefits, such as Redshift Spectrum or Delta Lake. Similarly, data lakes have been adding technologies that offer warehouse-style features, such as SQL functionality and schema. 
+
+#### Syncing between warehouses and lakes
+
+Different data warehouses and lakes are bridged by a data integration layer. Data integration tools, such as AWS Glue, Fivetran, and Matillion, collect data from disparate sources, unify this data, and transform it into an upstream source. A classic use case of data integration would be to collect lake data and load it in a structured format into one’s data warehouse.
+
+Extract-transform-load, or ETL, is one well-known process within data integration. ETL generally describes integration steps where data is first extracted from one or more data stores, transformed into a new structure or format, and finally loaded into a destination data store
+
+### Collecting DQ metrics
+
+#### What are DQ metrics?
+
+In assessing whether your data is down, we might build a list of questions:
+
+* is the data up-to-date?
+* is the data complete?
+* are fields within expected ranges?
+* is the null rate higher or lower than it should be?
+* has the schema changed?
+
+This list is a good place to start
+
+#### How to pull DQ metrics
+
+1. scalability challenges: managing large datasets involves batching, query optimization, deduplication, schema normalization, and scalable storage. A dedicated data pipeline is essential for long-term operation
+
+2. monitoring across the data stack: true data observability extends beyond warehouses to include data lakes, dashboards, and ML models. Integrated solutions help ensure reliability across all assets
+
+3. plling metrics from Snowflake:  
+   - step 1: inventory mapping: use SQL queries to list and analyze database schemas, tables, and metadata for tracking changes over time. Include views and external tables for completeness  
+   - step 2: monitor data freshness and volume: track row counts, data size, and timestamps of the latest updates to identify anomalies or missing data  
+   - step 3: query history: analyze query logs to trace table usage, data lineage, and optimize costs. Include logs for data load operations to understand data movement and freshness  
+   - step 4: health checks: evaluate key metrics like completeness, distinctness, and distribution of both string and numeric fields. Identify issues such as missing data, duplicates, or format mismatches
+
+4. practical insights:  
+   - automate monitoring and alert systems for fast issue resolution
+   - dentralize data quality insights in a user-friendly interface for root cause analysis
+
+#### Using query logs to understand DQ in the warehouse
+
+Questions we can answer:
+
+* who is accessing this data?
+* where does it come from upstream? Where is it going downstream?
+* how often, on average, is this particular transformation executed?
+* how many rows are affected?
+* when was this table last queried?
+* was that update part of a regular cadence, or does it break a pattern?
+* what’s the load on this warehouse as a function of the time of day?
+* is this query taking progressively longer than it was two months ago?
+* who (or what bot) has access to this resource who shouldn’t?
+
+#### Using query logs to understand DQ in the lake
+
+Example questions we can answer:
+
+* which job pipeline or user was responsible for this object’s creation?
+* what schema is the object using or relying on?
+
+### Designing a data catalog
+
+Data catalogs, like to library catalogs, inventory metadata to assess data accessibility, health, and location. Tools like Alation, Collibra, and Informatica enhance discoverability, collaboration, and compliance through machine learning and automation. Acting as a single source of truth, data catalogs track data lineage, ensure trust, and manage personally identifiable information across pipelines.
+
+Key questions addressed include data relevance, representation, and usability. While early cataloging relied on spreadsheets, manual methods fail to scale with vast, evolving datasets. Automation is essential for handling unstructured, dynamic data suited for machine learning pipelines. Modern catalogs automate metadata discovery and organization, overcoming the limitations of manual cataloging and catering to diverse data producer and consumer perspectives.
+
+### Building a data catalog
+
+Building a data catalog requires collaboration with stakeholders to prioritise important data, assign ownership, and document metadata. While manual methods like spreadsheets were traditionally used, automation through SQL parsers and open-source databases is now essential. Cataloging tools like Amundsen and Apache Atlas can visualize metadata, while advanced catalogs focus on automated discovery and active metadata for self-service use.
+
+Modern data management emphasises data discovery over static catalogs, offering real-time insights into data health, lineage, and usage across domains. This approach aligns with data mesh principles, assigning domain-specific accountability. Key features include:
+
+- self-service discovery and automation for easier access and reduced silos
+- scalability to adapt to growing, unstructured datasets
+- data lineage for tracking dependencies and troubleshooting pipeline issues
+
+By integrating automated monitoring and observability platforms, teams ensure reliable, dynamic data discovery and quickly address pipeline issues, fostering trust and informed decision-making.
+
+Example:
+
+| Table name                  | Dashboard/​report               | Last updated     | Owner                          | Notes                                                               |
+|-----------------------------|---------------------------------|------------------|--------------------------------|---------------------------------------------------------------------|
+| LIOR_GOOD_TABLE_3.csv       | Exec Forecasting V3 (Looker)   | March 3, 2022    | Lior Gavish (lior@internet.org) | Lior’s table; used for executive financial forecasts, i.e., ARR    |
+| MEETINGS_DOWNTIME_2022.csv  | Report 1234 (Tableau)          | February 2, 2022 | Barr Moses (barr@internet.org) | Outages occurred during in-person meetings                         |
+| DONT_USE_4_MV.csv           | Dashboard Yikes (Chartio)      | October 30, 2021 | Molly Vorwerck (molly@internet.org) | Who knows?                                                      |
+| RYANS_DATA.csv              | Marketing Model (Looker)       | March 3, 2022    | Ryan Kearns (ryan@internet.org) | For demand generation models to inform ad spend across social channels |
+
+## DQ Fundamentals - Chapter 3 - Collecting, Cleaning, Transforming, and Testing Data
+
+### Collecting data
+
+The entrypoint is where raw, unprocessed data enters the pipeline. It is critical because it shapes the pipeline's downstream processes. Data at this stage is often noisy and heterogeneous, originating from application logs, API responses, or sensors.
+
+**Data Sources**  
+
+- application log data  
+  - records events within software, often with timestamps and log levels (info, warn, error)  
+  - structure is variable, typically text-based, and chosen by developers  
+  - used for diagnostics (tracking errors) or auditing (recording actions)  
+
+- api responses  
+  - data exchanged between programs, often in structured formats like json  
+  - includes response codes (e.g 200 ok, 500 error) 
+  - purpose depends on context, such as monitoring errors or using response content  
+
+- sensor data  
+  - collected from iot devices or research equipment, often noisy or prone to silent failure  
+  - requires downstream cleaning and anomaly detection  
+  - used in tasks like machine learning or real-time alerts, where high volume and low latency matter
+
+The next step is cleaning and preparing the data for processing.  
+
+### Cleaning data
+
+Common ways:
+
+* outlier removal
+* assessing features
+* normalisation
+* data reconstruction (interpolation, extrapolation, labeling)
+* time zone conversion
+* type coercion
+
+### Batch vs. Stream Processing
+
+Batch processing collects and processes data in large chunks over a set period, making it cost-effective and ideal for handling large datasets. Stream processing, in contrast, processes data in near real time, enabling timely insights for applications like fraud detection or ride-sharing.  
+
+- **batch processing**: traditional method, cost-efficient, used for periodic tasks (e.g. Hadoop, BigQuery)  
+- **stream processing**: supports real-time insights, suitable for dynamic use cases (e.g. Kafka, Spark, Flink)  
+
+Stream processing has gained popularity as real-time needs increase, though batch remains widely used for non-urgent data tasks.  
+
+### DQ for streaming 
+
+Stream processing prioritizes speed over volume, often compromising data quality compared to batch processing. Traditional testing methods (unit, functional, integration) struggle to scale for real-time, unpredictable data, necessitating new approaches to ensure reliable streaming systems.  
+
+- AWS kinesis:  
+  - serverless, scalable, integrates well with AWS stack (e.g. S3, Redshift)
+  - cost-effective and supports multiple SDKs but fixed 7-day data retention 
+
+![image](https://github.com/user-attachments/assets/96a31e27-518c-4980-8d7d-b0fe6fc86e72)
+
+- Kafka:  
+  - open source, customizable, high throughput, low latency (e.g. ~2ms)  
+  - requires more expertise but allows better configurability  
+
+Kinesis suits smaller teams for quick implementation, while Kafka offers flexibility for advanced needs. Stream processing systems need robust data normalization and quality frameworks to handle real-time challenges effectively.  
+
+### Normalising data
+
+* handling heterogeneous data sources
+* schema checking and type coercion
+* syntactic vs semantic ambiguity in data
+
+### Running analytical data transformations
+
+Analytical data transformations, often part of ETL or ELT workflows, involve preparing data for analytical use.  
+
+- **reasons for transformations**:  
+  - schema alignment, data cleaning (deduplication, filtering, etc.)  
+  - unit/currency standardization and encryption for compliance  
+  - governance audits and quality checks  
+
+The transform step is critical for ensuring data quality and usability in downstream systems.
+
+### Alerting and testing
+
+Most common tests:
+
+* null values
+* data volume/any data at all
+* check data distribution
+* uniqueness
+* known invariants
+
+To run data quality tests, you need to do two simple things:
+
+* load the transformed data into a temporary staging table/data set
+* run tests to ensure that the data in the staging table falls within the thresholds demanded of production (i.e., you need to answer “yes” to the question: is this what reliable data looks like?)
+
+### Managing DQ with Airflow
+
+Apache Airflow improves data quality in pipelines by orchestrating workflows and monitoring for issues like deteriorating queries or buggy code. Tools like SLAs, circuit breakers, and SQL check operators help prevent and manage data downtime.  
+
+- SLAs: set maximum task durations and trigger alerts or callbacks when tasks exceed thresholds
+
+```python
+def sla_callback(dag, task_list, blocking_task_list, slas, blocking_tis):
+    print(
+        "The callback arguments are: ",
+        {
+            "dag": dag,
+            "task_list": task_list,
+            "blocking_task_list": blocking_task_list,
+            "slas": slas,
+            "blocking_tis": blocking_tis,
+        },
+    )
+  
+@dag(
+    schedule_interval="*/2 * * * *",
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    catchup=False,
+    sla_miss_callback=sla_callback,
+    default_args={'email': "email@example.com"},
+)
+def example_sla_dag():
+    @task(sla=datetime.timedelta(seconds=10))
+    def sleep_20():
+        """Sleep for 20 seconds"""
+        time.sleep(20)
+  
+    @task
+    def sleep_30():
+        """Sleep for 30 seconds"""
+        time.sleep(30)
+  
+    sleep_20() >> sleep_30()
+  
+dag = example_sla_dag()
+```
+
+- circuit breakers: pause pipelines if data fails quality checks (e.g., freshness, volume, schema) to prevent downstream issues and reduce backfilling costs  
+- sql check operators: validate pipeline data against expectations using SQL queries; can halt pipelines for critical data failures  
+
+Good use of these can ensure data reliability while minimizing unnecessary disruptions.  
+
+## More SQL on interviewquery.com
+
+<img width="286" alt="image" src="https://github.com/user-attachments/assets/9a79e8fe-0e85-4393-83af-980e526e05ea">
+
+Did 10 more medium Qs ^^
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 01798e50c4be766bec4bcddf89f490a4fe21e888 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 1 Dec 2024 01:05:38 +0900
Subject: [PATCH 515/638] Create 2024-11-30-day-334.md

---
 _posts/2024-11-30-day-334.md | 226 +++++++++++++++++++++++++++++++++++
 1 file changed, 226 insertions(+)
 create mode 100644 _posts/2024-11-30-day-334.md

diff --git a/_posts/2024-11-30-day-334.md b/_posts/2024-11-30-day-334.md
new file mode 100644
index 000000000..93096b489
--- /dev/null
+++ b/_posts/2024-11-30-day-334.md
@@ -0,0 +1,226 @@
+---
+layout: post
+title: (Day 334) Starting to learn about Apache Iceberg
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 334!
+A quick summary of today:
+* more DQ fundamentals
+* learning about apache iceberg
+* streamed
+
+## DQ Fundamentals - Chapter 4 - Monitoring and Anomaly Detection for Your Data Pipelines
+
+When it comes to understanding when data breaks, our best course of action is to lean on monitoring, specifically anomaly detection techniques that identify when our expected thresholds for volume, freshness, distribution, and other values don’t meet expectations.
+
+Up until recently, anomaly detection was considered a nice-to-have—not a need-to-have—for many data teams. Now, as data systems become increasingly complex and companies empower employees across functions to use data, it’s imperative that teams take both proactive and reactive approaches to solving for data quality.
+
+### Knowing your known unknowns and unknown unknowns
+
+![image](https://github.com/user-attachments/assets/276bd1d8-5bf6-4269-9693-0c94a1710938)
+
+Unknown unknowns refer to data downtime that even the most comprehensive testing can’t account for, issues that arise across our entire data pipeline, not just the sections covered by specific tests. Unknown unknowns might include:
+
+* a distribution anomaly in a critical field that causes our Tableau dashboard to malfunction
+* a JSON schema change made by another team that turns 6 columns into 600
+* an unintended change to ETL leading to tests not running and bad data being missed
+* incomplete or stale data that goes unnoticed until several weeks later, affecting key marketing metrics
+* a code change that causes an API to stop collecting data feeding an important new product
+* data drift over time, which can be challenging to catch, particularly if our tests look only at the data being written at the time of our ETL jobs, which don’t normally take into account data that is already in a given table
+
+While testing and circuit breakers can handle many of our known unknowns, monitoring and anomaly detection can cover our bases when it comes to unknown unknowns.
+
+### Building an anomaly detection algorithm
+
+The first pillar of data observability we monitor for is freshness, which can give us a strong indicator of when critical data assets were last updated. If a report that is regularly updated on the hour suddenly looks very stale, this type of anomaly should give us a strong indication that something is inaccurate or otherwise wrong.
+
+We also want to assess the field-level, distributional health of our data. Distribution tells us all of the expected values of our data, as well as how frequently each value occurs. One of the simplest questions is, “How often is my data NULL?” In many cases, some level of incomplete data is acceptable—but if a 10% null rate turns into 90%, we’ll want to know.
+
+### Building monitors for schema and lineage
+
+Good anomaly detection is certainly part of the data observability puzzle, but it’s not everything. Equally important is context. If a data anomaly occurred, great. But where? What upstream pipelines may be the cause? What downstream dashboards will be affected by a data anomaly? And has the formal structure of my data changed? Good data observability hinges on our ability to properly leverage metadata to answer these data anomaly questions.
+
+### Scaling anomaly detection with python and ML
+
+* set up monitoring tools
+* account for false positives and false negatives
+* improving precision and recall
+* F-scores
+* being careful with model accuracy
+
+### Other useful anomaly detection approaches
+
+* rule definitions and hard thresholding
+* autoregressive models for time-series anomaly detection
+* exponential smoothing
+* clustering
+* hyperparam tuning
+* ensemble method approaches
+
+### Designing DQ monitors for warehouses vs lakes
+
+It’s important to distinguish whether we’re working with structured, monolithic data from a warehouse or entering the wild west of the modern data lake ecosystem.
+
+The primary differences between designing anomaly detection algorithms for warehouses and lakes:
+
+* the number of entrypoints we have to account for
+* how the metadata is collected and stored
+* how we can access that metadata
+
+First, data lake systems tend to have high numbers of entrypoints, meaning one should assume high heterogeneity in data entering from different sources. In monitoring, say, null rates in tabular data entering from Postgres, application logs, and a web API, a data scientist might notice clusters of table behavior corresponding to the different endpoints. More likely than not, different model architectures (and different hyperparameters) may work better at predicting anomalies in each different format. One way to do that is to condition on the endpoint of the data itself, forming a new feature for input into the machine learning model. Another is to use an ensemble model architecture, or simply to have separate models for each of our use cases.
+
+Second, metadata collected straight into a data lake may need varying levels of preprocessing before we can expect an anomaly detection algorithm to derive anything of value from it. Types may need coercion, schemas may need alignment, and we may find deriving entirely new augmented features in the data before running the detector’s training task.
+
+## Chapter 5 - Architecting for data reliability
+
+### Measuring and maintaining high data reliability at ingestion
+
+Key practices include data cleaning, enrichment, and testing. Data cleaning removes irrelevant, incomplete, or duplicate data, while data enrichment adds value by merging first- or third-party data. Data testing validates assumptions about data through methods like unit, functional, and integration testing, checking aspects like null values, freshness, and distribution. Automating these processes using tools like Apache Griffin, Great Expectations, or dbt improves efficiency and reliability. Proactive and reactive approaches can be used to enhance DQ and combining testing with monitoring to address unexpected data issues during production.
+
+### Measuring and maintaining DQ in the pipeline
+
+Increasingly, data teams are coming to rely on similar principles of observability and monitoring to track data quality in production pipelines, with companies developing their own unique methodology for how to measure it, depending on the needs of the business.
+
+Data observability can be broken down into five major pillars
+
+* Freshness - Is the data recent? When was the last time it was generated? What upstream data is included/omitted?
+* Distribution - Is the data within accepted ranges? Is it properly formatted? Is it complete?
+* Volume - Has all the data arrived?
+* Schema - What is the schema, and how has it changed? Who has made these changes and for what reasons?
+* Lineage - For a given data asset, what are the upstream sources and downstream assets that are impacted by it? Who are the people generating this data, and who is relying on it for decision making?
+
+The five pillars of data observability serve as key measurements for understanding the health of data at each stage in its life cycle, and provide a fresh lens with which to view the quality of our data.
+
+### Understanding DQ downstream
+
+DQ issues often become apparent downstream, in the analytics layer or applications. To address this, teams use tools like data reliability dashboards to monitor metrics such as time to detection (TTD) and time to resolution (TTR). Service-level agreements (SLAs), indicators (SLIs), and objectives (SLOs) help define and measure data reliability based on stakeholder needs. Data quality is traditionally evaluated on six dimensions: completeness, timeliness, validity, accuracy, consistency, and uniqueness. For data engineers, practical measures include tracking data errors, null values, timeliness, duplicates, and consistency, with a focus on aligning with stakeholders to prioritize critical data.
+
+### Developing trust in your data
+
+#### Importance of trust in data  
+
+- advanced systems are ineffective without reliable data  
+- trust enables accurate insights and business value  
+
+#### Data observability  
+
+- observability ensures data health across freshness, distribution, volume, schema, and lineage  
+- borrowed from DevOps principles: monitoring, alerting, and triaging data issues  
+- tools provide end-to-end lineage and automate monitoring without security risks  
+
+#### Measuring the impact of poor DQ
+
+- consequences include wasted time, revenue loss, compliance risks, and erosion of trust  
+- key metrics to measure impact  
+  - time to detection (TTD): time taken to identify issues  
+  - time to resolution (TTR): time taken to fix issues  
+
+- downtime costs formula  
+  - (TTD hours + TTR hours) × downtime hourly cost  
+
+- factors affecting cost  
+  - labor for engineers  
+  - compliance risks  
+  - opportunity cost of lost trust  
+
+#### Setting data reliability goals  
+
+- inspired by DevOps principles like SLAs, SLOs, and SLIs  
+- steps to establish reliability:
+  - define reliability through inventory and stakeholder input  
+  - measure indicators like update frequency and error distribution  
+  - track objectives to monitor and improve reliability  
+
+#### Managing data incidents  
+
+- effective incident resolution minimizes impact on stakeholders  
+- communication strategies ensure alignment during issue resolution  
+- borrowed practices from site reliability engineering streamline processes  
+
+#### Outcomes  
+
+- improved data quality and reduced downtime  
+- enhanced stakeholder trust and business alignment
+
+## Apache Iceberg: The Definitive Guide - Chapter 1 - Introduction to Apache Iceberg
+
+### Data Warehouses
+
+DWs act as centralised repositories for large volumes of data ingested from various sources. They are designed to support analytical workloads like Business Intelligence (BI) reports and dashboards. They comprise storage, file format, table format, storage engine, catalog, and compute engine, all tightly coupled within a single system. 
+
+* Pros: data warehouses are a single source of truth for data, enable fast query performance on vast amounts of historical data, and provide effective data governance policies
+* Cons: they often lock data into vendor-specific systems, can be expensive for storage and computation, primarily support structured data, and are not natively suited for advanced analytical workloads like ML
+
+### Data Lakes
+
+Data lakes emerged as a cheaper alternative to data warehouses, capable of storing vast amounts of both structured and unstructured data in open formats. Data lakes utilise distributed filesystems like HDFS or cloud object storage for storage, rely on open file formats like Parquet or Avro, and employ table formats like Hive to organise data.  They also leverage distributed query engines like Spark or Presto for computation.
+
+* Pros: offer lower storage and compute costs compared to data warehouses, store data in open formats, can handle unstructured data, and support ML use cases
+* Cons: can suffer from performance issues due to the decoupled nature of their components, require significant configuration effort, and lack built-in ACID transaction guarantees, leading to potential data inconsistency issues
+
+### Data Lakehouses
+
+Data lakehouses represent the latest evolution, combining the best aspects of data warehouses and data lakes. They provide data warehouse-like functionalities (ACID transactions, performance) on top of data lake storage. They leverage data lake table formats like Apache Iceberg, Hudi, or Delta Lake to provide a metadata layer that enables ACID transactions, better performance, and features like time travel and schema evolution directly on data lake storage.
+
+#### Benefits:
+  * Fewer data copies: reducing data drift and costs
+  * Faster query performance: leading to faster insights
+  * Historical data snapshots: enabling rollback capabilities and reducing the impact of errors
+  * Affordable architecture:  due to reduced data duplication and lower storage and compute costs
+  * Open architecture:  avoiding vendor lock-in by using open formats like Iceberg and Parquet
+
+### Apache Iceberg
+
+Apache Iceberg is a popular open table format for data lakehouses. It addresses limitations of older table formats like Hive by defining tables as a list of individual files instead of directories.
+
+#### Key Features
+
+  * ACID transactions: ensuring data consistency
+  * Partition evolution:  allowing changes to partitioning without rewriting the entire table
+  * Hidden partitioning: simplifying queries and improving performance
+  * Row-level table operations: providing flexibility for handling updates and deletes
+  * Time travel: enabling queries on historical data snapshots
+  * Version rollback: allowing reversion to previous table states
+  * Schema evolution: supporting changes to table schemas
+
+## Apache Iceberg: The Definitive Guide - Chapter 2. The Architecture of Apache Iceberg
+
+Iceberg's architecture is designed to overcome the limitations of traditional table formats like Hive, enabling efficient data management and analysis in data lakes.
+
+### Three-Layered Structure:
+
+![image](https://github.com/user-attachments/assets/3fe7764d-2d68-4911-a884-a88b66802c97)
+
+Apache Iceberg tables consist of three main layers:
+
+* Data layer: this stores the actual data files, typically in formats like Parquet, ORC, or Avro. Iceberg is file format agnostic, providing flexibility and future-proofing  
+* Metadata layer:  this crucial layer manages metadata about the table's data and operations. It uses a tree structure comprising manifest files, manifest lists, and metadata files. This layered approach ensures efficient data access and enables key features like time travel and schema evolution
+* Catalog layer: This acts as a central registry, pointing to the current metadata file for each Iceberg table. It provides a single entry point for engines and tools to locate and interact with Iceberg tables. Various backends like Amazon S3, Hive Metastore, or Nessie can serve as an Iceberg catalog
+
+### Components of the Data Layer:
+
+* Datafiles: these files contain the actual data of the table, supporting various file formats like Parquet, ORC, and Avro.  Parquet, with its columnar structure, is the most widely used format due to its performance advantages for OLAP workloads
+* Delete Files: these files track records that have been deleted from the dataset. They facilitate the Merge-on-Read (MOR) approach for updates and deletes, where changes are tracked separately and merged during query execution.  Two types of delete files exist:
+    * Positional delete files: identify deleted rows by their exact position in the data file
+    * Equality delete files: identify deleted rows based on the values of specific fields
+
+### Components of the Metadata Layer:
+
+* Manifest files: these track the data files and delete files within the data layer, including their location and statistics such as record counts and column bounds.  Maintaining these statistics at the file level enables Iceberg to overcome the limitations of Hive's table format, leading to significant performance improvements
+* Manifest lists: a manifest list represents a snapshot of the Iceberg table at a specific point in time. It contains a list of all active manifest files, their locations, partition information, and statistics
+* Metadata files: these store high-level metadata about the Iceberg table, including schema information, partition specifications, snapshots, and the current snapshot. Every change to the table results in a new metadata file, ensuring a linear history of table commits and facilitating concurrent writes
+* Puffin files: while datafiles and metadata files already contain statistics to enhance query performance, Puffin files introduce additional structures for advanced performance optimization. These files store specialized statistics and indexes, such as Theta sketches from the Apache DataSketches library, to accelerate queries that require distinct value counts or other specific computations
+
+
+## [Stream](https://www.youtube.com/watch?v=Bo09fvitFtM&t=6s)
+
+I streamed and read/watched a few things about Apache Iceberg and it seems very intriguing and will keep looking into it.
+
+---
+
+I kind of fell asleep and forgot to post this ~ so here it is at 1am 😆
+
+That is all for today!
+
+See you tomorrow :)

From d536a9f0cd73947b5833307318b7f4a6f3a69547 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 1 Dec 2024 23:54:42 +0900
Subject: [PATCH 516/638] Create 2024-12-01-day-335.md

---
 _posts/2024-12-01-day-335.md | 315 +++++++++++++++++++++++++++++++++++
 1 file changed, 315 insertions(+)
 create mode 100644 _posts/2024-12-01-day-335.md

diff --git a/_posts/2024-12-01-day-335.md b/_posts/2024-12-01-day-335.md
new file mode 100644
index 000000000..ed7c575c1
--- /dev/null
+++ b/_posts/2024-12-01-day-335.md
@@ -0,0 +1,315 @@
+---
+layout: post
+title: (Day 335) New content from Zach Wilson's free YT bootcamp
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 335!
+A quick summary of today:
+* spark lectures + labs by Zach Wilson
+
+## Spark by Zach Wilson [1](https://www.youtube.com/watch?v=3R-SLYK-P_0) [2](https://www.youtube.com/watch?v=JriG4d9o2Mk)
+
+Here are my notes from the two lectures
+
+![Spark week3 P1](https://github.com/user-attachments/assets/aea844e2-9692-4640-b96e-bcc525bfb94a)
+![Spark week3 P2](https://github.com/user-attachments/assets/fa3390a9-80ec-4f9d-9e28-81dff06369a3)
+![Spark week3 P3](https://github.com/user-attachments/assets/4ecc266a-ed70-4ced-a541-70ba15d79333)
+![Spark week3 P4](https://github.com/user-attachments/assets/c8b0e525-7349-4046-b8ef-2b09019b4539)
+![Spark week3 P5](https://github.com/user-attachments/assets/f4817cc0-e604-4293-9ab7-9ce7daf077cf)
+![Spark week3 P6](https://github.com/user-attachments/assets/ad853804-3274-4460-92e0-db5bfd0efc69)
+![Spark week3 P7](https://github.com/user-attachments/assets/837e021c-4b7f-4649-9f1f-1612cb9db4ee)
+
+Then I decided to do the homework. Doing the homework did not *really* require watching the lectures but I wanted to watch the lectures first anyway ~
+
+There are probably better ways to do this than locally, but for now it's fine. And also I was able to confim some of the results with another student
+
+In each code block below, the 1st line is the asked task:
+
+(it's 5 different scripts as I think that's how they will need to be submitted, but can easily put it in 1 if needed)
+
+```python
+# 1. Disabled automatic broadcast join
+from pyspark.sql import SparkSession
+spark = SparkSession.builder \
+                    .appName("Jupyter") \
+                    .config("spark.sql.autoBroadcastJoinThreshold", "-1") \
+                    .getOrCreate()
+```
+
+```python
+# 2. Explicitly broadcast JOINs medals and maps
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+spark = SparkSession.builder \
+                    .appName("Jupyter") \
+                    .config("spark.sql.autoBroadcastJoinThreshold", "-1") \
+                    .getOrCreate()
+
+medals = spark.read.option('header', 'true').csv('/home/iceberg/data/medals.csv')
+medals_matches_players = spark.read.option('header', 'true').csv('/home/iceberg/data/medals_matches_players.csv')
+maps = spark.read.option('header', 'true').csv('/home/iceberg/data/maps.csv')
+matches = spark.read.option('header', 'true').csv('/home/iceberg/data/matches.csv')
+
+medals_joined = medals_matches_players.join(F.broadcast(medals), medals_matches_players.medal_id == medals.medal_id)
+maps_joined = matches.join(F.broadcast(maps), maps.mapid == matches.mapid)
+```
+
+```python
+# 3. Bucket join match_details, matches, and medal_matches_players on match_id with 16 buckets
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+spark = SparkSession.builder \
+                    .appName("Jupyter") \
+                    .config("spark.sql.autoBroadcastJoinThreshold", "-1") \
+                    .getOrCreate()
+
+matches = spark.read.option("header", "true") \
+                    .option("inferSchema", "true") \
+                    .csv("../data/matches.csv")
+
+matches.select("match_id", "is_team_game", "playlist_id", "completion_date", "mapid") \
+        .write.mode("append") \
+        .partitionBy("completion_date") \
+        .bucketBy(16, "match_id") \
+        # .saveAsTable("matches_bucketed")
+
+match_details = spark.read.option("header", "true") \
+                          .option("inferSchema", "true") \
+                          .csv("../data/match_details.csv")
+
+medals_matches_players = spark.read.option("header", "true") \
+                                  .option("inferSchema", "true") \
+                                  .csv("../data/medals_matches_players.csv")
+
+match_details.select("match_id", "player_gamertag", "player_total_kills", "player_total_deaths") \
+             .write.mode("append") \
+             .bucketBy(16, "match_id") \
+        #      .saveAsTable("match_details_bucketed")
+
+medals_matches_players.select("match_id", "medal_id", "count") \
+                     .write.mode("append") \
+                     .bucketBy(16, "match_id") \
+                #      .saveAsTable("medals_matches_players_bucketed")
+
+join1 = matches.alias("m").join(
+    match_details.alias("md"), 
+    on=F.col("m.match_id") == F.col("md.match_id"), 
+    how="inner"
+)
+
+final_join = join1.join(
+    medals_matches_players.alias("mmp"), 
+    on=F.col("m.match_id") == F.col("mmp.match_id"), 
+    how="inner"
+)
+
+final_join = final_join.select(
+    F.col("m.match_id").alias("matches_match_id"),
+    F.col("m.mapid"),
+    F.col("m.is_team_game"),
+    F.col("m.playlist_id"),
+    F.col("m.completion_date"),
+    F.col("md.player_gamertag"),
+    F.col("md.player_total_kills"),
+    F.col("md.player_total_deaths"),
+    F.col("mmp.medal_id"),
+    F.col("mmp.count")
+)
+```
+
+```python
+# 4. Aggregate the joined data frame to figure out questions
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+spark = SparkSession.builder \
+                    .appName("Jupyter") \
+                    .config("spark.sql.autoBroadcastJoinThreshold", "-1") \
+                    .getOrCreate()
+
+matches = spark.read.option("header", "true") \
+                    .option("inferSchema", "true") \
+                    .csv("../data/matches.csv")
+
+matches.select("match_id", "is_team_game", "playlist_id", "completion_date", "mapid") \
+        .write.mode("append") \
+        .partitionBy("completion_date") \
+        .bucketBy(16, "match_id") \
+        # .saveAsTable("matches_bucketed")
+
+match_details = spark.read.option("header", "true") \
+                          .option("inferSchema", "true") \
+                          .csv("../data/match_details.csv")
+
+medals_matches_players = spark.read.option("header", "true") \
+                                  .option("inferSchema", "true") \
+                                  .csv("../data/medals_matches_players.csv")
+
+match_details.select("match_id", "player_gamertag", "player_total_kills", "player_total_deaths") \
+             .write.mode("append") \
+             .bucketBy(16, "match_id") \
+        #      .saveAsTable("match_details_bucketed")
+
+medals_matches_players.select("match_id", "medal_id", "count") \
+                     .write.mode("append") \
+                     .bucketBy(16, "match_id") \
+                #      .saveAsTable("medals_matches_players_bucketed")
+
+join1 = matches.alias("m").join(
+    match_details.alias("md"), 
+    on=F.col("m.match_id") == F.col("md.match_id"), 
+    how="inner"
+)
+
+final_join = join1.join(
+    medals_matches_players.alias("mmp"), 
+    on=F.col("m.match_id") == F.col("mmp.match_id"), 
+    how="inner"
+)
+
+final_join = final_join.select(
+    F.col("m.match_id").alias("matches_match_id"),
+    F.col("m.mapid"),
+    F.col("m.is_team_game"),
+    F.col("m.playlist_id"),
+    F.col("m.completion_date"),
+    F.col("md.player_gamertag"),
+    F.col("md.player_total_kills"),
+    F.col("md.player_total_deaths"),
+    F.col("mmp.medal_id"),
+    F.col("mmp.count")
+)
+
+# Which player averages the most kills per game?
+player_kills = final_join.groupBy("md.player_gamertag").agg(
+    F.avg("md.player_total_kills").alias("avg_kills_per_game")
+)
+
+player_kills_sorted = player_kills.orderBy(F.desc("avg_kills_per_game"))
+
+player_kills_sorted.show(1)
+
+# Which playlist gets played the most?
+playlist_counts = final_join.groupBy("m.playlist_id").agg(
+    F.count("matches_match_id").alias("matches_played")
+)
+
+playlist_sorted = playlist_counts.orderBy(F.desc("matches_played"))
+
+playlist_sorted.show(1)
+
+# Which map gets played the most?
+map_counts = final_join.groupBy("m.mapid").agg(
+    F.count("matches_match_id").alias("matches_played")
+)
+
+map_sorted = map_counts.orderBy(F.desc("matches_played"))
+
+map_sorted.show(1)
+```
+
+```python
+# 5. Try different .sortWithinPartitions to see which has the smallest data size
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+spark = SparkSession.builder \
+                    .appName("Jupyter") \
+                    .config("spark.sql.autoBroadcastJoinThreshold", "-1") \
+                    .getOrCreate()
+
+matches = spark.read.option("header", "true") \
+                    .option("inferSchema", "true") \
+                    .csv("../data/matches.csv")
+
+matches.select("match_id", "is_team_game", "playlist_id", "completion_date", "mapid") \
+        .write.mode("append") \
+        .partitionBy("completion_date") \
+        .bucketBy(16, "match_id") \
+        # .saveAsTable("matches_bucketed")
+
+match_details = spark.read.option("header", "true") \
+                          .option("inferSchema", "true") \
+                          .csv("../data/match_details.csv")
+
+medals_matches_players = spark.read.option("header", "true") \
+                                  .option("inferSchema", "true") \
+                                  .csv("../data/medals_matches_players.csv")
+
+match_details.select("match_id", "player_gamertag", "player_total_kills", "player_total_deaths") \
+             .write.mode("append") \
+             .bucketBy(16, "match_id") \
+        #      .saveAsTable("match_details_bucketed")
+
+medals_matches_players.select("match_id", "medal_id", "count") \
+                     .write.mode("append") \
+                     .bucketBy(16, "match_id") \
+                #      .saveAsTable("medals_matches_players_bucketed")
+
+join1 = matches.alias("m").join(
+    match_details.alias("md"), 
+    on=F.col("m.match_id") == F.col("md.match_id"), 
+    how="inner"
+)
+
+final_join = join1.join(
+    medals_matches_players.alias("mmp"), 
+    on=F.col("m.match_id") == F.col("mmp.match_id"), 
+    how="inner"
+)
+
+final_join = final_join.select(
+    F.col("m.match_id").alias("matches_match_id"),
+    F.col("m.mapid"),
+    F.col("m.is_team_game"),
+    F.col("m.playlist_id"),
+    F.col("m.completion_date"),
+    F.col("md.player_gamertag"),
+    F.col("md.player_total_kills"),
+    F.col("md.player_total_deaths"),
+    F.col("mmp.medal_id"),
+    F.col("mmp.count")
+)
+
+mapid_playlist_sort_df = final_join.sortWithinPartitions(F.col("m.mapid"), F.col("m.playlist_id"))
+matchid_mapid_playlist_sort_df = final_join.sortWithinPartitions(F.col("matches_match_id"), F.col("m.mapid"), F.col("m.playlist_id"))
+mapid_playlist_matchid_sort_df = final_join.sortWithinPartitions(F.col("m.mapid"), F.col("m.playlist_id"), F.col("matches_match_id"))
+playlist_mapid_matchid_sort_df = final_join.sortWithinPartitions(F.col("m.playlist_id"), F.col("m.mapid"), F.col("matches_match_id"))
+
+final_join.write.mode("overwrite").parquet("final_join")
+mapid_playlist_sort_df.write.mode("overwrite").parquet("mapid_playlist_sort")
+matchid_mapid_playlist_sort_df.write.mode("overwrite").parquet("matchid_mapid_playlist_sort")
+mapid_playlist_matchid_sort_df.write.mode("overwrite").parquet("mapid_playlist_matchid_sort")
+playlist_mapid_matchid_sort_df.write.mode("overwrite").parquet("playlist_mapid_matchid_sort")
+
+sc = spark.sparkContext
+fs = sc._jvm.org.apache.hadoop.fs.FileSystem.get(sc._jsc.hadoopConfiguration())
+
+def get_file_size_and_count(path):
+    files = fs.globStatus(sc._jvm.org.apache.hadoop.fs.Path(path + "/*"))
+    file_sizes = sum(file.getLen() for file in files)
+    return file_sizes
+
+final_join_size = get_file_size_and_count("final_join")
+mapid_playlist_size = get_file_size_and_count("mapid_playlist_sort")
+matchid_mapid_playlist_size = get_file_size_and_count("matchid_mapid_playlist_sort")
+mapid_playlist_matchid_size = get_file_size_and_count("mapid_playlist_matchid_sort")
+playlist_mapid_matchid_sort_df = get_file_size_and_count("playlist_mapid_matchid_sort")
+
+print(f"final_join: {final_join_size:,} bytes") # 8,087,514 bytes   
+print(f"mapid_playlist_sort: {mapid_playlist_size:,} bytes") # 15,453,444 bytes
+print(f"matchid_mapid_playlist_sort: {matchid_mapid_playlist_size:,} bytes") # 8,103,554 bytes
+print(f"mapid_playlist_matchid_sort: {mapid_playlist_matchid_size:,} bytes") # 7,837,038 bytes
+print(f"playlist_mapid_matchid_sort: {playlist_mapid_matchid_sort_df:,} bytes") # 7,847,491 bytes
+```
+
+I hung out a bunch on Zach Wilson's community discord today as well ~ trying to discover/fix some bugs people (including me as well) had.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From efbb3b48defd57e4bfd2bac961622c9ef21222b4 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 2 Dec 2024 20:53:27 +0900
Subject: [PATCH 517/638] Create 2024-12-02-day-336.md

---
 _posts/2024-12-02-day-336.md | 402 +++++++++++++++++++++++++++++++++++
 1 file changed, 402 insertions(+)
 create mode 100644 _posts/2024-12-02-day-336.md

diff --git a/_posts/2024-12-02-day-336.md b/_posts/2024-12-02-day-336.md
new file mode 100644
index 000000000..95c13d78d
--- /dev/null
+++ b/_posts/2024-12-02-day-336.md
@@ -0,0 +1,402 @@
+---
+layout: post
+title: (Day 336) Learning about Write and Read Queries in Apache Iceberg
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 336!
+A quick summary of today:
+* more about Apache Iceberg
+* testing Spark jobs
+
+## Apache Iceberg: The Definitive Guide - Chapter 3 - Lifecycle of Write and Read Queries
+
+The Apache Iceberg table format provides high-performance queries during reads and writes, allowing you to run OLAP workloads directly on the data lake. What facilitates this performance is the way the various components of the Iceberg table format are designed. It is therefore critical to understand the structure of these components so that query engines can effectively use them for faster query planning and execution.
+
+Apache Iceberg’s components
+
+![image](https://github.com/user-attachments/assets/496526c0-bf74-4f9e-8fd2-8914390038cb)
+
+**Catalog layer**
+
+_A catalog holds the references to the current metadata pointer, that is, the latest metadata file for each table. Irrespective of whether you are doing a read operation or a write operation, the catalog is the first component that a query engine interacts with. In the case of reads, the engine reaches out to the catalog to learn about the current state of the table, and for writes, the catalog is used to adhere to the schema defined and to know about the table’s partitioning scheme._
+
+**Metadata layer**
+
+_The metadata layer in Apache Iceberg consists of three components: metadata files, manifest lists, and manifest files. Each time a query engine writes something to an Iceberg table, a new metadata file is created atomically and is defined as the latest version of the metadata file. This ensures that a linear history of the table commits, and it helps during scenarios such as concurrent writes (i.e., multiple engines writing data simultaneously). Also, during read operations, engines will always see the latest version of the table. Query engines interact with the manifest lists to get information about partition specifications that help them skip the nonrequired manifest files for faster performance. Finally, information from the manifest files, such as upper and lower bounds for a specific column, null value counts, and partition-specific data, is used by the engine for file pruning._
+
+**Data layer**
+
+_Query engines filter through the metadata files to read the datafiles required by a particular query efficiently. On the write side, datafiles get written on the file storage, and the related metadata files are created and updated accordingly._
+
+### Writing queries in Apache Iceberg
+
+Overview:
+
+![image](https://github.com/user-attachments/assets/c26500c0-c4da-44ec-86fd-3aa6ebbdc9fe)
+
+The write process in Apache Iceberg involves a series of steps that enable query engines to efficiently insert and update data. When a write query is initiated, it is sent to the engine for parsing. The catalog is then consulted to ensure consistency and integrity in the data and to write the data as per the defined partition strategies. The datafiles and metadata files are then written based on the query. Finally, the catalog file is updated to reflect the latest metadata, enabling subsequent read operations to access the most up-to-date version of the data
+
+**Creating a table**
+
+```
+# Spark SQL
+CREATE TABLE orders (
+   order_id BIGINT,
+   customer_id BIGINT,
+   order_amount DECIMAL(10, 2),
+   order_ts TIMESTAMP
+) 
+USING iceberg
+PARTITIONED BY (HOUR(order_ts))
+
+# Dremio 
+CREATE TABLE orders (
+  order_id BIGINT,
+  customer_id BIGINT,
+  order_amount DECIMAL(10, 2),
+  order_ts TIMESTAMP
+)
+PARTITION BY (HOUR(order_ts))
+```
+
+First, the query is sent to the query engine for parsing. Then, since it is a CREATE statement, the engine will start creating and defining the table.
+
+Then, the engine starts creating a metadata file named v1.metadata.json in the data lake filesystem to store information about the table. Based on the information on the table path, /path/to/warehouse/db1/table1, the engine writes the metadata file. It then defines the schema of the table orders by specifying the columns and data types and stores it in the metadata file. Finally, it assigns a unique identifier to the table: table-uuid. Once the query executes successfully, the metadata file v1.metadata.json is written to the data lake file storage
+
+If we inspect the metadata file we can fine:
+
+```json
+{
+ "table-uuid" : "072db680-d810-49ac-935c-56e901cad686",
+ "schema" : {
+    "type" : "struct",
+    "schema-id" : 0,
+    "fields" : [ {
+      "id" : 1,
+      "name" : "order_id",
+      "required" : false,
+      "type" : "long"
+    }, {
+      "id" : 2,
+      "name" : "customer_id",
+      "required" : false,
+      "type" : "long"
+    }, {
+      "id" : 3,
+      "name" : "order_amount",
+      "required" : false,
+      "type" : "decimal(10, 2)"
+    }, {
+      "id" : 4,
+      "name" : "order_ts",
+      "required" : false,
+      "type" : "timestamptz"
+    } 
+},
+ "partition-spec" : [ {
+    "name" : "order_ts_hour",
+    "transform" : "hour",
+    "source-id" : 4,
+    "field-id" : 1000
+  } ]
+}
+```
+
+This is the current state of the table - created and empty. In Iceberg terms, this is called a snapshot. Since there is no actual data in the table, so there are no datafiles in your data lake. Therefore, the snapshot doesn’t point to any manifest list; hence, there are no manifest files.
+
+Finally, the engine updates the current metadata pointer to point to the v1.⁠me⁠ta⁠da⁠ta.​js⁠on file in the catalog file version-hint.text, as this is the present state of the table.
+
+**Insert query**
+
+```
+# Spark SQL/Dremio's SQL Query Engine
+INSERT INTO orders VALUES (
+   123,
+   456,
+   36.17,
+   '2023-03-07 08:10:23'
+)
+```
+
+The query is sent to the query engine for parsing. Since this is an INSERT statement, the engine needs information about the table, such as its schema, to start with query planning.
+
+First, the query engine makes a request of the catalog to determine the location of the current metadata file and then reads it. 
+
+Although the engine’s motivation in this case is to insert new datafiles, it still interacts with the catalog, primarily for two reasons:
+
+* it needs to understand the current schema of the table to adhere to it
+* it needs to learn about the partitioning scheme to organize data accordingly while writing
+
+After the engine learns about the table schema and the partitioning scheme, it starts writing the new datafiles and the related metadata files. Here’s what happens in this process.
+
+The engine first writes the records as a Parquet datafile (Parquet is the default, but this can be changed) based on the hourly defined partitioning scheme of the table. Additionally, if a sort order is defined for the table, records will be sorted before being written into the datafile
+
+After writing the datafile, the engine creates a manifest file. This manifest file is given information about the path of the actual datafile the engine created. In addition, the engine writes statistical information, such as the upper and lower bounds of a column and the null value counts, in the manifest file, which is highly beneficial for the query engine to prune files and provide the best performance. The engine computes this information while processing the data it’s going to write, so this is a relatively lightweight operation, at least compared to a process that starts from scratch and has to compute the statistics.
+
+Here is a part of a manifest file’s content
+
+```json
+{
+"data_file" : {
+    "file_path" : 
+"s3://datalake/db1/orders/data/order_ts_hour=2023-03-07-08/0_0_0.parquet",
+"file_format" : "PARQUET",
+        "block_size_in_bytes" : 67108864,
+        "null_value_counts" : [],
+        "lower_bounds" : {
+        "array": [{
+"key": 1,
+        "value": 123
+}],
+}
+        "upper_bounds" : {
+        "array": [{
+"key": 1,
+        "value": 123
+}],
+},
+   }
+}
+```
+
+Next, the engine creates a manifest list to keep track of the manifest file. If existing manifest files are associated with this snapshot, those will also be added to this new manifest list. The engine writes this file to the data lake with information such as the manifest file’s path, the number of datafiles/rows added or deleted, and statistics about partitions, such as the lower and upper bounds of the partition columns. Again, the engine already has all this information, so it’s a lightweight operation to have these statistics. This information helps read queries exclude any nonrequired manifest files, facilitating faster queries
+
+Here's a part of the content of the manifest list:
+
+```json
+{
+ "manifest_path": 
+"s3://datalake/db1/orders/metadata/62acb3d7-e992-4cbc-8e41-58809fcacb3e.avro",
+ "manifest_length": 6152,
+ "added_snapshot_id": 8333017788700497002,
+ "added_data_files_count": 1,
+ "added_rows_count": 1,
+ "deleted_rows_count": 0,
+ "partitions": {
+        "array": [ {
+            "contains_null": false,
+            "lower_bound": {
+                "bytes": "¹Ô\\u0006\\u0000"
+            },
+            "upper_bound": {
+                "bytes": "¹Ô\\u0006\\u0000"
+            }
+        } ]
+    }
+}
+```
+
+Finally, the engine creates a new metadata file, v2.metadata.json, with a new snapshot, s1, by considering the existing metadata file, v1.metadata.json (previously current), while keeping track of the previous snapshot, s0. This new metadata file includes information about the manifest list created by the engine, with details such as the manifest list filepath, snapshot ID, and summary of the operation. Also, the engine makes a reference that this manifest list (or snapshot) is now the current one
+
+Here is what the content of the new metadata file looks like:
+
+```
+    "current-snapshot-id" : 8333017788700497002,
+      "refs" : {
+        "main" :  {
+         "snapshot-id" : 8333017788700497002,
+          "type" : "branch"
+}
+},
+"snapshots" : [ {
+    "snapshot-id" : 8333017788700497002,
+    "summary" : {
+      "operation" : "append",
+      "added-data-files" : "1",
+      "added-records" : "1",
+},
+    "manifest-list" : "s3://datalake/db1/orders/metadata/
+snap-8333017788700497002-1-4010cc03-5585-458c-9fdc-188de318c3e6.avro",
+  } ],
+```
+
+Now the engine goes to the catalog again to ensure that no other snapshots were committed while this INSERT operation was being run. By doing this validation, Iceberg guarantees no interference in operations in a scenario where multiple writers write data concurrently. With any write operation, Iceberg creates metadata files optimistically, assuming that the current version will remain unchanged until the writer commits. Upon completing the write, the engine commits atomically by switching the table’s metadata file pointer from the existing base version to the new version, v2.metadata.json, which now becomes the current metadata file.
+
+![image](https://github.com/user-attachments/assets/5e692231-01fd-49d4-ba76-5202d91dfc45)
+
+**Merge query**
+
+```
+# Spark SQL
+MERGE INTO orders o
+USING (SELECT * FROM orders_staging) s
+ON o.order_id = s.order_id
+WHEN MATCHED THEN UPDATE SET order_amount = s.order_amount
+WHEN NOT MATCHED THEN INSERT *;
+
+# Dremio 
+MERGE INTO orders o
+USING (SELECT * FROM orders_staging) s
+ON o.order_id = s.order_id
+WHEN MATCHED THEN UPDATE SET order_amount = s.order_amount
+WHEN NOT MATCHED THEN INSERT (order_id, customer_id, order_amount, order_ts)
+VALUES (s.order_id, s.customer_id, s.order_amount, s.order_ts)
+```
+
+The query engine begins by parsing the query and retrieving metadata, including the schema and partitioning details, from the catalog. It then loads data from both tables into memory to determine matching records. For this example, the operation uses the copy-on-write (COW) strategy, rewriting datafiles entirely when changes occur.
+
+Matching records, such as an updated order_amount for order_id=123, are written to a new datafile, while non-matching records, like a new order with order_id=124, are treated as regular inserts and written to a separate partition based on the timestamp. After writing the updated datafiles, the engine creates a manifest file referencing the new file paths and includes statistics like value counts and column bounds.
+
+A new metadata file is generated to reflect these changes, pointing to the updated manifest files and snapshot information. The catalog is updated with the location of the latest metadata file to commit the changes. The process ensures data consistency and tracks modifications efficiently, adhering to the Iceberg table format's design.
+
+![image](https://github.com/user-attachments/assets/cdefd185-7f86-46e1-9e1a-4331e8ae5e8d)
+
+### Reading queries in Apache Iceberg
+
+When a read query is initiated, it is sent to the query engine first. The engine leverages the catalog to retrieve the latest metadata file location, which contains critical information about the table’s schema and other metadata files, such as the manifest list that ultimately leads to the actual datafiles. Statistical information about columns is used in this process to limit the number of files being read, which helps improve query performance.
+
+**The SELECT query**
+
+```
+# Spark SQL/Dremio Sonar
+SELECT *
+FROM orders
+WHERE order_ts BETWEEN '2023-01-01' AND '2023-01-31'
+```
+
+![image](https://github.com/user-attachments/assets/25c25ce5-9e5e-429c-8a7e-9e82930d98e0)
+
+1. The query engine interacts with the catalog to get the current metadata file (v3.metadata.json)
+2. It then gets the current-snapshot-id (S2 in this case) and the manifest list location for that snapshot
+3. The manifest file path is then retrieved from the manifest list
+4. The engine determines the datafile path based on the partition filter (20⁠23⁠-03-​07⁠-08⁠) from the manifest file
+5. The matching data from the required datafile is then returned to the user
+
+**The time-travel query**
+
+```
+# Spark SQL
+SELECT * FROM catalog.db.orders.history;
+# Dremio
+SELECT * FROM TABLE (table_history('orders'))
+```
+
+We get a table with cols like: made_current_at, snapshot_id, parent_id, is_current_ancestor
+
+To perform the time travel:
+
+```
+# Spark SQL
+SELECT * FROM orders
+TIMESTAMP AS OF '2023-03-07 20:45:08.914'
+# Dremio Sonar
+SELECT * FROM orders
+AT TIMESTAMP '2023-03-07 20:45:08.914'
+```
+
+or
+
+```
+# Spark SQL
+SELECT *
+FROM orders
+VERSION AS OF 8333017788700497002
+# Dremio
+SELECT *
+FROM orders
+AT SNAPSHOT 8333017788700497002
+```
+
+![image](https://github.com/user-attachments/assets/68719428-f51a-4ab1-b15d-9077f05f45e0)
+
+1. The query engine interacts with the catalog to get the current metadata file (v3.metadata.json)
+2. It then selects the snapshot (S1 in this case) based on either the timestamp or the version ID supplied in the time-travel query and gets the manifest list location for that snapshot
+3. The manifest file path is then retrieved from the manifest list
+4. The engine determines the datafile path based on the partition filter (20⁠23⁠-03-​07⁠-08⁠) from the manifest file
+5. The matching data from the required datafile is then returned to the user
+
+## [Testing in Spark by Zach Wilson](https://www.youtube.com/watch?v=QH1339Fy40Y)
+
+![Unit and e2e testing spark week 3 P1](https://github.com/user-attachments/assets/455f11b2-a020-4463-9d8e-bed8f09ff14f)
+![Unit and e2e testing spark week 3 P2](https://github.com/user-attachments/assets/a766b010-f3e1-4549-b1e9-893e234de094)
+
+Not that much notes for this one ~
+
+For the lab we used a library called chispa to create some unit tests. 
+
+Example:
+
+```python
+from pyspark.sql import SparkSession
+
+query = """
+
+WITH teams_deduped AS (
+    SELECT *, ROW_NUMBER() OVER(PARTITION BY team_id ORDER BY team_id) as row_num
+    FROM teams
+)
+SELECT
+    team_id AS identifier,
+    'team' AS `type`,
+    map(
+        'abbreviation', abbreviation,
+        'nickname', nickname,
+        'city', city,
+        'arena', arena,
+        'year_founded', CAST(yearfounded AS STRING)
+        ) AS properties
+FROM teams_deduped
+WHERE row_num = 1
+
+"""
+
+
+def do_team_vertex_transformation(spark, dataframe):
+    dataframe.createOrReplaceTempView("teams")
+    return spark.sql(query)
+
+
+def main():
+    spark = SparkSession.builder \
+        .master("local") \
+        .appName("players_scd") \
+        .getOrCreate()
+    output_df = do_team_vertex_transformation(spark, spark.table("players"))
+    output_df.write.mode("overwrite").insertInto("players_scd")
+```
+
+And the test:
+
+```python
+from chispa.dataframe_comparer import *
+
+from ..jobs.team_vertex_job import do_team_vertex_transformation
+from collections import namedtuple
+
+TeamVertex = namedtuple("TeamVertex", "identifier type properties")
+Team = namedtuple("Team", "team_id abbreviation nickname city arena yearfounded")
+
+
+def test_vertex_generation(spark):
+    input_data = [
+        Team(1, "GSW", "Warriors", "San Francisco", "Chase Center", 1900),
+        Team(1, "GSW", "Bad Warriors", "San Francisco", "Chase Center", 1900),
+    ]
+
+    input_dataframe = spark.createDataFrame(input_data)
+    actual_df = do_team_vertex_transformation(spark, input_dataframe)
+    expected_output = [
+        TeamVertex(
+            identifier=1,
+            type='team',
+            properties={
+                'abbreviation': 'GSW',
+                'nickname': 'Warriors',
+                'city': 'San Francisco',
+                'arena': 'Chase Center',
+                'year_founded': '1900'
+            }
+        )
+    ]
+    expected_df = spark.createDataFrame(expected_output)
+    assert_df_equality(actual_df, expected_df, ignore_nullable=True)
+```
+
+And the homework is to convert 2 more queries from week 1-2 to spark jobs and write a test for them but I will do that in the coming days ~
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 329938bad119a9582e500ccc2b01c951f5b1d469 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 4 Dec 2024 00:21:45 +0900
Subject: [PATCH 518/638] Create 2024-12-03-day-337.md

---
 _posts/2024-12-03-day-337.md | 225 +++++++++++++++++++++++++++++++++++
 1 file changed, 225 insertions(+)
 create mode 100644 _posts/2024-12-03-day-337.md

diff --git a/_posts/2024-12-03-day-337.md b/_posts/2024-12-03-day-337.md
new file mode 100644
index 000000000..31b94d9c0
--- /dev/null
+++ b/_posts/2024-12-03-day-337.md
@@ -0,0 +1,225 @@
+---
+layout: post
+title: (Day 337) More about Iceberg and DQ
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 337!
+A quick summary of today:
+* Apache Iceberg: The Definitive Guide - Chapter 4 - Optimizing the Performance of Iceberg Tables
+* Data Quality Fundamentals - Chapter 6 - Fixing Data Quality Issues at Scale
+
+## Apache Iceberg: The Definitive Guide - Chapter 4 - Optimizing the Performance of Iceberg Tables
+
+### Compaction
+
+Compaction takes many smaller files and processes them into fewer bigger files
+
+![image](https://github.com/user-attachments/assets/9b418417-63b0-48e6-abc6-af1073686256)
+
+### Hands-on with Compaction
+
+**Compaction strategies**
+
+| Strategy  | What it does | Pros | Cons |
+|-----------|--------------|------|------|
+| **Binpack** | Combines files only; no global sorting (will do local sorting within tasks) | This offers the fastest compaction jobs. | Data is not clustered. |
+| **Sort** | Sorts by one or more fields sequentially prior to allocating tasks (e.g., sort by field a, then within that, sort by field b) | Data clustered by often queried fields can lead to much faster read times. | This results in longer compaction jobs compared to binpack. |
+| **Z-order** | Sorts by multiple fields that are equally weighted, prior to allocating tasks (X and Y values in this range are in one grouping; those in another range are in another grouping) | If queries often rely on filters on multiple fields, this can improve read times even further. | This results in longer-running compaction jobs compared to binpack. |
+
+### Sorting
+
+Sorting (or "clustering") data improves query performance by reducing the number of files that need to be scanned. When data is sorted, similar values are grouped into fewer files, making queries more efficient. For example, if NFL player data is sorted by team, querying for players from the Detroit Lions will require scanning fewer files compared to unsorted data.
+
+![image](https://github.com/user-attachments/assets/af6c797e-5b61-483a-ad45-340c20d77eaa)
+
+Sorted datasets results in scanning fewer datafiles
+
+We can combine compacting and sorting:
+
+![image](https://github.com/user-attachments/assets/be821343-f630-40dd-9bce-35cbcac1f50c)
+
+### Z-order
+
+There are times when multiple fields are a priority when querying a table, and this is where a z-order sort may be quite helpful. With a z-order sort you are sorting the data by multiple data points, which allows engines a greater ability to reduce the files scanned in the final query plan
+
+![image](https://github.com/user-attachments/assets/db538ba0-92a4-4a5c-a04e-dd023ee502ac)
+
+So if we know what data we are looking for based on fields we z-ordered by, we can possibly avoid searching large portions of the data since it’s sorted by both fields. We can then take that quadrant and break it down even further and apply another z-order sort to the data in the quadrant, as shown in “C” in the figure. Since our search is based on multiple factors (X and Y), we could eliminate 75% of the searchable area by taking this approach.
+
+Usage:
+
+```sql
+CALL catalog.system.rewrite_data_files(
+  table => 'people',
+  strategy => 'sort',
+  sort_order => 'zorder(age,height)'
+)
+```
+
+Using the sort and z-order compaction strategies not only allows you to reduce the number of files your data exists in, but also makes sure the order of the data in those files enables even more efficient query planning.
+
+While sorting is effective, it comes with some challenges. First, as new data is ingested, it becomes unsorted, and until the next compaction job, the data remains somewhat scattered across multiple files. This occurs because new data is added to a new file and is potentially sorted within that file but not in the context of all previous records. Second, files may still contain data for multiple values of the sorted field, which can be inefficient for queries that only require data with a specific value.
+
+### Partitioning
+
+If you know a particular field is pivotal to how the data is accessed, you may want to go beyond sorting and into partitioning. When a table is partitioned, instead of just sorting the order based on a field, it will write records with distinct values of the target field into their own datafiles.
+
+For example in politics, we may query a table a lot by party
+
+![image](https://github.com/user-attachments/assets/b0bf7c56-6846-45a6-9f78-17cd4cb291b9)
+
+### Copy-on-Write Versus Merge-on-Read
+
+Another consideration when it comes to the speed of your workloads is how you handle row-level updates. When you are adding new data, it just gets added to a new datafile, but when you want to update preexisting rows to either update or delete them, there are some considerations you need to be aware of:
+
+* in data lakes, and therefore in Apache Iceberg, datafiles are immutable, meaning they can’t be changed. This provides lots of benefits, such as the ability to achieve snapshot isolation (since files that old snapshots refer to will have consistent data)
+* if you’re updating 10 rows, there is no guarantee they are in the same file, so you may have to rewrite 10 files and every row of data in them to update 10 rows for the new snapshot
+
+There are 3 approaches:
+
+| Update style                   | Read speed       | Write speed          | Best practice                                                   |
+|---------------------------------|------------------|----------------------|-----------------------------------------------------------------|
+| Copy-on-write                   | Fastest reads    | Slowest updates/deletes |                                                                 |
+| Merge-on-read (position deletes)| Fast reads       | Fast updates/deletes  | Use regular compaction to minimize read costs.                  |
+| Merge-on-read (equality deletes)| Slow reads       | Fastest updates/deletes | Use more frequent compaction to minimize read costs.            |
+
+
+**Copy-on-Write**
+
+![image](https://github.com/user-attachments/assets/1ea63fce-5f87-497e-b5a3-524991b70a26)
+
+**Merge-on-Read**
+
+If you are deleting a record:
+
+* the record is listed in a delete file
+* when a reader reads the table, it will reconcile the datafile with the delete file
+
+If you are updating a record:
+
+* the record to be updated is tracked in a delete file
+* a new datafile is created with only the updated record
+* when a reader reads the table, it will ignore the old version of the record because of the delete file and use the new version in the new datafile
+
+![image](https://github.com/user-attachments/assets/90e44654-2fed-4dd1-ba52-722b78c50093)
+
+When you have a ton of data and you want to kick out a specific row, you have a couple of options:
+
+* we can look for the row data based on where it’s sitting in the dataset, kind of like finding your friend in a movie theater by their seat number, or
+* we can look for the row data based on what it’s made of, like picking out your friend in a crowd because they’re wearing a bright red hat
+
+If we use the first option, we’ll use what are called positional delete files. But if we use the second option, we’ll need equality delete files.
+
+### Other considerations
+
+This section outlines various techniques for optimizing performance when working with data files and tables, particularly in systems like Apache Iceberg. Key areas discussed include:
+
+**Metrics collection**:
+   - metrics like counts, bounds, and distinct values are tracked to optimize queries
+   - overhead from metrics tracking in wide tables can be reduced by customizing column-specific metrics using table properties (e.g., `none`, `counts`, `truncate`, `full`)
+
+**Rewriting manifests**:
+   - consolidates manifest files to reduce overhead when listing data files without rewriting the data itself
+   - helps when datafile sizes are optimal but too many manifests exist
+   - spark caching can be disabled to handle memory issues during this process
+
+**Optimizing storage**:
+   - periodic snapshot expiration removes old, unnecessary data files to save space but disables time-travel to expired snapshots
+   - orphan files created by failed jobs can be identified and removed with a specific procedure
+
+**Write distribution mode**:
+   - optimizes file writes by controlling how records are distributed across tasks
+   - distribution modes include `none` (fastest), `hash` (grouped by partition key), and `range` (grouped by sort order)
+   - custom distribution modes can be specified for updates, deletes, and merges
+
+**Object storage optimization**:
+   - addresses request throttling issues in object storage by distributing files across multiple prefixes using hashing
+   - improves scalability and concurrency for large partitions
+
+**Bloom filters**:
+   - efficiently checks for the potential presence of values in datasets to reduce unnecessary scans
+   - implemented via table properties, enabling faster queries by allowing engines to skip irrelevant data files
+
+
+## Data Quality Fundamentals - Chapter 6 - Fixing Data Quality Issues at Scale
+
+### Fixing quality issues in software development
+
+Principles from DevOps and site reliability engineering (SRE) can be applied to improve data quality management at scale. DevOps uses a continuous feedback loop with eight stages: plan, code, build, test, release, deploy, operate, and monitor. This cycle ensures reliable, performant software aligned with business objectives.
+
+Adopting similar incident management practices for data systems allows teams to proactively address data quality issues, moving beyond reactive approaches. Effective incident management involves identifying, resolving, analyzing, and preventing disruptions to maintain operational continuity. By integrating these practices and investing in appropriate tools, data teams can enhance analytics scalability and reliability.
+
+### Data incident management
+
+As data systems become increasingly distributed and companies ingest more and more data, the opportunity for error (and incidents) only increases. For decades, software engineering teams have relied on a multistep process to detect, resolve, and prevent issues from taking down their applications. As data operations mature, it’s time we treat our data systems with the same diligence, particularly when it comes to building more reliable pipelines.
+
+The authors recommend to use the 'data reliability life cycle', inspired by DevOps practices;
+
+![image](https://github.com/user-attachments/assets/49159ffb-e84f-46d1-a523-98cd6b1e05d9)
+
+#### Incident detection  
+
+* anomaly detection identifies deviations from normal data patterns  
+* anomaly detection should be implemented across the entire data ecosystem not just in isolated areas  
+* while essential anomaly detection alone is insufficient for effective incident management it should be combined with testing versioning observability lineage and other tools and processes  
+
+#### Incident response  
+
+* effective communication is key to a successful incident response  
+* data teams should develop runbooks instructions for specific services and playbooks step-by-step procedures for handling incidents to streamline incident response  
+* role delegation similar to traditional site reliability engineering can improve incident response this includes an incident commander to coordinate tasks and communication and an incident responder to troubleshoot the issue  
+* end-to-end lineage helps to understand the impact of data downtime and quickly communicate with affected teams  
+
+#### Root cause analysis (RCA)
+
+* identifying the root cause of data incidents is often complex and requires a holistic approach  
+* the 'five whys' approach can be a helpful framework for rca this involves repeatedly asking "why" to drill down to the root cause  
+* the sources suggest a five-step process for rca  
+    1. examine the lineage to identify the upstream source of the problem  
+    2. review the code that generated the affected table to identify potential issues  
+    3. analyse the data for patterns and clues related to the incident  
+    4. investigate the operational environment for errors delays or infrastructure issues  
+    5. consult with peers for insights and past experiences with similar incidents  
+
+#### Resolution  
+
+* once the root cause is identified the next step is to resolve the issue and communicate the resolution to stakeholders  
+* resolutions can involve pausing and restarting pipelines troubleshooting code or implementing more permanent solutions  
+* continuous communication through dedicated channels is essential during the resolution process  
+
+#### Blameless postmortem  
+
+* a postmortem meeting along with documentation should be conducted after resolving an incident to review the incident identify lessons learned and improve future responses  
+* postmortems should focus on learning and improvement rather than assigning blame  
+* postmortems are opportunities to  
+    * assess incident readiness and update runbooks  
+    * review monitoring alerting and workflow tools  
+    * document the incident and share it with the data team  
+    * revisit service-level agreements slas  
+    * improve system resilience and prevent future data downtime  
+
+### Incident response and mitigation
+
+Proactive approach to data reliability that incorporates testing, CI/CD, discovery, and observability, necessary components of managing and preventing data incidents
+
+![image](https://github.com/user-attachments/assets/c5e30f4e-e580-47ee-8059-598bf47ddde5)
+
+#### Establishing a routine of incident management
+
+Establishing a routine for incident management ensures data reliability by assigning clear responsibilities and improving response times. A key role is the **incident commander**, who coordinates efforts, assesses severity, and maintains communication during data issues. Rotating commanders weekly or assigning them to specific datasets fosters accountability.  
+
+#### Key steps:  
+
+1. route notifications to relevant team members: centralized or decentralized teams must quickly identify and notify the appropriate stakeholders using tools like Slack
+2. assess incident severity: prioritize issues based on their business impact using lineage tools to identify critical data assets and reduce time spent on non-essential issues  
+3. communicate status updates: use runbooks, status pages, and clear delegation to keep stakeholders informed and reduce duplication of effort  
+4. define and align on SLOs and SLIs: measure metrics like time to detection (TTD) and time to resolution (TTR) to enhance reliability and prevent future downtime  
+
+Incident commanders, supported by automation and structured processes, play a crucial role in minimizing downtime and improving data pipeline resilience.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 1bea4c958d3dbe65e060b239c40a1967a7bd3b25 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 4 Dec 2024 23:05:24 +0900
Subject: [PATCH 519/638] Create 2024-12-04-day-338.md

---
 _posts/2024-12-04-day-338.md | 199 +++++++++++++++++++++++++++++++++++
 1 file changed, 199 insertions(+)
 create mode 100644 _posts/2024-12-04-day-338.md

diff --git a/_posts/2024-12-04-day-338.md b/_posts/2024-12-04-day-338.md
new file mode 100644
index 000000000..2cb82ddbd
--- /dev/null
+++ b/_posts/2024-12-04-day-338.md
@@ -0,0 +1,199 @@
+---
+layout: post
+title: (Day 338) Going deeper into data lakehouses and Iceberg
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 338!
+A quick summary of today:
+* Apache Iceberg: The Definitive Guide - Chapter 5 - Iceberg Catalogs
+* Data Quality Fundamentals - Chapter 7 - Building End-to-End Lineage
+* Streamed - creating a local data lakehouse
+
+---
+
+## Apache Iceberg: The Definitive Guide - Chapter 5 - Iceberg Catalogs
+
+### Requirements of an Iceberg catalog
+
+Iceberg catalogs require implementations of basic functions: listing, creating, dropping, checking existence, and renaming tables. Catalogs must map a table path (e.g. `db1.table1`) to the filepath of the current metadata file, with different implementations achieving this differently, such as using a version-hint.text file in filesystem-based catalogs or a location property in Hive Metastore.
+
+While these basic requirements enable functionality, production usage demands additional safeguards, particularly support for atomic operations. This ensures consistent table states for readers and writers and prevents data loss during concurrent writes, a critical feature for avoiding business impacts in production environments.
+
+### Catalog comparison
+
+#### Hadoop
+
+Simplest to start with, uses filesystem-based storage.
+
+- Pros: no external systems needed; works with various filesystems (HDFS, S3, ADLS, GCS)
+- Cons: 
+  - limited production suitability
+  - requires atomic rename operations (e.g., S3 needs DynamoDB workaround)
+  - restricted to one warehouse directory
+  - performance issues with many namespaces/tables
+  - dropping tables also deletes data
+
+Best for quick starts, experimentation, or non-production environments.
+
+#### Hive Catalog
+
+Uses Hive Metastore for metadata management.
+
+- Pros: 
+  - compatible with many tools
+  - cloud-agnostic
+- Cons:
+  - requires user-managed Hive Metastore
+  - no support for multitable transactions
+
+Suitable if Hive Metastore is already part of the infrastructure and multitable transactions aren't required.
+
+#### AWS Glue Catalog
+
+Managed service leveraging AWS Glue Data Catalog.
+
+- Pros:
+  - reduces operational overhead
+  - tight integration with AWS services
+- Cons:
+  - no multitable transaction support
+  - AWS-specific, complicating multicloud scenarios
+
+Ideal for AWS-centric setups without multitable transaction needs.
+
+#### Nessie
+
+Git-like version control for data, enabling multitable transactions.
+
+- Pros:
+  - supports multitable/multistatement transactions
+  - data-as-code paradigm
+  - cloud-agnostic
+- Cons:
+  - limited engine/tool support
+  - requires user-managed infrastructure (or hosted options like Dremio)
+
+Excellent for cloud-agnostic setups requiring version control and transactional consistency.
+
+#### REST Catalog
+
+Interface-based catalog using RESTful services.
+
+- Pros:
+  - minimal dependencies
+  - highly flexible and customizable
+  - supports multitable transactions
+  - cloud-agnostic
+- Cons:
+  - requires self-managed process and data store
+  - no public backend implementation; must build or use hosted services (e.g., Tabular)
+  - limited engine/tool support
+
+Best for flexibility, cloud-agnostic goals, and multitable transaction needs.
+
+#### JDBC Catalog
+
+Uses JDBC-compliant databases (e.g. MySQL, PostgreSQL)
+
+- Pros:
+  - easy setup with existing infrastructure
+  - high availability with cloud-hosted databases
+  - cloud-agnostic
+- Cons:
+  - no multitable transaction support
+  - tools must package JDBC drivers
+ 
+Suitable for environments with existing JDBC databases, focusing on simplicity and availability
+
+### Catalog migration
+
+Migrating catalogs in Apache Iceberg is straightforward due to the table data residing in data lake storage. This flexibility reduces vendor lock-in risks, future-proofs data, and allows for seamless transitions to better or more cost-effective catalog solutions. Common scenarios for migration include moving from an experimental catalog to a production-ready one, utilizing advanced features of a new catalog, or shifting environments (e.g. on-premises to cloud).
+
+1. lightweight migration: migration involves remapping the table path to metadata without data duplication. However, surrounding factors like active jobs and tool dependencies must be managed
+2. testing before full migration: tables can be registered in a new catalog for testing while still active in the current one. However, avoid making changes in the new catalog during this phase to prevent conflicts
+
+#### Migration Methods
+
+* using the Iceberg CLI tool
+* using an engine (like Apache Spark)
+
+Both methods retain the table’s full history, enabling operations like time travel and metadata inspection post-migration. Proper planning ensures smooth transitions and avoids data integrity issues.
+
+## Hands-on with Iceberg - Chapter 6, 7, 8, 9
+
+This chapter was just following [this notebook](https://github.com/developer-advocacy-dremio/definitive-guide-to-apache-iceberg/blob/main/Resources/Chapter_6/Chapter6Spark.ipynb) on the book's repo after setting up minio, dremio and spark-iceberg. The chapter showed the SparkSQL and Dataframe API for a variety of READ/WRITE/UPDATE/DELETE queries - all of the SparkSQL queries are in the notebook. 
+
+Chapter 7 was just executing queries in Dremio's query engine - just showing it's flexibility and that we can execute queries there as well.
+
+Chapter 8 - some basic setup on setting up AWS Glue jobs for Spark ETL jobs, however it was very short (suspeciously short to be honest) and it was about doing things in the AWS console which I cannot really follow at the moment.
+
+Chapter 9 - Flink set up and basic READ/WRITE/UPDATE/DELETE operations.
+
+I think these 2 videos are way better to get practice - [Building a Data Lakehouse with Apache Iceberg, Spark, Dremio, Nessie & Minio](https://www.youtube.com/watch?v=X3wfVaSQS_c&pp=ygUtZHJlbWlvIGFwYWNoZSBpY2ViZXJnIGRhdGEgbGFrZWhvdXNlIHR1dG9yaWFs) and [Ingesting Data Using Apache Flink into Apache Iceberg with a Nessie Catalog](https://youtu.be/epJY4CQx55k)
+(and I bookmarked them to checkout on stream some time)
+
+---
+
+## Data Quality Fundamentals - Chapter 7 - Building End-to-End Lineage
+
+### Building E2E field-level lineage for modern data systems
+
+Example table-level lineage with upstream and downstream connections between objects in the data warehouse and tables
+
+![image](https://github.com/user-attachments/assets/f2fe767b-f9f7-4a95-8142-8cbc03cf7d99)
+
+Due to the complexity of even the most basic SQL queries, however, building data lineage from scratch is no easy feat. Historically, lineage is parsed manually, requiring an almost encyclopedic knowledge of a company’s data environment and how each component interacts with each other.
+
+**Basic lineage requirements**
+
+To enable fast value realization, data teams need field-level data lineage to quickly identify and address impacts on downstream fields and reports. A secure architecture should prioritize metadata access over direct user data or PII handling. Automation is essential for updating data assets dynamically, reducing manual effort. Effective solutions must integrate with popular data tools across the entire pipeline, including warehouses, transformation tools, and BI platforms. Column-level lineage is critical for understanding data anomalies and requires detailed parsing beyond table-level lineage approaches.
+
+In an analytics capacity, data lineage can be used for a variety of applications, including:
+
+* reviewing suspicious numbers in a revenue report
+* reducing data debt
+* managing PII
+
+**Data lineage design**
+
+At a foundational level, the structure of most lineages incorporates three elements
+
+* the destination table, stored in the downstream report
+* the destination fields, stored in the destination table
+* the source tables, stored in the data warehouse
+
+![image](https://github.com/user-attachments/assets/314ad9d8-91d8-4c4f-bdec-b3f113833ef0)
+
+![image](https://github.com/user-attachments/assets/828e49e5-f8e9-4971-b050-502a81ede3da)
+
+Like any agile development process, building field-level lineage is an exercise in prioritization, listening, and quick iteration. Here are some best practices when building lineage
+
+* listen to your teammates and take everyone’s advice into consideration
+* invest in prototyping
+* ship and iterate
+
+### Case study: Architecting for data reliability at Fox
+
+Alex Tverdohleb, VP of Data Services at Fox Networks, transformed the organization’s data architecture to enable self-serve analytics while ensuring reliability and trust. He developed a hybrid decentralized approach, termed "controlled freedom," that allows internal stakeholders to access and use trustworthy data independently, without relying heavily on centralized data teams.  
+
+Key strategies include:  
+
+- controlled freedom: setting strict parameters for data ingestion, security, and optimization, while granting users freedom to explore data for ad hoc analytics
+- decentralized team structure: five specialized teams (data tagging, engineering, analytics, science, and architecture) collaborate to meet business needs, with analysts bridging the gap between business requirements and engineering execution
+- purpose-driven technology: adopting a lakehouse architecture optimized for self-service, avoiding unnecessary "shiny" tools. Data flows through three layers—raw, optimized, and published—to ensure usability and consistency  
+- data observability: leveraging tools like Monte Carlo for QA, validation, and alerting to monitor data quality and proactively address incidents, ensuring trust and transparency
+
+---
+
+## [Streamed - creating a local data lakehouse](https://www.youtube.com/watch?v=vCNdcjmjAtU&t=2s)
+
+I followed the blog version of this - [Building a Data Lakehouse with Apache Iceberg, Spark, Dremio, Nessie & Minio](https://www.youtube.com/watch?v=X3wfVaSQS_c&pp=ygUtZHJlbWlvIGFwYWNoZSBpY2ViZXJnIGRhdGEgbGFrZWhvdXNlIHR1dG9yaWFs) video and explored dremio, nessie, minio and how data files, metadata, and manifest files get stored after running and inserting some queries. 
+
+After stream I casually watched [this series of videos](https://www.youtube.com/watch?v=wepFB_WP_9g&list=PLsLAVBjQJO0qVfGet6FEQw-nZ6ygLtYuH) of the same authour as the above blog and one of the autors of the Apache Iceberg book. They are 16 in total and are very well made but have less than 1000 views each :( 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From b9f2a51c9bbf4ac4fc09df6e299997edecb73ad3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 5 Dec 2024 22:10:18 +0900
Subject: [PATCH 520/638] Create 2024-12-05-day-339.md

---
 _posts/2024-12-05-day-339.md | 800 +++++++++++++++++++++++++++++++++++
 1 file changed, 800 insertions(+)
 create mode 100644 _posts/2024-12-05-day-339.md

diff --git a/_posts/2024-12-05-day-339.md b/_posts/2024-12-05-day-339.md
new file mode 100644
index 000000000..dba5a61ba
--- /dev/null
+++ b/_posts/2024-12-05-day-339.md
@@ -0,0 +1,800 @@
+---
+layout: post
+title: (Day 339) Going deeper and deeper into DE
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 339!
+A quick summary of today:
+* finishing DQ Fundamentals
+* finishing Apache Iceberg: The Definitive Guide
+* finally reading Learning Spark 2nd ed
+
+---
+
+## Data Quality Fundamentals - Chapter 8 - Democratising DQ
+
+### Treating data like a product
+
+Regardless of what data the product visualizes, crunches, or puts to work, there are specific outcomes it should deliver:
+
+* increased data accessibility (surface data where people need it when they need it)
+* increased data democratization (make it easier for people to manipulate the data)
+* faster ROI on data (quicker insights)
+* time savings for the data team / data consumers
+* more precise insights (i.e. experimentation platforms)
+
+There are also important characteristics or qualities a data product should have:
+
+* reliability and observability
+* scalability
+* extensibility
+* usability
+* security and compliance
+* release discipline and roadmap
+
+**Applying the Data-as-a-Product approach**
+
+* gain stakeholder alignment early–and often
+* apply a product management mindset
+* invest in self-serve tooling
+* prioritize data quality and reliability
+* find the right team structure for your data organization
+
+### Building trust in your data platform
+
+* align your product’s goals with the goals of the business
+* gain feedback and buy-in from the right stakeholders
+* prioritise long-term growth and sustainability vs. short-term gains
+* sign off on baseline metrics for your data and how you measure them
+* know when to build vs buy
+
+### Assigning ownership for DQ
+
+Blast radius caused by data downtime
+
+![image](https://github.com/user-attachments/assets/0ad115fa-2fe2-440d-9cd9-7b8a87354e84)
+
+| Role                        | Facilitate Data Accessibility | Make It Easy to Interpret Data | Drive Insights and Recommendations Based on Data | Ensure Data Compliance | Maintain High Data Quality | Deliver on Data Reliability |
+|-----------------------------|--------------------------------|--------------------------------|------------------------------------------------|------------------------|---------------------------|-----------------------------|
+| CDO                        | A                             | A                              | A                                              | A                      | A                         | A                           |
+| Business Intelligence       | R                              | R                              | R                                              | I                      | C/I                       | C/I                         |
+| Analytics Engineer          | A                              | C                              | R                                              | I                      | A                         | R                           |
+| Data Science                | C                              | R                              | R                                              | I                      | R                         | C                           |
+| Data Engineering            | R                              | C                              | C                                              | I                      | R                         | R                           |
+| Data Governance             | C                              | I                              | C                                              | R                      | C                         | I                           |
+| Product Manager             | R                              | C                              | C                                              | C                      | R                         | R                           |
+
+_R = Responsible, A = Accountable, C = Consulted, and I = Informed_
+
+### Certifying data
+
+Data certification is the process by which data assets are approved for use across the organization after having met mutually agreed upon SLAs for data quality, observability, ownership/accountability, issue resolution, and communication.
+
+Data certification requirements vary based on the needs of the business, the capacity of the data engineering team, and the availability of data, but they typically incorporate features such as the following:
+
+* automated quality checks for freshness, volume, schema, and distribution
+* delivery SLAs with defined uptime
+* data owners who are accountable for investigating alerts
+* alerts routed to Slack (or email)
+* set communication process for outages
+
+### 7 steps to implementing a data certification program
+
+1. Build out your data observability capabilities
+2. Determine your data owners
+3. Understand what 'good' data looks like
+4. Set clear SLAs, SLOs, and SLIs for your most important data sets
+5. Develop your communication and incident management processes
+6. Determine a mechanism to tag the data as certified
+7. Train your data team and downstream consumers
+
+### Increasing data literacy
+
+* data catalogs
+* DBMS
+* data modelling tools
+* operational analytics dashboards
+
+### Prioritise data governance and compliance
+
+Data governance, the management of data across organizations, is increasingly critical due to regulations like GDPR and CCPA. It ensures data availability, usability, provenance, and security but often fails in execution due to outdated methods and scalability challenges in cloud-based systems. Data catalogs, serving as metadata repositories, are emerging as essential tools, aiding in data lineage, accessibility, and PII management.  
+
+Types of data catalogs:
+
+1. in-house: customizable but resource-intensive and prone to visibility issues over time
+2. third-party: often ML-powered with compliance integrations but sometimes criticized for poor usability
+3. open-source: flexible but manual-intensive, lacking off-the-shelf support for diverse stacks
+
+Despite their utility, data catalogs alone are insufficient for comprehensive governance. Modern programs require tools for data lineage, observability, and automated policy enforcement. Success also hinges on fostering a data culture that aligns teams with governance priorities. Collaboration across engineering, product, and analytics teams is essential to address governance gaps effectively and ensure data reliability at scale.
+
+### Building a DQ strategy
+
+**Make leadership accountable for DQ**
+
+Before you start trying to secure leadership and stakeholder buy-in, it’s important to be transparent about the current state of your data quality strategy. Consider how you might answer the following questions:
+
+* How do you measure the data quality of the assets your company collects and stores?
+* What are the key KPIs or goals you’re going to hold your data quality strategy accountable for meeting?
+* Do you have cross-functional involvement from leadership and data users in other parts of the company?
+* Who at the company will be held accountable for meeting your strategy’s KPIs and goals?
+* What checks and balances do you have to ensure KPIs are measured correctly and goals can be met?
+
+**Set DQ KPIs**
+
+* avoid focusing on data quality measurements. Instead, keep it simple. Measure for tangible metrics like completeness, freshness, accuracy, consistency, and validity as opposed to obscure 'accuracy' scores or other homegrown measurements
+
+**Spearhead a data governance program**
+
+* be sure to communicate how data quality affects their functional areas, from marketing to sales, and make it easy for them to share and enforce with their team. Focus on short-term or quick wins to get traction while promoting and executing on the long-term strategy
+
+**Automate your lineage and data governance tooling**
+
+* invest in automated tools that can quickly validate, monitor, and alert for data quality issues as they arise
+
+**Create a communications plan**
+
+* the final step is to put together a robust and comprehensive, program-level communications plan that will keep leadership in the loop, stakeholders aligned with your project’s progress, and data stewards abreast of their marching orders
+
+---
+
+## DQ Fundamentals - Chapter 9 - Data Quality in the Real World: Conversations and Case Studies
+
+### Building a data mesh for greater DQ
+
+Traditional systems like siloed data warehouses and lakes fail to meet the growing demands of modern organizations. A data mesh offers a solution by adopting a distributed, domain-oriented, and self-serve design inspired by microservices and domain-driven design principles.  
+
+Key features:
+
+1. domain-oriented ownership: data ownership is federated among domains, each responsible for managing their own ETL pipelines and treating data as a product. Functional domain owners handle ingestion, transformation, and aggregation of data for analytics or operational purposes
+
+2. self-serve platform: centralized infrastructure supports domain-specific pipelines, enabling teams to abstract technical complexities while maintaining autonomy over their processes
+
+3. interoperability and standards: a universal interoperability layer standardizes governance, data quality, formatting, and metadata to ensure cross-domain collaboration and consistency. Each domain adheres to service-level agreements and quality measures
+
+### Why implement a data mesh?
+
+The authors put together some Qs to help answer if a company should build a data mash
+
+* Quantity of data sources: How many data sources does your company have?
+* Size of your data team: How many data analysts, data engineers, and product managers (if any) do you have on your data team?
+* Number of data domains: How many functional teams (marketing, sales, operations, etc.) rely on your data sources to drive decision making, how many products does your company have, and how many data-driven features are being built? Add the total
+* Data engineering bottlenecks: How frequently is the data engineering team a bottleneck to the implementation of new data products on a scale of 1 to 10, with 1 being “never” and 10 being “always”?
+* Data governance: How much of a priority is data governance for your organization on a scale of 1 to 10, with 1 being “I could care less” and 10 being “It keeps me up all night”?
+
+Score breakdown:
+
+* 1–15: given the size and unidimensionality of your data ecosystem, you may not need a data mesh
+* 15–30: the organization is maturing rapidly and may even be at a crossroads in terms of really being able to lean into data. We strongly suggest incorporating some data mesh best practices and concepts so that a later migration might be easier
+* 30 or above: the data organization is an innovation driver for your company, and a data mesh will support any ongoing or future initiatives to democratize data and provide self-serve analytics across the enterprise
+
+### Case study insights from a game company
+
+* “Building your own data stack pays off, as it gives you all of these capabilities and enables you to be data-driven on your product development or working on your team.”
+* “We’ve been through a lot of iterations over our data platform, so you have to choose and be able to understand when it’s the right time to change technology, for which right amount of data, for which process that you’re running.”
+* “It’s very important to have a higher degree of data observability if you want to establish trust in your data. It’s important that you are able to understand when there is a problem, and that you’re able to indicate that easily.”
+* “It’s important to get the basics right before advancing to more advanced data applications. In our case, we should have hired analysts earlier to make more use of the data.”
+* “Establishing a data-driven culture is quite important and sometimes even more important than building the right tech stack.”
+
+### Making metadata work for the business
+
+Lineage can be useful in:
+
+* how to understand data changes that will impact consumers and determine the best course of action to resolve that use case
+* how to troubleshoot the root cause of an issue when data assets break
+* how to communicate the impact of broken data to consumers
+
+The metadata (including but not limited to lineage) should answer more than the basic “who, what, where, when, why?” about your data. It should enable your customers (internal or external) to be equipped with up-to-date and accurate answers to questions that relate back to your customer’s pain points and use cases, including:
+
+* does this data matter?
+* what does this data represent?
+* is this data relevant and important to my stakeholders?
+* can I use this data in a secure and compliant way?
+* where does the answer to this question come from?
+* who is relying on this asset when I’m making a change to it?
+* can we trust this data?
+
+The true power of metadata lies in where, when, and how we use it—specifically, how we apply it to a specific, timely problem we are trying to solve. In addition to collecting metadata and building metadata solutions, data teams also need to ask themselves:
+
+* what purpose is this metadata serving?
+* how can I apply it to solve real and relevant customer pain points?
+
+### Deciding when to get started with DQ at your company
+
+* you've recently migrated to the cloud
+* your data stack is scaling with more data sources, more tables, and more complexity
+* your data team is growing
+* your team is spending at least 30% of their time firefighting DQ issues
+* your team has more data consumers than they did 1 yr ago
+* your company is moving to a self-service analytics model
+* data is a key part of the customer value proposition
+
+---
+
+## Apache Iceberg: The Definitive Guide - Chapter 10 - Apache Iceberg in Production
+
+### Apache Iceberg Metadata Tables
+
+One of the most powerful features of Apache Iceberg is that from its robust metadata, several metadata tables can be generated that can be used to help monitor the health of the table and diagnose where bottlenecks may exist.
+
+**The history table**
+
+The history metadata table records the table’s evolution. Each of the four fields in this table provides unique insights into the table’s history
+
+* the snapshot_id field serves as a unique identifier for each snapshot. This identifier enables you to track and reference specific snapshots within the table’s history
+* the parent_id field provides the unique ID of the parent snapshot of the current snapshot. This effectively maps out the lineage of each snapshot, thus facilitating the tracking of the table’s evolution over time
+* the is_current_ancestor field indicates whether a snapshot is an ancestor of the table’s current snapshot. This boolean value (true or false) helps identify snapshots that are part of the table’s present state lineage and those that have been invalidated from table rollbacks
+
+We can use the history metadata table for data recovery and version control as well as to identify table rollbacks.
+
+**The metadata_log_entries table**
+
+The metadata_log_entries metadata table keeps track of the evolution of the table by logging the metadata files generated during table updates.
+
+* the timestamp field records the exact date and time when the metadata was updated
+* the file field indicates the location of the datafile that corresponds to that particular metadata log entry
+* the latest_snapshot_id field provides the identifier of the most recent snapshot at the time of the metadata update
+* the latest_schema_id field contains the ID of the schema being used when the metadata log entry was created
+* the latest_sequence_number field signifies the order of the metadata updates
+
+We can use the metadata_log_-entries metadata table to find the latest snapshot with a previous schema
+
+**The snapshots table**
+
+The snapshots metadata table is essential for tracking dataset versions and histories. It maintains metadata about every snapshot for a given table, representing a consistent view of the dataset at a specific time.
+
+* the committed_at field signifies the precise timestamp when the snapshot was created
+* the snapshot_id field is a unique identifier for each snapshot
+* the operation field lists a string of the types of operations that occurred, such as APPEND and OVERWRITE
+* the parent_id field links to the snapshot ID of the snapshot’s parent
+* the manifest_list field offers detailed insights into the files comprising the snapshot. It’s like a directory or inventory that keeps a record of all the datafiles associated with a given snapshot
+* the summary field holds metrics about the snapshot, such as the number of added or deleted files, number of records, and other statistical data that provides a quick glance into the snapshot’s content
+
+**The files table**
+
+The files metadata table showcases the current datafiles within a table and furnishes detailed information about each of them, from their location and format to their content and partitioning specifics.
+
+* content, represents the type of content in the file, with a 0 signifying a datafile, 1 a position delete file, and 2 an equality delete file
+* file_path gives the exact location of each file
+* file_format field indicates the format of the datafile
+* spec_id field corresponds to the partition spec ID that the file adheres to
+* partition field provides a representation of the datafile’s specific partition
+* record_count field reports the number of records contained within each file
+* file_size_in_bytes field provides the total size of the file in bytes, while column_sizes furnishes the sizes of the individual columns
+* value_counts, null_value_counts, and nan_value_counts
+* lower_bounds and upper_bounds fields hold the minimum and maximum values in each column
+* key_metadata field contains implementation-specific metadata, if any exists
+* split_offsets field provides the offsets at which the file is split into smaller segments for parallel processing
+* equality_ids and sort_order_id fields correspond to the IDs relating to equality delete files, if any exist, and the IDs of the table’s sort order, if it has one
+
+**The manifests table**
+
+The manifests metadata table details each of the table’s current manifest files. This table offers an array of useful information that assists in understanding the table’s structure and changes over time.
+
+* the path field provides the filepath where the manifest is stored
+* the length field shows the size of the manifest file
+* the partition_spec_id field indicates the specification ID of the partition that the manifest file is associated with
+* the added_snapshot_id field provides the ID of the snapshot that added this manifest file, offering a link between snapshots and manifests
+* three count fields—added_data_files_count, existing_data_files_count, and deleted_data_files_count—respectively relay the number of new files added in this manifest, the number of existing datafiles that were added in previous snapshots, and the number of files deleted in this manifest
+* the partition_summaries field is an array of field_summary structs that summarize partition-level statistics. It contains the following information: contains_null, contains_nan, lower_bound, and upper_bound
+
+With the manifests metadata table, users can perform various operations, including finding manifests that need rewriting, summing the total number of files added per snapshot, finding snapshots where files were deleted, and determining whether the table is sorted well.
+
+**The partitions table**
+
+The partitions metadata table provides a snapshot of how the data in a table is divided into distinct, nonoverlapping regions, known as partitions. Each row represents a specific partition within the table.
+
+* partition, represents the actual partition values, usually based on certain columns of your data
+* the record_count field indicates the total number of records within a given partition
+* the file_count field gives the total number of datafiles present in the partition
+* the spec_id field corresponds to the ID of the partition specification used to generate this partition
+
+For unpartitioned tables, the partitions metadata table will have a single record that will contain only the record_count and file_count fields, as no partitioning is applied to such tables
+
+While the partitions metadata table provides a snapshot of the current state of partitions, it’s important to note that delete files are not applied. As a result, in certain scenarios, partitions may be listed even though all their data rows have been marked for deletion by delete files
+
+There are many use cases for the partitions metadata table, including finding how many files are in a partition, summing the total size in bytes of a partition, and finding the number of partitions per partition scheme. For instance, you may want to see how many files are in a partition, because if a particular partition has a large number of files, it may be a candidate for compaction. 
+
+**The all_data_files table**
+
+It provides comprehensive details about every datafile across all valid snapshots in the table.
+
+* the first field, content, signifies the type of the file. A value of 0 indicates a datafile, 1 a position delete file, and 2 an equality delete file
+* the file_path field is a string that represents the complete path to the datafile
+* the file_format field indicates the format of the datafile
+* the spec_id field corresponds to the ID of the partition specification used to generate this partition
+* the partition field represents the partition to which this datafile belongs
+* the record_count field gives the total number of records within the file, while file_size_in_bytes represents the size of the datafile in bytes
+* the column_sizes field provides a map between the column ID and the size of that column in bytes
+* the value_counts field gives a map that represents the total count of values for each column in the datafile
+* the lower_bounds and upper_bounds fields are maps that store the minimum and maximum values for each column in the datafile
+* the key_metadata field contains implementation-specific metadata
+* the split_offsets field provides information about split points within the file
+* the equality_ids field relates to equality deletes and helps in identifying rows deleted by equality deletes
+* the sort_order_id field contains the ID of the sort order used to write the datafile
+* the readable_metrics field is a derived field that provides a human-readable representation of the file’s metadata including column size, value counts, null counts, and lower and upper bounds
+
+Some of the use cases include finding the largest table across all snapshots, finding the total file size across all snapshots, and assessing partitions across snapshots.
+
+**The all_manifests table**
+
+It provides detailed insights into every manifest file across all valid snapshots in the table
+
+* the first field, content, signifies the type of the file, similar to the all_data_files table. A value of 0 indicates the manifest tracks datafiles; a value of 1 indicates that it tracks delete files
+* the path field is a string representing the complete path to the manifest file
+* the length field represents the size of the manifest file in bytes
+* the partition_spec_id field corresponds to the ID of the partition specification used to write this manifest file
+* the added_snapshot_id field represents the ID of the snapshot when the manifest was created
+* the partition_summaries field is an array of structures, where each structure provides a summary for a specific partition in the manifest file
+* the reference_snapshot_id field represents the ID of the snapshot that this record is associated with
+
+Some of the use cases for this table include finding all manifests for a particular snapshot, monitoring the growth of manifests from snapshot to snapshot, and getting the total size of all valid manifests.
+
+**The refs table**
+
+It provides a list of all the named references within an Iceberg table. Named references can be thought of as pointers to specific snapshots of the table data, providing an ability to bookmark or version the table state.
+
+* the first field, name, represents the unique identifier for a named reference. Named references are categorized into two types
+* the type can be one of two values: BRANCH, a mutable reference that can be moved to a new snapshot, or TAG, an immutable reference that, once created, always points to the same snapshot
+* the max_reference_age_in_ms field indicates the maximum duration in milliseconds that a snapshot can be referenced. This age is measured from the time the snapshot was added to the table. If the age of a snapshot exceeds this duration, it will no longer be valid and will be a candidate for cleanup during maintenance operations
+* the min_snapshots_to_keep field provides a lower limit on the number of snapshots to keep in the table history. The Iceberg table will always maintain at least this many snapshots, even if they are older than the max_snapshot_age_ms setting
+* the max_snapshot_age_in_ms field indicates the maximum age in milliseconds for any snapshot in the table. Snapshots that exceed this age could be removed by the maintenance operations, unless they are protected by the min_snapshots_to_keep setting
+
+The refs metadata table helps you understand and manage your table’s snapshot history and retention policy, making it a crucial part of maintaining data versioning and ensuring that your table’s size is under control
+
+**The entries table**
+
+It offers insightful details about each operation that has been performed on the table’s data and deletes files across all snapshots. Each row in this table captures operations that affected many files at a certain point in the table’s history, making it an essential resource for understanding the evolution of your dataset.
+
+* the first field, status, is an integer that indicates whether a file was added or deleted in the snapshot. A value of 0 represents an existing file, while 1 indicates an added file and 2 a deleted file
+* snapshot_id is the unique identifier of the snapshot in which the operation took place
+* the sequence_number field indicates the order of operations. This is a global counter across all snapshots of the table, and it increments for each change made, whether the change is an addition, a modification, or a deletion
+* data_file is a struct that encapsulates extensive details about the file involved in the operation
+
+**Using the metadata tables in conjunction**
+
+We can join the above tables together and gain extra insight. 
+
+* get data on all the files added in a snapshot
+* get a detailed overview of the lifecycle of a particular datafile
+* track the evolution of the table by partition across snapshots
+* monitor files associated with a particular branch
+* find file differences between two branches of a table
+* find the growth in storage by the latest snapshot of each branch
+
+### Isolation of changes with branches
+
+**Table branching and tagging (Iceberg)**
+
+Branching allows creating independent snapshot lineages, useful for isolating data for validation (e.g., ingestion-validation branches). This ensures data integrity during testing before merging into the main table.
+
+Tagging provides named references to snapshots, aiding reproducibility (e.g., tagging snapshots for end-of-quarter audits). Tags specify retention duration and enable easy access to historical states.
+
+**Catalog branching and tagging (Nessie)**
+
+Branching facilitates safe data validation across multiple tables. Example: weekly_ingest_branch allows testing data before merging into the main production branch.
+
+Tagging marks specific catalog versions for tracking and reproducing states (e.g., quarterly analytics tags). Simplifies accessing historical data.
+
+Benefits of it all:
+
+* facilitates the safe and isolated testing and validation of new data batches
+* enables easy reproduction of data at regular intervals, such as at the end of each quarter, improving the reliability of analytics
+* assists in maintaining an audit trail of data changes over time
+* aids in identifying different versions of the table for different analytics requirements
+
+### Multitable transactions
+
+Multitable transactions ensure atomicity and data consistency by treating multiple operations across tables as a single unit of work, where all changes succeed or are rolled back on failure. This prevents data inconsistencies, supports isolation, and is crucial for systems like supply chain management. Using Nessie, transactions can be isolated in a catalog branch, allowing safe multitable operations. Once validated, changes can be merged into the main branch or discarded, ensuring reliable and consistent data management.
+
+### Rolling back changes
+
+* at the table level with `rollback_to_snapshot`, `rollback_to_timestamp`, `set_current_snapshot`, `cherrypick_snapshot`
+* at the catalog level with  git-like version control
+
+---
+
+## Apache Iceberg: The Definitive Guide - Chapter 11 - Streaming with Apache Iceberg
+
+* scalability and performance
+
+Apache Iceberg is designed to efficiently store and retrieve information from large datasets. The file management procedures enable it to optimize performance of an ever-changing/growing dataset, making it an excellent choice for streaming analytics.
+
+* schema evolution
+
+As data changes over time, the structure of the data (the schema) may need to evolve as well. Apache Iceberg allows for schema evolution without interrupting ongoing data streaming processes, making it easier to adapt to changing data requirements
+
+* reliability
+
+Apache Iceberg provides snapshot isolation, meaning that each transaction operates on an unchanging snapshot of the table. This feature ensures consistent and reliable data, even in environments with many concurrent operations.
+
+* time travel
+
+Iceberg stores a full history of the table’s metadata, which allows time-travel queries that can access previous versions of the table.
+
+### Streaming with Spark
+
+Spark streaming benefits:
+
+* fault tolerance
+* integration
+* real-time processing
+* window operations
+* high throughput
+* multiple data sources
+
+### Streaming with Flink
+
+Benefits:
+
+* event time processing
+* fault tolerance
+* backpressure handling
+* high throughput and low latency
+* windowing and complex event processing
+* integration
+* exactly-once semantics
+
+### Streaming with Kafka Connect
+
+Key features:
+
+* high throughput
+* fault tolerance
+* real-time integration
+* durability
+* wide integration
+* exactly-once semantics
+* connector framework
+
+**The Iceberg Kafka Sink**
+
+A Kafka sink is a component in Apache Kafka that allows data to be consumed from Kafka topics and written to external systems or databases. It plays a crucial role in data integration and data flow by enabling seamless transfer of data from Kafka to various data stores, databases, or analytics platforms.
+
+The Apache Iceberg Sink Connector is a specialized Kafka sink connector that is used for writing data from Kafka into Iceberg tables. It provides various features and capabilities, making it a powerful tool for data integration and storage. There are many features of the Apache Iceberg Sink Connector:
+
+* commit coordination
+* exactly-once delivary semantics
+* multitable fanout
+* row mutations
+* upsert mode
+
+---
+
+## Apache Iceberg: The Definitive Guide - Chapter 14 - Real-World Use Cases of Apache Iceberg
+
+### Ensuring High-Quality Data with Write-Audit-Publish in Apache Iceberg
+
+The Write-Audit-Publish (WAP) pattern provides a systematic approach to ensure that data is of good quality
+
+1. Write: Data is first extracted from sources and written to a nonproduction location, isolating production data from potential inconsistencies
+2. Audit: Once staged, the data undergoes a thorough validation process. This could involve inspecting null or duplicate values, validating data types, and checking data integrity.
+3. Publish: After validation, the data is atomically pushed to production tables, ensuring that consumers see the entire updated dataset or none of it.
+
+**WAP using Iceberg’s branching**
+
+The WAP process at the table level
+
+![image](https://github.com/user-attachments/assets/9f0ee7f1-9917-4679-bba5-ea65b3dc388a)
+
+_Create a branch_
+
+```python
+# Create a Branch on the Table called 'etl_branch'
+spark.sql("ALTER TABLE catalog.db.table CREATE BRANCH etl_branch").show()
+
+# Get a count of all records in the table
+spark.sql("SELECT COUNT(*) as total_records FROM catalog.db.table").show()
+
+# List all Table References
+spark.sql("SELECT * FROM catalog.db.table.refs").show()
+```
+
+_Write the data_
+
+```python
+# Enabling the Apache Iceberg WAP feature
+spark.sql("ALTER TABLE catalog.db.table SET TBLPROPERTIES ('write.wap.ena
+bled'='true')")
+
+# Setting the table for WAP
+spark.conf.set('spark.wap.branch', 'etl_branch')
+
+# Insert New Records Into the Table
+spark.sql("INSERT INTO catalog.db.table SELECT * FROM new_data")
+
+# The Number of Records on the 'etl_branch' - will see the table has additional records
+spark.sql("SELECT COUNT(*) as total_records FROM catalog.db.table VERSION AS OF
+'etl_branch'").show()
+
+# Count of Records on the Main Branch
+spark.sql("SELECT COUNT(*) as total_records FROM catalog.db.table VERSION AS OF
+'main'").show()
+```
+
+_Audit the data_
+
+After writing the new data into the isolated local branch, etl_branch, it is essential to ensure that this new dataset stands up to the organization’s quality standards. The audit phase acts as a checkpoint where we subject our data to rigorous evaluation, ensuring its fitness for purpose.
+
+```python
+# df = dataframe with our "catalog.db.table" table
+# Check for nulls in each column
+null_counts = df.select([count(when(col(c).isNull(), c)).alias(c) for c in
+df.columns])
+
+# Show the result
+null_counts.show()
+
+# df = dataframe with our "catalog.db.table" table
+# Group by all columns and count
+duplicates = df.groupBy(df.columns).count()
+
+# Filter out groups with count > 1, which indicates duplicates
+duplicates = duplicates.filter(col("count") > 1)
+
+# Optionally, drop the count column if not needed
+duplicates = duplicates.drop("count")
+
+# Show the result (duplicates)
+duplicates.show()
+
+# df = dataframe with our "catalog.db.table" table
+# Define your date range
+start_date = datetime(2024, 1, 1)  # for example, Jan 1, 2024
+end_date = datetime(2024, 1, 31)  # for example, Jan 31, 2024
+
+# Convert the date column to date type if it's not already
+df = df.withColumn("date_column", col("date_column").cast("date"))
+
+# Filter the DataFrame to find records within the date range
+within_range = df.filter((col("date_column") >= start_date) & (col("date_col
+umn") <= end_date))
+
+# Count the records that fall within the desired date range
+count_within_range = within_range.count()
+```
+
+We compare this count to the difference between the count of the records on the main branch and the count on etl_branch. If they don’t match, we can inspect whether any records have incorrect dates or null dates causing the inconsistency.
+
+If we identify problems, we can fix the data.
+
+```python
+# 1. Identifying Null Records
+# Check for nulls in each column and create a filter condition
+is_null_condition = [col(c).isNull() for c in df.columns]
+combined_null_condition = is_null_condition[0]
+for condition in is_null_condition[1:]:
+    combined_null_condition = combined_null_condition | condition
+
+# 2. Identifying Duplicated Records
+# Group by all columns, count occurrences, and filter for counts greater than 1
+duplicates_condition = df.groupBy(df.columns) \
+                         .count() \
+                         .filter(col("count") > 1) \
+                         .drop("count") \
+                         .distinct()
+
+# 3. Creating DataFrame for Records Needing Remediation
+# Union the null and duplicated conditions to find all records needing remedia
+tion
+records_needing_remediation = df.filter(combined_null_condi
+tion).union(df.join(duplicates_condition, df.columns, "inner")).distinct()
+
+# 4. Creating DataFrame for Valid Records
+# Use exceptAll to find records that are neither null nor duplicated
+valid_records = df.exceptAll(records_needing_remediation)
+
+# Show the records needing remediation
+records_needing_remediation.show()
+
+# Show the valid records
+valid_records.show()
+
+# Overwrite the table with only the validated records
+valid_records.write.format("iceberg").mode("overwrite").save("catalog.db.table")
+```
+
+Now our branch has new validated records, and we’ve shipped off the invalid records to our stakeholders, who can fix them for later backfilling.
+
+_Publish the changes_
+
+```python
+#Query The List of References for the Table
+spark.sql("SELECT * FROM catalog.db.table.refs")
+
+#Cherry-picking the snapshot from 'etl_branch' over to 'main'
+spark.sql("CALL catalog.system.cherrypick_snapshot('db.table',
+2668401536062194692)").show()
+
+# Record count on the 'main' branch
+spark.sql("SELECT count(*) FROM catalog.db.table VERSION AS OF 'main'").show();
+
+# Record count on the 'etl_branch' branch - both should be identical
+spark.sql("SELECT count(*) FROM catalog.db.table VERSION AS OF 
+'etl_branch'").show();
+
+#Turn off the WAP feature
+spark.conf.unset('spark.wap.branch')
+```
+
+### Running BI Workloads on the Data Lake
+
+The workflow would typically go as follows:
+
+1. raw data is landed as Apache Iceberg tables in the data lake
+2. virtual data marts and data products are created by creating layers of logical views on these tables
+3. reflections are enabled on a view that a dashboard will be made from. An Apache Iceberg representation of the aggregates will be created behind the scenes
+4. a dashboard will be created on the source, which will feel performant as aggregate queries are executed against the reflection instead of the raw sources
+
+**Land the data into the data lake**
+
+Once your data is in the form of Apache Iceberg tables, just need to connect your data lake storage or Apache Iceberg catalog (Hive, Nessie, AWS glue) to Dremio to be able to have access to your Apache Iceberg tables
+
+**Create virtual data marts/data products**
+
+In data warehouses, cleanup, validation, and business logic are typically applied through layered datasets organized for different business units, often referred to as data marts. Dremio facilitates the creation of virtual data marts on the data lakehouse by using logical views instead of physical tables. These views, built on raw Apache Iceberg tables, encapsulate cleanup, validation, and business logic and can be organized into folders for each business unit, effectively structuring data into data marts or data products.
+
+![image](https://github.com/user-attachments/assets/e53c0c0a-fc19-4a21-a887-74a7c16876b2)
+
+**Create a reflection to accelerate our dashboard**
+
+If we are working with a really large dataset to fuel a dashboard, we may want to create an aggregate reflection to ensure performance
+
+For example, if we were creating a dashboard of Ohio shipments based on the oh_shipments view, we could enable an aggregate reflection on that table with a few clicks on the Dremio UI or through a simple SQL query:
+
+```sql
+ALTER TABLE arctic.logistics.oh_shipments
+  CREATE AGGREGATE REFLECTION oh_shipments_agg
+  USING
+  DIMENSIONS (shipper_id, destination_city, shipping_method)
+  MEASURES (shipment_id (COUNT), total_cost (SUM), delivery_time (AVG))
+  LOCALSORT BY (shipper_id, destination_city);
+```
+
+This will create a reflection that is optimized for the particular dimensions and measures needed for the dashboard we are creating and the Apache Iceberg table containing precomputed aggregates called oh_shipments_agg. Although our analysts need to be aware of the existence of this reflection, Dremio will swap out oh_shipments_agg anytime it sees aggregate queries coming in for oh_shipments across the same dimensions and measures. Also, since our source tables are Apache Iceberg tables, reflections can be incrementally applied, allowing the reflection to maintain near-real-time freshness.
+
+### Implementing CDC with Iceberg
+
+By focusing on incremental changes—whether they involve inserts, updates, or deletions—CDC ensures that data systems remain synchronized without needing to repeatedly process extensive static data
+
+A sample CDC in a mart:
+
+![image](https://github.com/user-attachments/assets/e8bbfac7-ad45-4e4a-a09e-d692bed303ef)
+
+**Create Iceberg tables**
+
+```python
+# table 1 - inventory
+spark.sql('''CREATE TABLE glue.test.inventory(
+  product_id int,
+  product_name string,
+  stock_level int,
+  price int,
+  last_updated date) USING iceberg''')
+
+spark.sql('''INSERT INTO glue.test.inventory VALUES (1, 'Pasta-thin', 60, 45, 
+'3/25/2023'),
+(2, 'Bread-white', 55, 6, '3/10/2023'),
+(3, 'Eggs-nonorg', 100, 8, '3/12/2023'),
+(4, 'Sausage-pork', 72, 25, '3/29/2023'),
+(5, 'Coffee-vanilla', 30, 45, '3/12/2023'),
+(6, 'Maple Syrup', 20, 85, '3/29/2023'),
+(7, 'Protein Bar', 120, 5, '3/15/2023')
+''')
+
+# and table 2 - inventory_summary
+
+spark.sql('''CREATE TABLE glue.test.inventory_summary(
+    product_id string,
+    total_stock string,
+    avg_price string) USING iceberg''')
+
+spark.sql('''INSERT INTO glue.test.inventory_summary 
+SELECT 
+    product_id,  
+    SUM(stock_level) AS total_stock, 
+    AVG(price) AS avg_price
+FROM glue.test.inventory_new
+GROUP BY product_id;
+''')
+```
+
+**Apply updates from operational systems**
+
+Iceberg allows us to do row-level updates with transactional guarantees
+
+```python
+spark.sql('''UPDATE glue.test.inventory
+SET stock_level = stock_level - 15
+WHERE product_name = 'Bread-white' ''')
+```
+
+**Create the change log view to capture changes**
+
+We’ll leverage Iceberg’s built-in Spark procedure, create_changelog_view(), to execute this task. To capture changes, we have two options: use start and end snapshot IDs or use start and end timestamps. In this example, we’ll use specific snapshots as our reference points, with the start occurring after the initial inventory table record ingestion and the end following the ETL job execution. To obtain these snapshot IDs, we’ll query the default history metadata table provided by Apache Iceberg
+
+```python
+spark.sql("SELECT * FROM glue.test.inventory.history").toPandas()
+```
+
+For example, let's assume that from inspecting our history table we’ve determined the target snapshot IDs are 4816648710583642722 and 2557325773776943708
+
+```python
+spark.sql("""CALL 
+glue.system.create_changelog_view( 
+     table => 'glue.test.inventory',
+     options => map(
+     'start-snapshot-id',
+     '4816648710583642722',
+          'end-snapshot-id',
+          '2557325773776943708'
+     ))""" )
+```
+
+This procedure creates a change log view called inventory_changes that allows us to see the changes made to the table between the first and second snapshots specified
+
+We can query it:
+
+```python
+spark.sql("SELECT * FROM inventory_changes").toPandas()
+```
+
+**Merge changed data in the aggregated table**
+
+The last step involves updating the downstream aggregated inventory_summary table, utilized by BI reports to extract store stock-level insights
+
+We’ll first create a view from our change log data that is made up of the aggregated changes. We can then merge those aggregated updates into our inventory_summary table to have an updated summary table for our BI dashboard:
+
+```python
+## Create the Aggregated View
+spark.sql("""
+    CREATE OR REPLACE TEMPORARY VIEW aggregated_changes AS 
+    SELECT
+        product_id,
+        SUM(CASE 
+            WHEN _change_type = 'INSERT' THEN stock_level
+            WHEN _change_type = 'DELETE' THEN -stock_level
+            ELSE 0 END) AS total_stock_change,
+        AVG(price) AS new_avg_price
+    FROM
+        inventory_changes
+    GROUP BY
+        product_id
+""")
+
+## Merge the Aggregated View into our Inventory Summary
+spark.sql("""
+    MERGE INTO glue.test.inventory_summary AS target
+    USING aggregated_changes AS source
+    ON target.product_id = source.product_id
+    WHEN MATCHED THEN 
+        UPDATE SET 
+            target.total_stock = target.total_stock + source.total_stock_change
+    WHEN NOT MATCHED THEN 
+        INSERT (product_id, total_stock, avg_price)
+        VALUES (source.product_id, source.total_stock_change, 
+source.new_avg_price)
+""")
+```
+
+In a prod env, these operations for CDC would typically be automated and continuously monitored for optimal performance and reliability (using Airflow, etc)
+
+---
+
+## [Streamed](https://www.youtube.com/watch?v=l5f_uV8j3BM)
+
+After the above books I decided to check out Learning Spark 2nd edition. 
+
+Following the book's suggestion I created a community edition account on databricks so that I can play around and execute some of the code myself
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/21dc6566-fd0a-46bd-a0a8-ae921e94094c">
+
+For now the Scala Spark and PySpark APIs are almost identical - maybe because I am not looking at advanced stuff. I chose to look at scala just so that I gain even minimal familiarity with the language and its peculiarities, but there is a Scala and Python version for each chapter's code. 
+
+I continued reading and following the book's code after stream casually mainly because it is just going over introductory Spark syntax but I can see why this book is so highly regarded.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From cf796ea1920690d6384ff7283cd8dfdf28c8448a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 6 Dec 2024 22:16:10 +0900
Subject: [PATCH 521/638] Create 2024-12-06-day-340.md

---
 _posts/2024-12-06-day-340.md | 647 +++++++++++++++++++++++++++++++++++
 1 file changed, 647 insertions(+)
 create mode 100644 _posts/2024-12-06-day-340.md

diff --git a/_posts/2024-12-06-day-340.md b/_posts/2024-12-06-day-340.md
new file mode 100644
index 000000000..5805d9438
--- /dev/null
+++ b/_posts/2024-12-06-day-340.md
@@ -0,0 +1,647 @@
+---
+layout: post
+title: (Day 340) Exploring Spark's ML library
+categories: [data-eng,theory,applying-knowledge,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 340!
+A quick summary of today:
+* read a bit of [Data Pipelines Pocket Reference](https://learning.oreilly.com/library/view/data-pipelines-pocket/9781492087823/)
+* more of the Learning Spark book
+* exploring Spark's ML library
+* streamed
+
+## Prediction underground demand for Daejeon's Subway Line 1
+
+![image](https://github.com/user-attachments/assets/87a0b623-0caa-493c-bcae-b6c94c7ac95b)
+
+I saw [this (in Korean)](https://bigdata.hannam.ac.kr/%ea%b3%b5%ec%a7%80%ec%82%ac%ed%95%ad/?uid=230&mod=document) competition in my university from the Big Data department and saw that its finals are offline this morning at 10.30am so I decided to go check out what kind of models people made. (I am extra keen even after Vincent from probabl made [this](https://www.youtube.com/watch?v=EnhyJx8l2LE&t=3s) video on time-series predictions)
+
+### Team 1 - Using XGBoost
+
+- added about the surrounding area around each station
+- added temperature data
+- 10.59 MAE
+- used K-fold and GridSearchCV
+
+### Team 2 - AutoGluon
+
+- looked into special (one-time/annual) events that happened during the time period, and added a new column that says which event occured near which station
+- added vacation information data (dates when people might be travelling)
+- features: year month week weekday time station event rain vacation; target: count
+- 10.7 MAE
+- no mention of CV 
+
+### Team 3 
+
+- extra features: date features, month_sin, day_sin, time_sin, weather features, station surrounding features, month X weather interactive features
+- RandomForest - 14.16 MAE + missed CatBoost MAE
+- combined RF + CatBosst -> rf 0.9 + catboost 0.1 = 14.03 MAE
+
+### Team 4 
+
+- [TimeXer library](https://arxiv.org/pdf/2402.19072) which is based on a recent paper
+- added weather features, holiday featres
+- I think the MAE was around 30
+- they shared their [repo](https://github.com/castleeun/time_series_data) which is awesome as I did not expect anyone to share their code publicly
+
+### Team 5
+
+- explored the difference between using the mean and median as predictors for demand
+- did not present a model, they joined the Big Data department this semester so they are quite new and presented their code for cleaning, transforming the data, and using mean/median for predictions👏
+
+### Team 6 
+
+- used data after 2023 only (removing covid data)
+- showed their code on the ppt
+- used ARIMA, used auto_arima to find its hyperparams (p,d,q)
+
+Definitely some interesting ideas ^^ Glad I went
+
+## Data Pipelines Pocket Reference - Chapter 1 - Introduction to Data Pipelines
+
+### What are they?
+
+_Data pipelines are sets of processes that move and transform data from various sources to a destination where new value can be derived. They are the foundation of analytics, reporting, and machine learning capabilities._
+
+![image](https://github.com/user-attachments/assets/6c5de580-234c-479e-b7ee-44df7ef7a17c)
+
+### Who builds them?
+
+Data engineers play a critical role in analytics teams by building and maintaining data pipelines to support machine learning, data science, and timely insights. Their responsibilities go beyond loading data into warehouses, focusing on scalable production systems, data validity, and timeliness through testing, alerting, and contingency planning.
+
+#### Key skills:
+
+1. SQL and data warehousing: expertise in high-performance SQL and warehousing fundamentals for efficient database querying and collaboration
+2. programming: proficiency in Python or Java, with emerging interest in Go, tailored to the team's tech stack
+3. distributed computing: knowledge of platforms like Apache Spark and Hadoop to handle high-volume data processing
+4. system administration: proficiency with Linux commands, cloud services and basic system tasks like log analysis and security troubleshooting
+5. goal-oriented mentality: understanding the purpose behind pipelines to make better architectural decisions aligned with team and stakeholder needs
+
+### Why build them?
+
+They are the key behind the fancy dashboards that executives see. Data pipelines do more than just extract data from sources and load them into simple database tables or flat files for analysts to use. Raw data is refined along the way to clean, structure, normalize, combine, aggregate, and at times anonymize or otherwise secure it
+
+Data pipelines ensure that the proper data is delivered so the rest of the analytics organization can focus their time on what they do best: delivering insights
+
+### How are they built?
+
+There are open- and closed-source solutions, many are written in Python, but there are in other languages like Java as well. 
+
+---
+
+## Data Pipelines Pocket Reference - Chapter 2 - A Modern Data Infrastructure
+
+### Diversity of data sources
+
+The majority of organizations have dozens, if not hundreds, of data sources that feed their analytics endeavors.
+
+**Source system ownership**
+
+It’s typical for an analytics team to ingest data from source systems that are built and owned by the organization as well as from third-party tools and vendors.
+
+![image](https://github.com/user-attachments/assets/cdf52aa5-9a25-4ebb-a038-2841d3b53f86)
+
+**Ingestion interface and data structure**
+
+Regardless of who owns the source data, how you get it and in what form is the first thing a data engineer will examine when building a new data ingestion.
+
+Ingestion interfaces are databases, csv files, json files, kafka streams, etc. 
+
+**Data volume**
+
+Orgs have both small and big datasets
+
+**Data cleanliness and validity**
+
+Just as there is great diversity in data sources, the quality of source data varies greatly. It’s important to understand the limitations and deficiencies of source data and address them in the appropriate sections of your pipelines.
+
+### Cloud data warehouses and data lakes
+
+Three things transformed the landscape of analytics and data warehousing over the last 10 years, and they’re all related to the emergence of the major public cloud providers
+
+* the ease of building and deploying data pipelines, data lakes, warehouses, and analytics processing in the cloud. No more waiting on IT departments and budget approval for large up-front costs. Managed services—databases in particular—have become mainstream
+* continued drop-in storage costs in the cloud
+* the emergence of highly scalable, columnar databases, such as Amazon Redshift, Snowflake, and Google Big Query
+
+### Data ingestion tools
+
+The book will discuss these 3 most common tools: Singer, Stitch, and Fivetran, that can help with ingesting data from various sources. 
+
+### Data transformation and modelling tools
+
+The terms data modeling and data transformation are often used interchangeably, however the book differentiates them
+
+_Data transformation: Transforming data is a broad term that is signified by the T in an ETL or ELT process. A transformation can be something as simple as converting a timestamp stored in a table from one time zone to another. It can also be a more complex operation that creates a new metric from multiple source columns that are aggregated and filtered through some business logic._
+
+_Data modeling: Data modeling is a more specific type of data transformation. A data model structures and defines data in a format that is understood and optimized for data analysis. A data model is usually represented as one or more tables in a data warehouse._
+
+### Workflow orchestration platforms
+
+Luigi, AWS Glue, Airflow, Kubeflow, others. 
+
+---
+
+## Chapter 3 - Common Data Pipeline Patterns
+
+### ETL and ELT
+
+ETL and ELT are widely used data processing patterns in data warehousing, business intelligence, and more recently, in data science and machine learning pipelines. They describe methods of preparing data for analysis by analysts and reporting tools, with their primary distinction being the order of the transform and load steps.
+
+### The emergence of ELT over ETL
+
+ETL was the standard data pipeline pattern for decades but has been largely supplanted by ELT due to advancements in modern, cloud-based columnar data warehouses like Snowflake and Amazon Redshift
+
+ETL challenges
+
+- older data warehouses lacked the storage and compute power to process large raw datasets
+- they were row-based, optimized for transactional use cases, but inefficient for bulk queries common in analytics
+- transformations had to occur outside the warehouse before loading data
+
+Emergence of ELT:
+
+- modern columnar databases support scalable, efficient processing of large datasets
+- features like I/O efficiency, data compression and distributed queries make it cost-effective to load raw data first and perform transformations within the warehouse
+- column-based storage optimizes analytics by reducing disk I/O, focusing only on relevant columns, and enabling better data compression
+
+Impact:
+
+- ELT enables streamlined pipelines where data is extracted, loaded into a warehouse, and transformed directly within the database
+- this approach suits analytics, ML, and data product development by leveraging the strengths of modern data warehouses
+
+### EtLT subpattern
+
+When ELT emerged as the dominant pattern, it became clear that doing some transformation after extraction, but before loading, was still beneficial. However, instead of transformation involving business logic or data modeling, this type of transformation is more limited in scope (hence a small t 😆). Some examples:
+
+* deduplicate records in a table
+* parse URL parameters into individual components
+* mask or otherwise obfuscate sensitive data
+
+These types of transforms are either fully disconnected from business logic or, in the case of something like masking sensitive data, at times required as early in a pipeline as possible for legal or security reasons
+
+### ELT for data analysis
+
+With the emergence of ELT, data analysts have become more autonomous and empowered to deliver value from data without being 'blocked' by data engineers. 
+
+![image](https://github.com/user-attachments/assets/a260d3ff-3ad8-48b6-b565-cf4c42864589)
+
+### ELT for data science
+
+Data pipelines for data science teams are similar to those for data analysis but differ in their focus as data scientists require more granular and often raw data for exploration and predictive modeling, unlike data analysts who build metrics and dashboards. This distinction influences pipeline design to meet the specific needs of data scientists.
+
+### ELT for data products and ML
+
+Data is used for more than analysis, reporting, and predictive models. It’s also used for powering data products. Some common examples of data products include the following:
+
+* a content recommendation engine that powers a video streaming home screen
+* a personalized search engine on an e-commerce website
+* an application that performs sentiment analysis on user-generated restaurant reviews
+
+**Steps in ML pipelines**
+
+* data ingestion
+* data preprocessing
+* model training
+* model deployment
+
+Any good ML pipeline will also include gathering feedback for improving the model
+
+---
+
+## Chapter 4 - Data ingestion
+
+This chapter gave code examples on ingesting data from different sources and how to set that up
+
+* mysql
+* postgres
+* mongodb
+* rest api
+* kafka
+
+---
+
+## Chapter 5 - Data Ingestion: Loading Data
+
+This chapter as well just shows code snippets and setting up Amazon Redshift/Snowflake and using it as a data warehouse. 
+
+---
+
+## Chapter 6 - Transforming Data
+
+### Noncontextual transformations
+
+**Deduping records in a table**
+
+There are a number of reasons it happens:
+
+* an incremental data ingestion mistakenly overlaps a previous ingestion time window and picks up some records that were already ingested in a previous run
+* duplicate records were inadvertently created in a source system
+* data that was backfilled overlapped with subsequent data loaded into the table during ingestion
+
+We can using group by and having, distinct, or window functions (row_number) to achieve that in SQL
+
+**Parsing URLs**
+
+* using `urllib3`
+
+### When to transform? During or after ingestion?
+
+Data transformations without business context can be run either during or after data ingestion from a technical standpoint. However, there are some reasons you should consider running them as part of the ingestion process (the EtLT pattern):
+
+* the transformation is easiest to do using a language besides SQL
+* the transformation is addressing a data quality issue
+
+### Data modelling foundations
+
+* measures: these are things you want to measure! Examples include a count of customers and a dollar value of revenue
+* attributes: these are things by which you want to filter or group in a report or dashboard. Examples include dates, customer names, and countries
+* granularity: the level of detail stored in a data model
+* source tables: tables that were loaded into the data warehouse or data lake via a data ingestion
+* slowly changing dimensions (I recommend checking out Zach Wilson's lectures for this! They are awesome!
+
+---
+
+## Learning Spark, 2nd Edition - Chapter 8 - Structured Streaming
+
+### Evolution of the Apache Spark stream processing engine
+
+**The advent of micro-batch stream processing**
+
+Spark introduced Spark Streaming (DStreams) which introduced the idea of micro-batch stream processing, where the streaming computation is modeled as a continuous series of small, map/reduce-style batch processing jobs (hence, 'micro-batches') on small chunks of the stream data
+
+![image](https://github.com/user-attachments/assets/dff3d93f-f98c-4c89-88f6-6682eaad526f)
+
+Spark Streaming divides the data from the input stream into, say, 1-second micro-batches. Each batch is processed in the Spark cluster in a distributed manner with small deterministic tasks that generate the output in micro-batches. Breaking down the streaming computation into these small tasks gives us two advantages over the traditional, continuous-operator model:
+
+* Spark’s agile task scheduling can very quickly and efficiently recover from failures and straggler executors by rescheduling one or more copies of the tasks on any of the other executors
+* the deterministic nature of the tasks ensures that the output data is the same no matter how many times the task is reexecuted. This crucial characteristic enables Spark Streaming to provide end-to-end exactly-once processing guarantees, that is, the generated output results will be such that every input record was processed exactly once
+
+**Lessons learning from DStreams**
+
+Despite all the advantages, the DStream API was not without its flaws
+
+* lack of a single API for batch and stream processing
+* lack of separation between logical and physical plans
+* lack of native support for event-time windows
+
+**Structured streaming**
+
+Based on these lessons from DStreams, Structured Streaming was designed from scratch with one core philosophy—for developers, writing stream processing pipelines should be as easy as writing batch pipelines. In a nutshell, the guiding principles of Structured Streaming are:
+
+* a single, unified programming model and interface for batch and stream processing
+* a broader definition of stream processing
+
+### The programming model of Structured streaming
+
+Structured streaming views a table as a stream as an unbounded, continuously appended table
+
+![image](https://github.com/user-attachments/assets/5154f7ce-f014-4483-9b62-0ba1ceda5ff4)
+
+Every new record received in the data stream is like a new row being appended to the unbounded input table. Structured Streaming will not actually retain all the input, but the output produced by Structured Streaming until time T will be equivalent to having all of the input until T in a static, bounded table and running a batch job on the table.
+
+![image](https://github.com/user-attachments/assets/b272b28f-5f56-4080-9643-b5553987264f)
+
+The last part of the model is the output mode. There are 3:
+
+* append mode: only the new rows appended to the result table since the last trigger will be written to the external storage. This is applicable only in queries where existing rows in the result table cannot change (e.g. a map on an input stream)
+* update mode: only the rows that were updated in the result table since the last trigger will be changed in the external storage. This mode works for output sinks that can be updated in place, such as a MySQL table
+* complete mode: the entire updated result table will be written to external storage
+
+### The fundamentals of a structured streaming query
+
+**5 steps to define a streaming query**
+
+1. define input sources
+2. transform data (stateless vs stateful transformations)
+3. define output sink and output mode
+4. specify processing details (triggering details, checkpoint location)
+5. start the query
+
+Full example 
+
+```python
+# In Python
+from pyspark.sql.functions import *
+spark = SparkSession...
+lines = (spark
+  .readStream.format("socket")
+  .option("host", "localhost")
+  .option("port", 9999)
+  .load())
+
+words = lines.select(explode(split(col("value"), "\\s")).alias("word"))
+counts = words.groupBy("word").count()
+checkpointDir = "..."
+streamingQuery = (counts
+  .writeStream
+  .format("console")
+  .outputMode("complete")
+  .trigger(processingTime="1 second")
+  .option("checkpointLocation", checkpointDir)
+  .start())
+streamingQuery.awaitTermination()
+```
+
+**Under the hood of an active streaming query**
+
+1. Spark SQL analyzes and optimizes this logical plan to ensure that it can be executed incrementally and efficiently on streaming data
+
+2. Spark SQL starts a background thread that continuously executes the following loop:
+
+* based on the configured trigger interval, the thread checks the streaming sources for the availability of new data
+* if available, the new data is executed by running a micro-batch. From the optimized logical plan, an optimized Spark execution plan is generated that reads the new data from the source, incrementally computes the updated result, and writes the output to the sink according to the configured output mode
+* for every micro-batch, the exact range of data processed (e.g., the set of files or the range of Apache Kafka offsets) and any associated state are saved in the configured checkpoint location so that the query can deterministically reprocess the exact range if needed
+
+3. This loop continues until the query is terminated, which can occur for one of the following reasons:
+
+* a failure has occurred in the query (either a processing error or a failure in the cluster)
+* the query is explicitly stopped using streamingQuery.stop()
+* if the trigger is set to Once, then the query will stop on its own after executing a single micro-batch containing all the available data
+
+![image](https://github.com/user-attachments/assets/8edc1610-b68a-475e-8f5b-534941fe0206)
+
+A key point to remember about Structured Streaming is that underneath it is using Spark SQL to execute the data. As such, the full power of Spark SQL’s hyperoptimized execution engine is utilized to maximize the stream processing throughput, providing key performance advantages.
+
+**Recovering from failures with exactly-once guarantees**
+
+To restart a terminated query in a new process, create a new Spark session, redefine DataFrames, and use the same checkpoint location when starting the query. This ensures the query resumes from where it left off, leveraging checkpoint data like Kafka offsets to process records precisely and maintain fault tolerance.
+
+Key points for ensuring exactly-once guarantees:
+
+1. replayable streaming sources: the source must support rereading the incomplete micro-batch range
+2. deterministic computations: transformations must yield consistent results with the same input
+3. idempotent sinks: the sink must handle duplicate micro-batches to prevent redundant writes
+
+Allowed modifications during query restarts:
+
+- transformations: minor changes like adding filters to handle corrupted data
+- source and sink options: modifications depend on specific source/sink semantics (e.g. changing console sink options is allowed; changing socket host/port is not)
+- processing details: the checkpoint location must remain unchanged, but settings like trigger intervals can be updated
+
+**Monitoring an active query**
+
+We can use a `StreamingQuery` instance to get current metrics and status. We can also create listeners to publish stats to a location of choice by setting `spark.sql.streaming.metricsEnabled` to true. 
+
+### Streaming data sources and sinks
+
+**Reading from files**
+
+* all the files must be of the same format and are expected to have the same schema
+* each file must appear in the directory listing atomically—that is, the whole file must be available at once for reading, and once it is available, the file cannot be updated or modified
+* when there are multiple new files to process but it can only pick some of them in the next micro-batch (e.g., because of rate limits), it will select the files with the earliest timestamps. Within the micro-batch, however, there is no predefined order of reading of the selected files; all of them will be read in parallel
+
+**Writing to files**
+
+* Structured Streaming achieves end-to-end exactly-once guarantees when writing to files by maintaining a log of the data files that have been written to the directory
+* if you change the schema of the result DataFrame between restarts, then the output directory will have data in multiple schemas. These schemas have to be reconciled when querying the directory
+
+**Kafka**
+
+We can read/write form/to Kafka topics
+
+**Custom sources and sinks**
+
+We can use `foreachBatch()` and `foreach()` methods to implement custom logic to write to our storage.
+
+### Data transformations
+
+**Stateless**
+
+All projection operations (e.g. select(), explode(), map(), flatMap()) and selection operations (e.g., filter(), where()) process each input record individually without needing any information from previous rows. This lack of dependence on prior input data makes them stateless operations.
+
+**Stateful**
+
+The simplest example of a stateful transformation is DataFrame.groupBy().count(), which generates a running count of the number of records received since the beginning of the query. In every micro-batch, the incremental plan adds the count of new records to the previous count generated by the previous micro-batch. This partial count communicated between plans is the state. This state is maintained in the memory of the Spark executors and is checkpointed to the configured location in order to tolerate failures.
+
+![image](https://github.com/user-attachments/assets/eb2b1b70-e84b-4c2b-9669-b7a09292b26b)
+
+Each micro-batch reads a new set of words, shuffles them within the executors to group them, computes the counts within the micro-batch, and finally adds them to the running counts to produce the new counts. These new counts are both the output and the state for the next micro-batch, and hence they are cached in the memory of the executors. The next micro-batch of data is grouped between executors in exactly the same way as before, so that each word is always processed by the same executor, and can therefore locally read and update its running count.
+
+For all stateful operations, Structured Streaming ensures the correctness of the operation by automatically saving and restoring the state in a distributed manner. Depending on the stateful operation, all you may have to do is tune the state cleanup policy such that old keys and values can be automatically dropped from the cached state
+
+_There are two types of stateful operations:_
+
+* managed stateful operations: these automatically identify and clean up old state, based on an operation-specific definition of “old.” You can tune what is defined as old in order to control the resource usage (e.g., executor memory used to store state). The operations that fall into this category are those for streaming aggregations, stream–stream joins, streaming deduplication
+
+* mnmanaged stateful operations: these operations let you define your own custom state cleanup logic. The operations in this category are MapGroupsWithState and FlatMapGroupsWithState
+
+### Stateful streaming aggregations
+
+* aggregations not based on time (global vs grouped)
+* aggregations with event-time windows (we can wait for late date with watermarks and a watermark delay which defines how long the engine will wait for late data to arrive)
+
+### Streaming joins
+
+**Stream-Static joins**
+
+* stream–static joins are stateless operations, and therefore do not require any kind of watermarking
+* the static DataFrame is read repeatedly while joining with the streaming data of every micro-batch, so you can cache the static DataFrame to speed up the reads
+* if the underlying data in the data source on which the static DataFrame was defined changes, whether those changes are seen by the streaming query depends on the specific behavior of the data source. For example, if the static DataFrame was defined on files, then changes to those files (e.g., appends) will not be picked up until the streaming query is restarted
+
+**Stream-Stream joins**
+
+Inner
+
+- combine matching events from two streams based on a condition (e.g. `adId`)
+- buffers unmatched events, optional watermarking limits state size and ensures cleanup
+
+Outer
+
+- includes all events, using `NULL`s for unmatched ones
+- requires watermarking to limit state and ensure correct results
+- introduces delay to confirm no future matches
+
+### Performance tuning
+
+Structured Streaming in Spark requires tuning strategies distinct from batch processing due to smaller data volumes and continuous operation.
+
+1. Cluster resource provisioning:
+   - allocate resources to balance performance and cost
+   - stateless queries require more cores; stateful queries need more memory
+
+2. Shuffle partition configuration:
+   - reduce shuffle partitions compared to batch jobs for lower overhead and higher throughput
+   - for stateful operations with frequent triggers, limit partitions to 2–3 times the allocated cores
+
+3. Source rate limits:
+   - prevent instability during data surges by setting rate limits for supported sources 
+   - avoid setting limits too low to prevent resource underutilization
+   - limits can't mitigate sustained data rate increases, leading to growing latencies
+
+4. managing multiple queries:
+   - running multiple queries in the same SparkContext may bottleneck task scheduling or exceed driver memory limits
+   - use separate scheduler pools (`spark.scheduler.pool`) for fair resource allocation between queries
+
+---
+
+## [Streamed](https://www.youtube.com/watch?v=qwX6nX-I-zI&t=1s)
+
+I covered the notebooks from chapter 9 (Spark with Delta Lake) and ch. 10 (Spark ML) on stream and they were very interesting. Spark's ML library has depth for sure.
+
+From the notebooks I also got these 3 extra resources I need to read:
+
+* [Yggdrasil: An Optimized System for Training Deep Decision Trees at Scale](https://people.eecs.berkeley.edu/~matei/papers/2016/nips_yggdrasil.pdf)
+* [Distributed ML slides](https://files.training.databricks.com/static/docs/distributed-linear-regression-1.pdf) and [2](https://files.training.databricks.com/static/docs/distributed-linear-regression-2.pdf) - they seem to be from some kind of databricks education curicula
+
+After streamed I checked the chapter 11 notebooks and this caught my attention - distributed training + inference
+
+XGBoost can be included directly in a PySpark Pipeline even though its not part of the library
+
+```python
+from sparkdl.xgboost import XgboostRegressor
+from pyspark.ml import Pipeline
+
+params = {"n_estimators": 100, "learning_rate": 0.1, "max_depth": 4, "random_state": 42, "missing": 0}
+
+xgboost = XgboostRegressor(**params)
+
+pipeline = Pipeline(stages=[stringIndexer, vecAssembler, xgboost])
+pipeline_model = pipeline.fit(trainDF)
+```
+
+**Distributed Inference with mapInPandas**
+
+```python
+import mlflow.sklearn
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+from sklearn.model_selection import train_test_split
+
+with mlflow.start_run(run_name="sklearn-rf-model") as run:
+  # Import the data
+  spark_df = spark.read.csv("dbfs:/databricks-datasets/learning-spark-v2/sf-airbnb/sf-airbnb-numeric.csv", header=True, inferSchema=True).drop("zipcode")
+  df = spark_df.toPandas()
+  X_train, X_test, y_train, y_test = train_test_split(df.drop(["price"], axis=1), df[["price"]].values.ravel(), random_state=42)
+
+  # Create model, train it, and create predictions
+  rf = RandomForestRegressor(n_estimators=100, max_depth=10)
+  rf.fit(X_train, y_train)
+  predictions = rf.predict(X_test)
+
+  # Log model
+  mlflow.sklearn.log_model(rf, "random-forest-model")
+
+  # Log params
+  mlflow.log_param("n_estimators", 100)
+  mlflow.log_param("max_depth", 10)
+
+  # Log metrics
+  mlflow.log_metric("mse", mean_squared_error(y_test, predictions))
+  mlflow.log_metric("mae", mean_absolute_error(y_test, predictions))  
+  mlflow.log_metric("r2", r2_score(y_test, predictions))
+```
+
+```python
+# Create Spark DataFrame and apply model to Spark DataFrame.
+sparkDF = spark.createDataFrame(X_train)
+```
+
+```python
+# Apply the model in parallel with mapInPandas
+def predict(iterator):
+  model_path = f"runs:/{run.info.run_uuid}/random-forest-model" # Load model
+  model = mlflow.sklearn.load_model(model_path)
+  for features in iterator:
+    yield pd.concat([features, pd.Series(model.predict(features), name="prediction")], axis=1)
+    
+display(sparkDF.mapInPandas(predict, """`host_total_listings_count` DOUBLE,`neighbourhood_cleansed` BIGINT,`latitude` DOUBLE,`longitude` DOUBLE,`property_type` BIGINT,`room_type` BIGINT,`accommodates` DOUBLE,`bathrooms` DOUBLE,`bedrooms` DOUBLE,`beds` DOUBLE,`bed_type` BIGINT,`minimum_nights` DOUBLE,`number_of_reviews` DOUBLE,`review_scores_rating` DOUBLE,`review_scores_accuracy` DOUBLE,`review_scores_cleanliness` DOUBLE,`review_scores_checkin` DOUBLE,`review_scores_communication` DOUBLE,`review_scores_location` DOUBLE,`review_scores_value` DOUBLE, prediction double"""))
+```
+
+****
+
+```python
+# Define a function that takes all the data for a given device, train a model, saves it as a nested run, and returns a DataFrame
+import mlflow
+import mlflow.sklearn
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_squared_error
+
+def train_model(df_pandas: pd.DataFrame) -> pd.DataFrame:
+  """
+  Trains an sklearn model on grouped instances
+  """
+  # Pull metadata
+  device_id = df_pandas["device_id"].iloc[0]
+  n_used = df_pandas.shape[0]
+  run_id = df_pandas["run_id"].iloc[0] # Pulls run ID to do a nested run
+  
+  # Train the model
+  X = df_pandas[["feature_1", "feature_2", "feature_3"]]
+  y = df_pandas["label"]
+  rf = RandomForestRegressor()
+  rf.fit(X, y)
+
+  # Evaluate the model
+  predictions = rf.predict(X)
+  mse = mean_squared_error(y, predictions) # Note we could add a train/test split
+ 
+  # Resume the top-level training
+  with mlflow.start_run(run_id=run_id):
+    # Create a nested run for the specific device
+    with mlflow.start_run(run_name=str(device_id), nested=True) as run:
+      mlflow.sklearn.log_model(rf, str(device_id))
+      mlflow.log_metric("mse", mse)
+      
+      artifact_uri = f"runs:/{run.info.run_id}/{device_id}"
+      # Create a return pandas DataFrame that matches the schema above
+      returnDF = pd.DataFrame([[device_id, n_used, artifact_uri, mse]], 
+        columns=["device_id", "n_used", "model_path", "mse"])
+
+  return returnDF 
+```
+
+```python
+# Use applyInPandas to grouped data
+with mlflow.start_run(run_name="Training session for all devices") as run:
+  run_id = run.info.run_uuid
+  
+  modelDirectoriesDF = (df
+    .withColumn("run_id", f.lit(run_id)) # Add run_id
+    .groupby("device_id")
+    .applyInPandas(train_model, schema=trainReturnSchema)
+  )
+  
+combinedDF = (df
+  .join(modelDirectoriesDF, on="device_id", how="left")
+)
+
+display(combinedDF)
+```
+
+**Joblib to parallelize the evaluation of sklearn models**
+
+```python
+from sklearn.utils import parallel_backend
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.model_selection import GridSearchCV
+import pandas as pd
+from joblibspark import register_spark
+
+register_spark() # register spark backend
+
+spark_df = spark.read.csv("dbfs:/databricks-datasets/learning-spark-v2/sf-airbnb/sf-airbnb-numeric.csv", header=True, inferSchema=True).drop("zipcode")
+df = spark_df.toPandas()
+X_train, X_test, y_train, y_test = train_test_split(df.drop(["price"], axis=1), df[["price"]].values.ravel(), random_state=42)
+
+rf = RandomForestRegressor(random_state=42)
+param_grid = {"max_depth": [2, 5, 10], "n_estimators": [20, 50, 100]}
+gscv = GridSearchCV(rf, param_grid, cv=3)
+
+with parallel_backend("spark", n_jobs=3):
+  gscv.fit(X_train, y_train)
+  
+# Uses R2 to score the models
+print(gscv.cv_results_)
+print(gscv.best_estimator_)
+```
+
+Tomorrow I will cover chapter 12 - related to Spark 3 (one of the earlier versions) and will see what else ~ 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 12ba71b5448e077a6912917b421b83425e701d1f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 8 Dec 2024 00:02:57 +0900
Subject: [PATCH 522/638] Create 2024-12-07-day-341.md

---
 _posts/2024-12-07-day-341.md | 116 +++++++++++++++++++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 _posts/2024-12-07-day-341.md

diff --git a/_posts/2024-12-07-day-341.md b/_posts/2024-12-07-day-341.md
new file mode 100644
index 000000000..1be6ddbf7
--- /dev/null
+++ b/_posts/2024-12-07-day-341.md
@@ -0,0 +1,116 @@
+---
+layout: post
+title: (Day 340) Kafka + Streaming
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 340!
+A quick summary of today:
+* long stream
+* starting the book Stream Processing with Apache Flink
+
+## [Stream for ~3.5hrs](https://youtu.be/FHrcIFNdy40)
+
+Found and covered [this](https://training.confluent.io/channeldetail/apache-kafka-fundamentals-and-accreditation) nice free course by confluent on Kafka and Confluent Platform fundamentals 
+
+It went in depth on how Kafka works and its core components - producers, brokers, and consumers. The videos are only 1.5hrs long but there are also practical exercises where I spun up a VM and executed different commands in the terminal to play with producers/consumers/topics. 
+
+The 2nd part of the course is an optional mini-test with 23 questions, and after I passed it I got:
+
+<img width="385" alt="image" src="https://github.com/user-attachments/assets/fae29e33-f6ac-41dd-a995-68cd05ab8339">
+
+---
+
+## Stream Processing with Apache Flink - Chapter 1 - Introduction to Stateful Stream Processing
+
+### Traditional data infrastructures
+
+**Transactional processing**
+
+![image](https://github.com/user-attachments/assets/02c76086-abee-41ba-afec-7a8e9f263907)
+
+**Analytical processing**
+
+![image](https://github.com/user-attachments/assets/7164a302-d7c0-462b-8358-310cc6e08ccd)
+
+### Stateful stream processing
+
+![image](https://github.com/user-attachments/assets/0d4aa550-82cb-4687-b465-3dc1add87c91)
+
+Stateful stream processing is a design pattern for handling unbounded streams of events, such as user interactions, orders, or sensor data. It enables applications to store and access intermediate data (state) for complex computations, with storage options including program variables, local files, or databases. Apache Flink is a distributed system that manages state locally in memory or embedded databases, protecting against data loss through periodic checkpoints stored remotely.  
+
+Applications often connect to event logs, such as Apache Kafka, which ensure durable, ordered streams that can be replayed. This connection allows state recovery after failures, updates, bug fixes, or A/B testing by restoring state and replaying events.  
+
+Stateful stream processing is used across three primary application categories:  
+
+* event-driven applications
+* data pipeline applications
+* data analytics applications  
+
+Real-world use cases often combine aspects of these categories.
+
+**Event-driven applications**
+
+Typical use cases for event-driven applications include:
+
+* real-time recommendations
+* pattern detection or complex event processing
+* anomaly detection
+
+Event-driven applications are an evolution of microservices. They communicate via event logs instead of REST calls and hold application data as local state instead of writing it to and reading it from an external datastore, such as a relational database or key-value store
+
+![image](https://github.com/user-attachments/assets/f60a4815-be75-49f4-a000-abf2ac544210)
+
+The applications above are connected by event logs. One application emits its output to an event log and another application consumes the events the other application emitted. The event log decouples senders and receivers and provides asynchronous, nonblocking event transfer. Each application can be stateful and can locally manage its own state without accessing external datastores. Applications can also be individually operated and scaled.
+
+Event-driven applications demand robust stream processors with advanced API expressiveness, reliable state management, and strong event-time processing support. Key requirements include exactly-once state consistency and scalability. Apache Flink meets these criteria, making it an excellent choice for running event-driven applications.
+
+**Data pipelines**
+
+Modern IT architectures involve multiple data stores optimized for specific access patterns, often replicating the same data across systems like databases, caches, and search indexes. Synchronizing these data stores is crucial. While periodic ETL jobs have traditionally been used, they often fail to meet the low-latency demands of modern applications.  
+
+An alternative is using event logs to distribute updates, with consumers applying transformations before syncing data to target systems. Stateful stream processing, such as with Apache Flink, excels in building data pipelines, enabling low-latency ingestion, transformation, and synchronization of large data volumes across diverse storage systems.
+
+**Streaming analytics**
+
+A streaming analytics application continuously ingests streams of events and updates its result by incorporating the latest events with low latency. This is similar to the maintenance techniques database systems use to update materialized views. Typically, streaming applications store their result in an external data store that supports efficient updates, such as a database or key-value store
+
+![image](https://github.com/user-attachments/assets/7eb3cf09-0dba-44cc-bd86-c305a2f84dd9)
+
+Streaming analytics offers faster integration of events into analytical results compared to traditional pipelines, which rely on separate ETL processes, storage systems, and schedulers. Stream processors like Apache Flink handle all these steps, including ingestion, computation, state maintenance, and result updates, with features like exactly-once state consistency, event-time processing, and dynamic resource scaling.  
+
+### A quick look at Flink
+
+Apache Flink is a third-generation distributed stream processor with a competitive feature set. It provides accurate stream processing with high throughput and low latency at scale. Features that make it stand out:
+
+* event-time and processing-time semantics. Event-time semantics provide consistent and accurate results despite out-of-order events. Processing-time semantics can be used for applications with very low latency requirements
+* exactly-once state consistency guarantees
+* millisecond latencies while processing millions of events per second. Flink applications can be scaled to run on thousands of cores
+* layered APIs with varying tradeoffs for expressiveness and ease of use. The book covers the DataStream API and process functions, which provide primitives for common stream processing operations, such as windowing and asynchronous operations, and interfaces to precisely control state and time
+* connectors to the most commonly used storage systems such as Apache Kafka, Apache Cassandra, Elasticsearch, JDBC, Kinesis, and (distributed) filesystems such as HDFS and S3
+* ability to run streaming applications 24/7 with very little downtime due to its highly available setup (no single point of failure), tight integration with Kubernetes, YARN, and Apache Mesos, fast recovery from failures, and the ability to dynamically scale jobs
+* ability to update the application code of jobs and migrate jobs to different Flink clusters without losing the state of the application
+* detailed and customizable collection of system and application metrics to identify and react to problems ahead of time
+* last but not least, Flink is also a full-fledged batch processor
+
+At the end of the chapter there were instructions on installing Flink locally ~
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/b6bc42b4-cfff-4071-85db-e4e07b976fdd">
+
+Exciting (this is some Scala version as per the book's suggestion) - I'm excited to learn some Scala
+
+Submitted a job:
+
+<img width="1238" alt="image" src="https://github.com/user-attachments/assets/835144f7-7e55-4744-9b43-8115110a3160">
+
+<img width="1249" alt="image" src="https://github.com/user-attachments/assets/eca81381-b1fa-426d-b319-0ab9255881c4">
+
+And once the job is running I can see logs:
+
+<img width="666" alt="image" src="https://github.com/user-attachments/assets/5ac6a4fc-dd9f-43bb-91b4-e288561cae8b">
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From b223c9b402fae5a93bb4464984ec0ed1f4bcffc1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 8 Dec 2024 21:57:26 +0900
Subject: [PATCH 523/638] Update 2024-12-07-day-341.md

---
 _posts/2024-12-07-day-341.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-12-07-day-341.md b/_posts/2024-12-07-day-341.md
index 1be6ddbf7..462729ae8 100644
--- a/_posts/2024-12-07-day-341.md
+++ b/_posts/2024-12-07-day-341.md
@@ -1,10 +1,10 @@
 ---
 layout: post
-title: (Day 340) Kafka + Streaming
+title: (Day 341) Kafka + Streaming
 categories: [data-eng,theory,applying-knowledge]
 ---
 
-# Hello :) Today is Day 340!
+# Hello :) Today is Day 341!
 A quick summary of today:
 * long stream
 * starting the book Stream Processing with Apache Flink

From eb6c8988b3dae5f16f00d9b9b21af5ff52e61c58 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 8 Dec 2024 23:45:24 +0900
Subject: [PATCH 524/638] Create 2024-12-08-day-342.md

---
 _posts/2024-12-08-day-342.md | 53 ++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 _posts/2024-12-08-day-342.md

diff --git a/_posts/2024-12-08-day-342.md b/_posts/2024-12-08-day-342.md
new file mode 100644
index 000000000..11918af3d
--- /dev/null
+++ b/_posts/2024-12-08-day-342.md
@@ -0,0 +1,53 @@
+---
+layout: post
+title: (Day 342) Apache Flink 101 + Data Quality
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 342!
+A quick summary of today:
+* apache flink 101 by confluent
+* two more homework As
+* DQ basics
+
+## [Apache Flink 101 on stream for ~2.5hrs](https://www.youtube.com/watch?v=QPrcCC-lrZQ)
+
+Today I found more free courses on streaming by Confluent [here](https://developer.confluent.io/courses/?course=for-developers#fundamentals)
+
+The course I covered yesterday (ApacheKafka 101) was the 1st course in the series, and today I covered the 2nd - [Apache Flink 101](https://developer.confluent.io/courses/apache-flink/intro/)
+
+The overview:
+
+* what Apache Flink is, and why you might use it
+* what stream processing is, and how it differs from batch processing
+* Flink’s runtime architecture
+* how to use Flink and Kafka together
+* how to use Flink SQL: tables, windows, event time, watermarks, and more
+* stateful stream processing
+* how watermarks support event time operations
+* how Flink uses snapshots (checkpoints) for fault tolerance
+
+The course containes short videos with very well-made explanations and also short exercises where I got to run Flink locally, and play around with FlinkSQL
+
+## Submitted next two homeworks for Zach Wilson's YT bootcamp
+
+![image](https://github.com/user-attachments/assets/638a4212-7b1a-4d31-b0eb-afa966d8cbdc)
+
+* 3rd - Spark queries (joins, aggregations)
+* 4th - converting 2 SQL queries to SparkSQL/Dataframe API jobs and writing a chispa test for them
+
+I already had the homework finished but I got a B at first because I forgot the include some parts of my code as I was doing dev in a notebook, but did not add the code to the submission .py file. Anyway got A for both 🥳
+
+## Some DQ basics from Zach Wilson's [latest lecture](https://www.youtube.com/watch?v=JiedBnTFCeg)
+
+![Gold pipelines MIDAS P1](https://github.com/user-attachments/assets/f93af940-c625-4fb5-842f-04dc962d54cc)
+![Gold pipelines MIDAS P2](https://github.com/user-attachments/assets/eb3efcdc-9245-498a-b73c-6b729db790ab)
+![Gold pipelines MIDAS P3](https://github.com/user-attachments/assets/35f1921c-21c3-47d4-a801-4fd858b945b8)
+
+---
+
+After stream, I wanted to finish a report I am writing for the company reviewer LLM I am fine-tuning for my professor. I think I will share the full one tomorrow :)
+
+That is all for today!
+
+See you tomorrow :)

From 0c7301011c1f606c52fb61539cc53d2c528e1c27 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 9 Dec 2024 22:46:49 +0900
Subject: [PATCH 525/638] Create 2024-12-09-day-343.md

---
 _posts/2024-12-09-day-343.md | 54 ++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 _posts/2024-12-09-day-343.md

diff --git a/_posts/2024-12-09-day-343.md b/_posts/2024-12-09-day-343.md
new file mode 100644
index 000000000..6aaa334b7
--- /dev/null
+++ b/_posts/2024-12-09-day-343.md
@@ -0,0 +1,54 @@
+---
+layout: post
+title: (Day 342) First Company Reviewer LLM Report + Dremio Verified Lakehouse Associate
+categories: [data-eng,theory,applying-knowledge,nlp,reading-research]
+---
+
+# Hello :) Today is Day 342!
+A quick summary of today:
+* submitting my 1st report for the company reviewer LLM
+* covering some courses on the Dremio university platform
+* stream
+
+## First Company Reviewer LLM Report ✔️
+
+[Here is a docs link](https://docs.google.com/document/d/1Feu4Zg076_PPLCh-9EiEb0-NEIRYVOa822LF5CyhLe8/edit?usp=sharing). It is 10 pages total, but actually it's not that long as there are many of tables. This is the 1st report I am writing for this project and I followed some base guidelines on including information about data, model selection, parameters, results, and discussion. 
+
+I send it to my lab professor so I am waiting for a response ~ 
+
+This took most of my time last night/today.
+
+---
+
+## Dremio University and getting a Dremio Verified Lakehouse Associate badge
+
+I saw that Alex Merced (one of the authors of the Apache Iceberg: The Definitive Guide) shared a new page by Dremio called [Dremio University](https://university.dremio.com/courses), which contains some cool short courses taught by Alex himself. I decided to check the courses for the free Dremio Verified Lakehouse Associate badge and they were amazing.
+
+![image](https://github.com/user-attachments/assets/8cc2e6c8-4b52-4313-886e-c255c91473fe)
+
+The material covered was partially from the book itself but it was good. 
+
+Here is the text extracted from the image:
+
+- What is Data Lakehouse?  
+- What is a Table Format?  
+- What is Apache Iceberg?  
+- How Iceberg Works  
+- Example Transactions  
+- Benefits of Apache Iceberg  
+- Apache Iceberg Catalogs  
+- Ingesting Into Iceberg with Dremio  
+
+I am definitely understanding more how Iceberg works, its benefits, and how to explain it to someone new - which I tried to do today and I think it went well.
+
+---
+
+## [Streamed](https://www.youtube.com/watch?v=7pBGkl6VDx4)
+
+I decided to go on stream and do one of the other courses on the Dremio University platform - [Iceberg, Spark and Dremio - A Hands on Experience](https://university.dremio.com/course/am-hands-on-with-apache-iceberg). I wanted to do it on stream as it has both video and practical parts but it turned out I had already covered the practical parts in an older stream, and also the theoretical part was something that I read in the book, and already covered in the associate badge course. Nevertheless, it's good videos by Alex!
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From a851a7af5343bc2dec7592b8f8b82b5874921756 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 9 Dec 2024 22:52:30 +0900
Subject: [PATCH 526/638] Update 2024-12-09-day-343.md

---
 _posts/2024-12-09-day-343.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_posts/2024-12-09-day-343.md b/_posts/2024-12-09-day-343.md
index 6aaa334b7..4c1272556 100644
--- a/_posts/2024-12-09-day-343.md
+++ b/_posts/2024-12-09-day-343.md
@@ -47,6 +47,8 @@ I am definitely understanding more how Iceberg works, its benefits, and how to e
 
 I decided to go on stream and do one of the other courses on the Dremio University platform - [Iceberg, Spark and Dremio - A Hands on Experience](https://university.dremio.com/course/am-hands-on-with-apache-iceberg). I wanted to do it on stream as it has both video and practical parts but it turned out I had already covered the practical parts in an older stream, and also the theoretical part was something that I read in the book, and already covered in the associate badge course. Nevertheless, it's good videos by Alex!
 
+<img width="822" alt="image" src="https://github.com/user-attachments/assets/7a407473-28f4-4876-8734-9dea6f396093">
+
 ---
 
 That is all for today!

From 115e85f8b4eecd80bdac29cff82a3b528767db7f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 10 Dec 2024 09:23:38 +0900
Subject: [PATCH 527/638] fix date

---
 _posts/2024-12-09-day-343.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-12-09-day-343.md b/_posts/2024-12-09-day-343.md
index 4c1272556..9f3da3c52 100644
--- a/_posts/2024-12-09-day-343.md
+++ b/_posts/2024-12-09-day-343.md
@@ -1,10 +1,10 @@
 ---
 layout: post
-title: (Day 342) First Company Reviewer LLM Report + Dremio Verified Lakehouse Associate
+title: (Day 343) First Company Reviewer LLM Report + Dremio Verified Lakehouse Associate
 categories: [data-eng,theory,applying-knowledge,nlp,reading-research]
 ---
 
-# Hello :) Today is Day 342!
+# Hello :) Today is Day 343!
 A quick summary of today:
 * submitting my 1st report for the company reviewer LLM
 * covering some courses on the Dremio university platform

From f38a42ce956a3baa491d06ab92cf5c63c90fd39a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 11 Dec 2024 01:03:25 +0900
Subject: [PATCH 528/638] Create 2024-12-10-day-344.md

---
 _posts/2024-12-10-day-344.md | 627 +++++++++++++++++++++++++++++++++++
 1 file changed, 627 insertions(+)
 create mode 100644 _posts/2024-12-10-day-344.md

diff --git a/_posts/2024-12-10-day-344.md b/_posts/2024-12-10-day-344.md
new file mode 100644
index 000000000..2a746f4e2
--- /dev/null
+++ b/_posts/2024-12-10-day-344.md
@@ -0,0 +1,627 @@
+---
+layout: post
+title: (Day 344) Checking out the first chapters from Andriy Burkov's new book
+categories: [data-eng,theory,applying-knowledge,nlp,reading-research,mathematics]
+---
+
+# Hello :) Today is Day 344!
+A quick summary of today:
+* new Andriy Burkov book
+* more about Flink and Kafka
+
+
+## [Andriy Burkov's new book on LLMs](https://thellmbook.com/)
+
+I saw that Andriy Burkov put out the first 4 chapters of his upcoming book online for free ~ so it's time to check them out.
+
+---
+
+## Chaprer 1 - ML Basics
+
+### AI
+
+* first chatbot ELIZA in 1966 that answered depending on input pattern matching
+* AI winters 1974-1980 and 1987-2000
+
+<img width="472" alt="image" src="https://github.com/user-attachments/assets/dfd605a2-c849-4766-852c-2e8871a92ecc">
+
+* the term AI was somewhat of a taboo during AI winters and researchers renamed their works to things like pattern matching, smart decision systems
+* things picked up again since 2012
+
+### ML
+
+* decision boundaries
+* decision trees
+* random forests
+* SVMs
+
+### Model
+
+* y = f(x)
+* f(x) = wx + b
+* MSE and the loss function
+* an overview of differentiation and derivatives
+
+### 4-step ML process
+
+1. collect a dataset
+2. define the model's structure
+3. define the loss function
+4. minimise the loss function
+
+---
+
+## Chapter 2 - Vectors, Matrices, and Neural networks
+
+### Vectors
+
+* dot product
+* element-wise sum and product
+* unit vectors, zero vectors, vector norm
+
+<img width="670" alt="image" src="https://github.com/user-attachments/assets/a40feca2-0d56-441a-bd4f-d3f20169d58b">
+
+_Unit vectors are useful since their dot product equals the cosine of the angle between them, and computing the dot product is fast. When documents are represented as unit vectors, finding similar ones to a query becomes efficient by calculating the dot product between the query’s vector and the document vectors._
+
+### Neural networks
+
+* a fixed non-linear function is applied to the outputs of trainable linear functions
+* deeper structure compared to linear models
+
+They are of the structure: ф(wx + b), where ф is an activation function. And with layers these are stacked
+
+<img width="423" alt="image" src="https://github.com/user-attachments/assets/34c3b73a-afb6-4599-949c-45e6e08b5471">
+
+### Matrices
+
+* sum and product of matrices
+* transposing a matrix
+* matrix-vector operations
+
+There are these very neat graphs showing how it works:
+
+<img width="728" alt="image" src="https://github.com/user-attachments/assets/9d34e308-a2de-4e7e-97c0-d3b2b2402c1c">
+
+### Gradient Descent
+
+* intro to the cross-entropy loss
+* intro to gradient descent
+  * initialise params
+  * compute preds
+  * compute the gradient
+  * update w and b based on a learning rate
+  * calculate the loss
+  * iterate
+
+---
+ 
+## Chapter 3 - Representing text
+
+### Bag of Words
+
+1. create a vocabulary: a list all unique words in the corpus
+2. vectorize documents: represent each document as a feature vector where each dimension of the vector indicates whether a word from the vocabulary is present, absent, or its frequency in the document
+
+Feature vectors can be organized into a document-term matrix, where rows represent documents, and columns represent tokens.
+
+<img width="721" alt="image" src="https://github.com/user-attachments/assets/765b00d1-e209-4dd9-9483-0a1117bd6ec7">
+
+_In natural languages, word frequencies follow Zipf’s Law. According to the law, a word’s frequency is inversely proportional to its rank in a frequency table. For example, the second most frequent word occurs half as often as the most frequent one. Consequently, most words in a document-term matrix are rare, resulting in a sparse matrix filled with many zeros._
+
+BoW cannot capture order or context of words - for this n-grams can be used. But for n-grams the problem becomes that the vocab grows significantly. BoW also does not work well with out-of-bag words - words not seen in the original vocabulary. This is all addressed with word embeddings
+
+### Word embeddings
+
+Word embeddings address the limitations of traditional bag-of-words (BoW) and one-hot encoding by representing words as dense vectors that encode semantic relationships. Unlike one-hot vectors, which are sparse and carry little information, embeddings are lower-dimensional, non-sparse, and map semantically similar words like films and movies to vectors with high cosine similarity.
+
+The word2vec skip-gram algorithm is a popular method for learning embeddings. It trains a model to predict a word's context from the word itself or vice versa
+
+<img width="574" alt="image" src="https://github.com/user-attachments/assets/97019a5b-af8e-44e2-9ec9-f0334fe942bf">
+
+The image is of a skip-gram model with a skip-gram size of 5 and the embedding layer of 300 units
+
+Word embeddings are fundamental in NLP and modern models also integrate subword tokenization, such as byte-pair encoding, to enhance representation capabilities.
+
+### Byte-Pair Encoding (BPE)
+
+It tries to handle the difficulty of handling out-of-bag words by splitting words into smaller pieces called subwords
+
+1. Initialization:  
+   - use a text corpus, splitting each word into individual characters (i.e 'hello' -> "h e l l o")
+   - the initial vocabulary comprises all unique characters in the corpus
+
+2. Iterative merging:  
+   - count adjacent symbol pairs: identify all pairs of adjacent symbols in the corpus (e.g. 'h e', 'e l', 'l l', 'l o')
+   - select the most frequent pair: choose the pair with the highest count (e.g.'l l')  
+   - merge the selected pair: replace all occurrences of the pair with a single merged symbol (e.g. 'l l' → 'll', transforming 'h e l l o' into 'h e ll o')  
+   - update the vocabulary: add the new merged symbol to the vocabulary
+
+3. Repeat:  
+   - continue merging the most frequent pairs until the vocabulary reaches the desired size
+
+---
+
+## Chapter 4 - Language Model Basics
+
+### Language model
+
+An LM predicts the next token in a sequence by estimating its conditional probability based on previous tokens. It assigns a probability to all possible next tokens, enabling the selection of the most likely next one. Such models are also called autoregressive because for the next prediction they only use the preceding elements.
+
+There are also masked LMs like BERT that mask a word in a sentence and try to predict it based on its surroundings
+
+### Evaluating LMs
+
+* perplexity: a perplexity of 5 (for instance) means that, on average, the model acts as if it selects from roughly 5 equally likely options for each word in the given context. A perplexity of 5 is relatively low, showing the model is confident in its predictions for this sequence. But for a proper evaluation, perplexity must be measured on a much larger test set
+* ROUGE: used for comparing summarisations and machine translations
+* human evaluation (likert scale ratings, pairwise comparison, elo ratings)
+
+---
+
+## Stream Processing with Apache Flink - Chapter 2 - Stream Processing Fundamentals
+
+### Introduction to dataflow programming
+
+**Dataflow graphs**
+
+ A dataflow program describes how data flows between operations
+
+![image](https://github.com/user-attachments/assets/0cccbb92-b750-40d5-a89a-60a0ad5a59e5)
+
+This is a logical dataflow graph to continuously count hashtags (nodes represent operators and edges denote data dependencies)
+
+The above is a logical graphe because it shows a high-level view of the computational logic
+
+In order to execute a dataflow program, its logical graph is converted into a physical dataflow graph, which specifies in detail how the program is executed. If we use distributed computing, the physical plan might look like:
+
+![image](https://github.com/user-attachments/assets/757945e5-3194-4623-97c7-dca5f4c06bf3)
+
+**Data parallelism and Task parallelism**
+
+We can exploit parallelism in dataflow graphs in different ways
+
+* we can partition the input data and have tasks of the same operation execute on the data subsets in parallel (data parallelism)
+* we can have tasks from different operators performing computations on the same or different data in parallel (task parallelism)
+
+**Data exchange strategies**
+
+They define how data items are assigned to tasks in a physical dataflow graph. They can also be automatically chosen by the execution engine depending on the semantics of the operators or explicitly imposed by the dataflow programmer. Here are some strategies:
+
+* forward: it sends data from a task to a receiving task. If both tasks are located on the same physical machine (which is often ensured by task schedulers), this exchange strategy avoids network communication
+* broadcast: it sends every data item to all parallel tasks of an operator. Because this strategy replicates data and involves network communication, it is fairly expensive
+* key-based: it partitions data by a key attribute and guarantees that data items having the same key will be processed by the same task
+* random: it uniformly distributes data items to operator tasks in order to evenly distribute the load across computing tasks
+
+![image](https://github.com/user-attachments/assets/0f299469-dfde-4280-9a14-bd42f6c81fd2)
+
+### Processing streams in parallel
+
+**Latency and Throughput**
+
+Latency indicates how long it takes for an event to be processed. Essentially, it is the time interval between receiving an event and seeing the effect of processing this event in the output.
+
+Throughput is a measure of the system’s processing capacity—its rate of processing. That is, throughput tells us how many events the system can process per time unit.
+
+**Operations on data streams**
+
+* stateless vs stateful operations
+
+This was covered **very** well by the Kafka 101 on Confluent. Definitely recommend that course. And to be honest this whole book's material is probably covered by the free video courses on Confluent 😆 I am reading in order to confirm my knowledge and see if there is something new mentioned.
+
+### Time semantics
+
+**Processing time**
+
+_Processing time is the time of the local clock on the machine where the operator processing the stream is being executed. A processing-time window includes all events that happen to have arrived at the window operator within a time period, as measured by the wall clock of its machine_
+
+![image](https://github.com/user-attachments/assets/aa8d1c22-74c4-4a2e-b743-9f19cc14b25e)
+
+**Event time**
+
+Event time is the time when an event in the stream actually happened. Event time is based on a timestamp that is attached to the events of the stream. Timestamps usually exist inside the event data before they enter the processing pipeline (e.g. the event creation time)
+
+![image](https://github.com/user-attachments/assets/7c77065f-03de-4740-b862-b85c56257b1c)
+
+Operations based on event time are predictable and their results are deterministic. An event time window computation will yield the same result no matter how fast the stream is processed or when the events arrive at the operator.
+
+Handling even delays can be a challenge
+
+**Watermarks**
+
+A progress metric used in event-time window processing to signal when all events up to a certain timestamp are expected to have been received. They help manage out-of-order events and trigger computations based on event time.
+
+Tradeoffs
+
+* eager watermarks: lower latency but less confidence, requiring mechanisms to handle late events
+* relaxed watermarks: higher confidence but increased processing latency
+
+Challenges:
+
+* perfect watermarking is difficult due to unpredictable delays in distributed systems (i.e straggler tasks, disconnected users)
+* systems must handle late events through strategies like ignoring, logging, or using them to adjust results
+
+**Processing vs Event time**
+
+Processing time and event time serve different purposes in data processing. Processing time is advantageous when low latency is critical, as it immediately triggers computation after a specified duration, ignoring late or out-of-order events. This makes it suitable for applications prioritizing speed over accuracy, such as real-time monitoring dashboards or scenarios requiring a direct representation of event streams, like outage detection. However, results depend on processing speed and lack determinism. Conversely, event time ensures deterministic results by handling late or out-of-order events, making it ideal for use cases requiring accuracy and completeness over immediacy.
+
+### State and consistency models
+
+State operations impose a few challenges:
+
+* state management
+* state partitioning
+* state recovery
+
+**Task failures**
+
+For each event in the input stream, a task is a processing step that performs the following steps: 
+
+1. receives the event, storing it in a local buffer
+2. possibly updates internal state
+3. produces an output record.
+
+A failure can occur during any of these steps and the system has to clearly define its behavior in a failure scenario.
+
+**Result guarantees**
+
+* at-most-once
+* at-least-once
+* exactly-once
+* end-to-end exactly-once
+
+---
+
+## [Kafka Streams 101](https://developer.confluent.io/courses/kafka-streams/get-started/)
+
+<img width="861" alt="image" src="https://github.com/user-attachments/assets/65121ebe-ecd4-49fa-b58b-48f7e9bee071">
+
+At the heart of Kafka is the log, which is simply a file where records are appended. The log is immutable, but you usually can't store an infinite amount of data, so you can configure how long your records live.
+
+The storage layer for Kafka is called a broker, and the log resides on the broker's filesystem. A topic is simply a logical construct that names the log—it's effectively a directory within the broker's filesystem.
+
+You put data into Kafka with producers and get it out with consumers: Producers send a produce request with records to the log, and each record, as it arrives, is given a special number called an offset, which is just the logical position of that record in the log. Consumers send a fetch request to read records, and they use the offsets to bookmark, like placeholders. For example, a consumer will read up to offset number 5, and when it comes back, it will start reading at offset number 6. Consumers are organized into groups, with partition data distributed among the members of the group.
+
+**Connectors**
+
+Connectors are an abstraction over producers and consumers. Perhaps you need to export database records to Kafka. You configure a source connector to listen to certain database tables, and as records come in, the connector pulls them out and sends them to Kafka. Sink connectors do the opposite: If you want to write records to an external store such as MongoDB, for example, a sink connector reads records from a topic as they come in and forwards them to your MongoDB instance.
+
+**Definition**
+
+Kafka Streams is a java library: we write code, create a JAR file, and then start our standalone application that streams records to and from Kafka (it doesn't run on the same node as the broker). And we can run Kafka Streams on anything from a laptop all the way up to a large server.
+
+### Basic operations
+
+* mapping (`map((key, value) -> ..)`)
+* map values (`mapValues(value -> value.substring(5))`)
+* filtering (`filter((key, value) -> Long.parseLong(value) > 1000)`)
+
+It's important to think of these operations as creating a new event stream, rather than modifying an existing one.
+
+### KTables
+
+At the start the videos mentioned that keys across records in event streams are completely independent of one another, even if they are identical.
+
+Update Streams are the exact opposite: if a new record comes in with the same key as an existing record, the existing record will be overwritten.
+
+**Define a KTable**
+
+```java
+ StreamsBuilder builder = new StreamsBuilder();
+ KTable<String, String> firstKTable = 
+    builder.table(inputTopic, 
+    Materialized.with(Serdes.String(), Serdes.String()));
+```
+
+**Operations**
+
+* mapping
+* mapping values
+* filtering
+
+**GlobalKTable**
+
+The main difference between a KTable and a GlobalKTable is that a KTable shards data between Kafka Streams instances, while a GlobalKTable extends a full copy of the data to each instance. You typically use a GlobalKTable with lookup data. There are also some idiosyncrasies regarding joins between a GlobalKTable and a KStream. 
+
+```java
+ StreamsBuilder builder = new StreamsBuilder();
+ GlobalKTable<String, String> globalKTable = 
+    builder.globalTable(inputTopic, 
+    Materialized.with(Serdes.String(), Serdes.String()));
+```
+
+### Serialisation
+
+<img width="746" alt="image" src="https://github.com/user-attachments/assets/4e1ac004-e385-4290-91ad-9a8ee3abab2f">
+
+The Kafka brokers receive and send bytes
+
+### JOINs
+
+```java
+KStream<String, String> leftStream = builder.stream("topic-A");
+KStream<String, String> rightStream = builder.stream("topic-B");
+
+ValueJoiner<String, String, String> valueJoiner = (leftValue, rightValue) -> {
+    return leftValue + rightValue;
+};
+leftStream.join(rightStream, 
+                valueJoiner, 
+                JoinWindows.of(Duration.ofSeconds(10)));
+```
+
+- Stream-Stream joins:
+  - combine two event streams into a new event stream
+  - join events based on a common key
+  - records arrive within a defined window of time
+  - possible to compute a new value type
+  - keys are available in read-only mode and can be used in computing the new value
+  
+- Stream-Table joins:
+  - KStream-KTable
+  - KStream-GlobalKTable
+
+- Table-Table joins
+
+### Stateful operations
+
+* reduce
+
+```java
+StreamsBuilder builder = new StreamsBuilder();
+KStream<String, Long> myStream = builder.stream("topic-A");
+Reducer<Long> reducer = (longValueOne, longValueTwo) -> longValueOne + longValueTwo;
+myStream.groupByKey().reduce(reducer,
+                             Materialized.with(Serdes.String(), Serdes.Long()))
+                            .toStream().to("output-topic");
+```
+
+* aggregate
+
+```java
+StreamsBuilder builder = new StreamsBuilder();
+KStream<String, String> myStream = builder.stream("topic-A");
+
+Aggregator<String, String, Long> characterCountAgg = 
+    (key, value, charCount) -> value.length() + charCount;
+myStream.groupByKey().aggregate(() -> 0L, 
+                                      characterCountAgg, 
+                                      Materialized.with(Serdes.String(), Serdes.Long()))
+                                      .toStream().to("output-topic");
+```
+
+Stateful operations don't emit results immediately. Kafka Streams has an internal buffering mechanism that caches results. Two factors control when the cache emits records: Records are emitted when the cache is full (defined equally per instance among the number of stores; it's 10MB), and by default, Kafka Streams calls commit every 30 seconds (you don't call commit yourself). At this point, you would see an update. In order to see every update that comes through your aggregation, you can set your cache size to zero (which is also useful for debugging).
+
+Even with caching, you will get multiple results, so for a single and final stateful result, you should use suppression overloads with aggregate/reduce operations.
+
+### Windowing
+
+**Hopping**
+
+A hopping window is bound by time: You define the size of the window, but then it advances in increments smaller than the window size, so you end up with overlapping windows. You might have a window size of 30 seconds with an advance size of five seconds. Data points can belong to more than one window.
+
+**Tumbling**
+
+It's a hopping window with an advance size that's the same as its window size. So basically you just define a window size of 30 seconds. When 30 seconds are up, you get a new window with a time of 30 seconds. So you don't get duplicate results like you do with the overlapping in hopping windows.
+
+**Sessions**
+
+So with session windows, you define an inactivityGap. Basically, as long as a record comes in within the inactivityGap, your session keeps growing. So theoretically, if you're keeping track of something and it's a very active key, your session will continue to grow.
+
+**Sliding**
+
+A sliding window is bound by time, but its endpoints are determined by user activity
+
+**Grade periods**
+
+With the exception of session windows, which are behavior-driven, all windows have the concept of a grace period. A grace period is an extension to the size of a window. Specifically, it allows events with timestamps greater than the window-end (but less than the window-end plus the grace period) to be included in the windowed calculation.
+
+(I am writing this as I watch, and there are lots of overlaping material with the previous Kafka 101 and Flink 101 videos, which is kind of understandable since we are talking about streaming but I guess these courses are meant to be standalone rather than taken in a series one after another)
+
+### Testing streams
+
+Kafka Streams unit testing can be challenging due to the need for broker connections, which are expensive for tests. While integration tests with a live broker are necessary, they should be limited. The `TopologyTestDriver` addresses this by allowing unit tests for a Kafka Streams topology without requiring a broker, supporting end-to-end tests at a faster pace. It works by mocking the Kafka components and using a `MockSchemaRegistry` for Schema Registry-related tests.
+
+In the `TopologyTestDriver`, you create and configure your topology, instantiate the driver, and then use `TestInputTopic` to send records through the topology. The records can have timestamps for triggering stream-time behavior, and wall-clock time can be advanced using `advanceWallClockTime`.
+
+`TestOutputTopic` is used to mock the sink nodes, allowing verification of results. For Kafka Streams DSL operators using lambdas, it's often better to write a concrete class for easier unit testing.
+
+Though `TopologyTestDriver` is useful for unit tests, integration testing against a live broker is still essential, particularly for stateful operations. The `TestContainers` library is recommended for controlling the broker lifecycle in integration tests, offering reduced testing time by sharing containers across tests.
+
+---
+
+## [Intro to ksqlDB](https://developer.confluent.io/courses/ksqldb/intro/)
+
+### Definition
+
+ksqlDB allows you to build stream processing applications on top of Apache Kafka with the ease of building traditional applications on a relational database. Using SQL to describe what you want to do rather than how, it makes it easy to build Kafka-native applications for processing streams of real-time data. Some key ksqlDB use cases include:
+
+* materialized caches
+* streaming ETL pipelines
+* event-driven microservices
+
+![image](https://github.com/user-attachments/assets/50b9fee4-9fee-413e-bffc-c5f795203203)
+
+If we have a stream, some basic operations we can with ksql:
+
+* filter
+* join streams
+* aggregate streams
+
+<img width="854" alt="image" src="https://github.com/user-attachments/assets/7562cec5-ad75-47c6-9c07-d7ebf1344ce0">
+
+### Hands-on
+
+Creating a stream
+
+```sql
+CREATE STREAM MOVEMENTS (PERSON VARCHAR KEY, LOCATION VARCHAR)
+  WITH (VALUE_FORMAT='JSON', PARTITIONS=1, KAFKA_TOPIC='movements');
+```
+
+Add data
+
+```sql
+INSERT INTO MOVEMENTS VALUES ('Allison', 'Denver');
+INSERT INTO MOVEMENTS VALUES ('Robin', 'Leeds');
+INSERT INTO MOVEMENTS VALUES ('Robin', 'Ilkley');
+INSERT INTO MOVEMENTS VALUES ('Allison', 'Boulder');
+```
+
+<img width="1249" alt="image" src="https://github.com/user-attachments/assets/cd448e02-e404-4bee-817d-5356106fb85f">
+
+Filtering
+
+```sql
+CREATE STREAM ORDERS_NY AS
+  SELECT *
+    FROM ORDERS
+   WHERE ADDRESS->STATE='New York';
+```
+
+JOINs
+
+```sql
+CREATE STREAM ORDERS_ENRICHED AS
+SELECT O.*,
+       I.*,
+       O.ORDERUNITS * I.UNIT_COST AS TOTAL_ORDER_VALUE,
+  FROM ORDERS O
+       LEFT OUTER JOIN ITEMS I
+       ON O.ITEMID = I.ID ;
+```
+
+Flattening nested records
+
+```sql
+CREATE STREAM ORDERS_FLAT AS
+  SELECT TIMESTAMPTOSTRING(ORDERTIME, 'yyyy-MM-dd HH:mm:ss') AS ORDER_TIMESTAMP,
+         ORDERID,
+         ITEMID,
+         ORDERUNITS,
+         ADDRESS->STREET AS ADDRESS_STREET,
+         ADDRESS->CITY   AS ADDRESS_CITY,
+         ADDRESS->STATE  AS ADDRESS_STATE
+    FROM ORDERS;
+```
+
+Splitting streams
+
+```sql
+CREATE STREAM ORDERS_US AS
+   SELECT * FROM ORDERS_COMBINED
+   WHERE SOURCE = 'US';
+
+CREATE STREAM ORDERS_UK AS
+   SELECT * FROM ORDERS_COMBINED
+   WHERE SOURCE = 'UK';
+
+CREATE STREAM ORDERS_OTHER AS
+   SELECT * FROM ORDERS_COMBINED
+   WHERE SOURCE != 'US'
+   AND SOURCE != 'UK';
+```
+
+Stateful aggregations
+
+```sql
+CREATE TABLE ORDERS_PER_HOUR_BY_MAKE AS
+  SELECT MAKE,
+         COUNT(*) AS ORDER_COUNT,
+         CAST(SUM(TOTAL_ORDER_VALUE) AS DECIMAL(9,2)) AS TOTAL_ORDER_VALUE
+    FROM ORDERS_ENRICHED
+          WINDOW TUMBLING (SIZE 1 HOUR)
+  GROUP BY MAKE
+  EMIT CHANGES;
+```
+
+Pull vs Push queries
+
+* Pull is like a normal select - shows a point in time view
+* Push has the `EMIT CHANGES` which runs indefinitely; it tells us all value changes
+
+Transform functions
+
+```sql
+CREATE STREAM transformed 
+  AS SELECT id,
+  TRANSFORM(map_field,(k,v)=> UCASE(k), (k, v)=> (v * 2))
+FROM stream1 EMIT CHANGES;
+```
+
+Reduce functions
+
+```sql
+CREATE STREAM reduced 
+  AS SELECT name,
+  REDUCE(scores,0,(s,x)=> (s+x)) AS total
+FROM stream2 EMIT CHANGES;
+```
+
+Filter functions
+
+```sql
+CREATE STREAM filtered 
+  AS SELECT id,
+  FILTER(numbers,x => (x%2 = 0)) AS even_numbers
+FROM stream3 EMIT CHANGES;
+```
+
+---
+
+## [Schema Registry 101](https://developer.confluent.io/courses/schema-registry/key-concepts/)
+
+<img width="849" alt="image" src="https://github.com/user-attachments/assets/6257b395-3e8a-4ea3-ba2c-c644a2a46b72">
+
+<img width="850" alt="image" src="https://github.com/user-attachments/assets/b373dd67-5522-44a9-8e58-df9e0742548a">
+
+<img width="850" alt="image" src="https://github.com/user-attachments/assets/7255518c-878e-4848-b000-8b2676e414d2">
+
+### The schema workflow
+
+Example writing a schema file with Avro
+
+![image](https://github.com/user-attachments/assets/e9cdd587-07b5-4ef5-b821-ad2eaa97e7b4)
+
+In Protobuf
+
+![image](https://github.com/user-attachments/assets/8de637d0-ff04-43e4-b0a7-688147d64104)
+
+There are also more complex data types that we can include in the schemas (maps, enums, unions, arrays) 
+
+Effective schema management requires using/doing:
+
+* schema IDs
+* schema registration
+* schema versioning
+* viewing and retrieving schemas
+
+**Schema lifecycle**
+
+![image](https://github.com/user-attachments/assets/a12fe055-cdbe-439f-bc27-f9845b8c9fe9)
+
+### Schema compatibility
+
+Producer and consumer clients have expectations of the structure of the object they’re working with.
+
+Schema Registry offers compatibility verification. The available compatibility modes and associated checks establish guardrails that guide you to safely update schemas and allow you to keep your clients operational as they are updated with the changes.
+
+**Schema compatibility modes**
+
+![image](https://github.com/user-attachments/assets/18cd2de1-8b4f-48f5-8db4-4111c88e769b)
+
+**Transitive schema compatibility modes**
+
+![image](https://github.com/user-attachments/assets/d3810159-c40b-4a98-8802-2ccbfee0127d)
+
+**Schema compatibility considerations**
+
+![image](https://github.com/user-attachments/assets/6ed7b9e1-6155-49f7-a3bd-9f9b1e160f86)
+
+* with Backward compatibility you can delete fields and add fields with default values and you update the consumer clients first
+* with Forward compatibility you can delete fields with default values and add new fields. You need to update your producer clients first under forward compatibility
+* with Full compatibility, both deleted or added fields must have a default value. Since every change to the schema has a default value, the order in which you update your clients doesn’t matter
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 719d4edf531ebf1dc3111bdb212a8e5a0966dc36 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 11 Dec 2024 22:48:48 +0900
Subject: [PATCH 529/638] Create 2024-12-11-day-345.md

---
 _posts/2024-12-11-day-345.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 _posts/2024-12-11-day-345.md

diff --git a/_posts/2024-12-11-day-345.md b/_posts/2024-12-11-day-345.md
new file mode 100644
index 000000000..97ef078e3
--- /dev/null
+++ b/_posts/2024-12-11-day-345.md
@@ -0,0 +1,27 @@
+---
+layout: post
+title: (Day 345) Using Apache Iceberg for a WAP exercise
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 345!
+A quick summary of today:
+* new video by Zach Wilson
+* new video by me ?!?!?
+
+## [The Write-Audit-Publish pattern](https://www.youtube.com/watch?v=ja7VOqDVkuo)
+
+![WAP P1](https://github.com/user-attachments/assets/480e6de7-a9d5-46f9-8c96-a94c4ed6909f)
+![WAP P2](https://github.com/user-attachments/assets/ce27521f-05d3-4afa-91e6-78585cc98244)
+
+Zach mentioned there won't be a lab for this lecture and given how much Apache Iceberg content I have been **consuming** the last week I decided to give it a go and create some kind of exercise. Especially, since I had read a random blog post about branching in iceberg. Ultimately, I learned a lot about the practical side of WAP. Even if my implementation and example were **super** simple 😆
+
+**[Here is my video](https://www.youtube.com/watch?v=CzkBkwv1Llc)**
+
+Every time I decide to record a video for something I do not remember how draining it is since I am so slow when it comes to editing 😆 and I can't edit well either 😆 so in terms of practicing writing code and also explaining things in a beginner-friendly way 💯💯💯 it is worth it. But after the recording is done, it is soo draining for me 😆
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From f0dae9ab5d96dc28be85e5306f451e4ab3169dec Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Dec 2024 22:43:48 +0900
Subject: [PATCH 530/638] Create 2024-12-12-day-346.md

---
 _posts/2024-12-12-day-346.md | 173 +++++++++++++++++++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100644 _posts/2024-12-12-day-346.md

diff --git a/_posts/2024-12-12-day-346.md b/_posts/2024-12-12-day-346.md
new file mode 100644
index 000000000..3c016f6f8
--- /dev/null
+++ b/_posts/2024-12-12-day-346.md
@@ -0,0 +1,173 @@
+---
+layout: post
+title: (Day 346) Some Flink + Airflow 101
+categories: [data-eng,theory,applying-knowledge,mlops]
+---
+
+# Hello :) Today is Day 346!
+A quick summary of today:
+* chapter 3 of Stream Processing with Apache Flink
+* reading the Flink learn docs
+* starting an Airflow 101 course by Astronomer
+
+---
+
+## Stream Processing with Apache Flink - Chapter 3 - The Architecture of Apache Flink
+
+### System architecture
+
+**Components of a Flink setup**
+
+A Flink setup consists of four different components that work together to execute streaming applications. These components are a JobManager, a ResourceManager, a TaskManager, and a Dispatcher. Since Flink is implemented in Java and Scala, all components run on JVMs
+
+* JobManager is the master process controlling the execution of a single application in Flink. When an application is submitted, it receives a JobGraph and accompanying JAR file, which it then transforms into an ExecutionGraph—a physical dataflow graph of parallel tasks. The JobManager's critical role involves requesting the necessary TaskManager slots from the ResourceManager and subsequently distributing these tasks across available TaskManagers. Throughout the execution, it maintains central coordination responsibilities, such as managing checkpoints and ensuring the overall orchestration of the application's workflow
+* ResourceManager acts as the resource allocation backbone of Flink. Its primary function is managing TaskManager slots, which represent Flink's fundamental processing resources. When a JobManager requests slots, the ResourceManager coordinates with TaskManagers to offer available processing capacity. If insufficient slots are available, it can interact with the underlying resource provider to provision additional containers for launching new TaskManager processes. This dynamic resource management ensures efficient utilization of computational resources and enables flexible scaling of Flink applications
+* TaskManagers serve as the worker processes in the Flink ecosystem, typically running multiple instances within a cluster. Each TaskManager provides a specific number of slots that determine its task execution capacity. After initialization, these workers register their slots with the ResourceManager and await instructions to offer them to JobManagers. Once assigned tasks, TaskManagers execute them and maintain data exchange capabilities with other TaskManagers running tasks from the same application. This distributed execution model allows Flink to process complex dataflows across multiple concurrent computing units
+* Dispatcher functions as a persistent service that spans across job executions, providing a crucial REST interface for application submissions. It initiates a JobManager for each submitted application and hands over the execution details. The Dispatcher also hosts a web dashboard that offers insights into job executions, making it a valuable monitoring and management component. Its ability to serve as an HTTP entry point is particularly useful for clusters situated behind firewalls, though its requirement can vary depending on the specific application deployment method.
+
+![image](https://github.com/user-attachments/assets/e95eec83-de17-4324-880d-ac8270c110f7)
+
+**Application deployment**
+
+* framework style: packeged in a jar file and submitted by a client to a running service
+* library style: bundled into a docker-style container image
+
+**Task execution**
+
+A TaskManager can execute several tasks at the same time. These tasks can be of the same operator (data parallelism), or from a different application (job parallelism). The TaskManager offers a certain number of processing slots to control the number of tasks it's able to concurrently execute. A processing slot can execute one slice of an app - one parallel task of each operator of the app.
+
+![image](https://github.com/user-attachments/assets/528114d5-459c-4bb8-9dcc-bf6000c0207c)
+
+A TaskManager executes its tasks multithreaded in the same JVM process. Threads are more lightweight than separate processes and have lower communication costs but do not strictly isolate tasks from each other. Hence, a single misbehaving task can kill a whole TaskManager process and all tasks that run on it.
+
+**Highly available setup**
+
+Streaming is supposed to run 24/7, so its important their execution doesn't stop even if some proecss fails. To recover from failures, the system first needs to restart failed processes, and second, restart the application and recover its state.
+
+* TaskManager failures: if in a  Flink setup with 4 TaskManagers each with two slots, 1 TM fails the JobManager will ask the ResourceManager to provide more processing slots. If this is not available, then the JobManager cannot restart the app until enough slots become available
+* JobManager failures: the JM controls  the execution of a streaming app and keeps metadata about its execution and a streaming app cannot continue processing if the responsible JM process disappears. Flink supports a high-availability mode that migrates the responsibility and metadata for a job to another JobManager in case the original JobManager disappears - this is Apache Zookeeper. When a JobManager fails, all tasks that belong to its application are automatically cancelled. A new JobManager that takes over the work of the failed master performs the following steps:
+
+1. it requests the storage locations from ZooKeeper to fetch the JobGraph, the jar file, and the state handles of the last checkpoint of the app from the remote storage
+2. it requests processing slots from the ResourceManager to continue executing the app
+3. it restarts the app and resets the state of all its tasks to the last complete checkpoint
+
+When running an application as a library deployment in a container environment, such as Kubernetes, failed JobManager or TaskManager containers are usually automatically restarted by the container orchestration service. 
+
+### Data transfer in Flink
+
+The TaskManagers take care of shipping data from sending tasks to receiving tasks. The network component of a TaskManager collects records in buffers before they are shipped -  records are not shipped one by one but batched into buffers.
+
+![image](https://github.com/user-attachments/assets/a7fd2867-664b-4ebe-acb4-75e863ce5764)
+
+Each TaskManager has a pool of network buffers (by default 32 KB in size) to send and receive data. In order to enable a smooth pipelined data exchange, a TaskManager must be able to provide enough buffers to serve all outgoing and incoming connections concurrently
+
+When a sender task and a receiver task run in the same TaskManager process, the sender task serializes the outgoing records into a byte buffer and puts the buffer into a queue once it is filled. The receiving task takes the buffer from the queue and deserializes the incoming records. Hence, data transfer between tasks that run on the same TaskManager does not cause network communication.
+
+### Event-time processing
+
+**Timestamps**
+
+When Flink processes a data stream in event-time mode, it evaluates time-based operators based on the timestamps of records. For example, a time-window operator assigns records to windows according to their associated timestamp
+
+**Watermarks**
+
+In addition to record timestamps, a Flink event-time application must also provide watermarks. Watermarks are used to derive the current event time at each task in an event-time application. Time-based operators use this time to trigger computations and make progress
+
+![image](https://github.com/user-attachments/assets/26ceac2c-24ee-4070-a916-d78116c91384)
+
+They have 2 basic properties:
+
+1. they must be monotonically increasing to ensure the event-time clocks of tasks are progressing and going forward
+2. they are related to record timestamps - a watermark with a timestamp T indicates that all subsequent records should have timestamps > T
+
+When a task receives a watermark, the following actions take place:
+
+1. the task updates its internal event-time clock based on the watermark’s timestamp
+2. the task’s time service identifies all timers with a time smaller than the updated event time. For each expired timer, the task invokes a callback function that can perform a computation and emit records
+3. the task emits a watermark with the updated event time
+
+### State management
+
+![image](https://github.com/user-attachments/assets/7b55a32d-21e3-4f26-b8e8-0d49860047e7)
+
+A task receives some input data. While processing the data, the task can read and update its state and compute its result based on its input data and state
+
+In Flink, state is always associated with a specific operator. In order to make Flink’s runtime aware of the state of an operator, the operator needs to register its state. There are two types of state, operator state and keyed state, that are accessible from different scopes
+
+**Operator state**
+
+Operator state is scoped to an operator task. This means that all records processed by the same parallel task have access to the same state. Operator state cannot be accessed by another task of the same or a different operator
+
+![image](https://github.com/user-attachments/assets/def76de5-0bd5-4668-ad46-40e01971e813)
+
+**Keyed state**
+
+Keyed state is maintained and accessed with respect to a key defined in the records of an operator’s input stream. Flink maintains one state instance per key value and partitions all records with the same key to the operator task that maintains the state for this key. When a task processes a record, it automatically scopes the state access to the key of the current record and all records with the same key access the same state.
+
+![image](https://github.com/user-attachments/assets/01a81f39-9083-4e0b-8a64-13ee5cab066c)
+
+### Checkpoints, Savepoints, and State recovery
+
+**Consistent checkpoints**
+
+A consistent checkpoint of a stateful streaming application is a copy of the state of each of its tasks at a point when all tasks have processed exactly the same input. This can be explained by looking at the steps of a naive algorithm that takes a consistent checkpoint of an application.
+
+![image](https://github.com/user-attachments/assets/9ad5f55d-3a5d-4865-91c2-d9eae486684b)
+
+The image shows an application that has a single source task that consumes a stream of increasing numbers—1, 2, 3, ~~. The stream of numbers is partitioned into a stream of even and odd numbers. Two tasks of a sum operator compute the running sums of all even and odd numbers. The source task stores the current offset of its input stream as state. The sum tasks persist the current sum value as state
+
+**Recovery from a consistant checkpoint**
+
+During the execution of a streaming application, Flink periodically takes consistent checkpoints of the application’s state. In case of a failure, Flink uses the latest checkpoint to consistently restore the application’s state and restarts the processing
+
+![image](https://github.com/user-attachments/assets/10815fca-274c-4166-97a1-bf3d79d736aa)
+
+An application is recovered in three steps:
+
+1. restart the whole application
+2. reset the states of all stateful tasks to the latest checkpoint
+3. resume the processing of all tasks
+
+Flink’s checkpointing algorithm produces consistent distributed checkpoints from streaming applications without stopping the whole application. However, it can increase the processing latency. Thankfully, there are tweaks that can alleviate the performance impact under certain conditions.
+
+**Savepoints**
+
+Savepoints are consistent snapshots of an application's state, created using the same algorithm as checkpoints but with additional metadata. Unlike checkpoints, savepoints are not automatically created or cleaned up; they must be explicitly triggered by the user or an external scheduler. While checkpoints primarily support failure recovery, savepoints enable additional use cases beyond recovery.
+
+---
+
+## Stream Processing with Apache Flink - Chapter 5 - The DataStream API
+
+This chapter was mainly to follow the examples in [Scala](https://github.com/streaming-with-flink/examples-scala)(or Java) and creating:
+
+* an input stream
+* basic transformations (map, filter, flatmap)
+* keyedStream transformations (keyBy, rolling aggs, reduce)
+* multistream transformations (union, connect, coMap, coFlatMap, split and select)
+
+In addition:
+
+* setting up parallelism
+* supported data types
+* defining keys and fields
+* defining custom functions
+
+After this I checked out the remaining few chapters and seem they are structured like a documentation of sort - going over the theoretical concepts covered in the previous chapters and showing the code for it - so I guess this is a very nice resource when starting to build your 1st Flink streaming job.
+
+---
+
+## [Stream](https://www.youtube.com/watch?v=_D0Ge869OUE)
+
+I found some Flink learn material on the [Flink docs](https://nightlies.apache.org/flink/flink-docs-master/docs/learn-flink/overview/). 
+
+Not my best stream 😆 maybe because I had a minor headache 😆
+
+## [Second stream on Airflow 101](https://youtu.be/MFEDL3vfVyA)
+
+I found that Astronomer has an online free academy series of courses on airflow and I got very excited 🥳 and after I rested after the 1st stream I decided to start the Apache Airflow 101 course and cover some of the 1st module. They are amazing and I already learned a bit about airflow's internals - web server, medatadabase, scheduler, executor, tasks, operators and how they communicate between each other. Excited to continue with the other courses on the [astronomer academy](https://academy.astronomer.io/path/airflow-101).
+
+---
+
+That's all for today!
+
+See you tomorrow :)

From 8ad81208c7d3a012d790da7a5fb5e02508751ec7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Dec 2024 22:50:14 +0900
Subject: [PATCH 531/638] Update default.html

---
 _layouts/default.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index ebe1a1ef3..9c871745b 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,6 +47,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
+          Learn + Blog everyday for 366 days! Math, Stats, Data, AI! Ends Jan 2nd, 2025
           {{ content }}
         </div>
       </div>

From 2ba91cca1099e804c5d880477f6912a30533c859 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Dec 2024 22:52:17 +0900
Subject: [PATCH 532/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 9c871745b..f0c4e547e 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,7 +47,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
-          Learn + Blog everyday for 366 days! Math, Stats, Data, AI! Ends Jan 2nd, 2025
+          <h2>Learn + Blog everyday for 366 days! Math, Stats, Data, AI! Ends Jan 2nd, 2025</h2>
           {{ content }}
         </div>
       </div>

From fb03256aaf65c9b19aabf66f82b3325a2abed7fe Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Dec 2024 22:54:12 +0900
Subject: [PATCH 533/638] Update default.html

---
 _layouts/default.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index f0c4e547e..8aaf033aa 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,7 +47,8 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
-          <h2>Learn + Blog everyday for 366 days! Math, Stats, Data, AI! Ends Jan 2nd, 2025</h2>
+          <h1>Learn + Blog everyday for 366 days! Math, Stats, Data, AI!</h1>
+          <h2>Ends Jan 2nd, 2025</h2>
           {{ content }}
         </div>
       </div>

From 5d2beb1c7bec75542577657d3efa306bad80d08c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Dec 2024 22:56:53 +0900
Subject: [PATCH 534/638] Update default.html

---
 _layouts/default.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 8aaf033aa..1e7e59fcc 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,7 +47,8 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
-          <h1>Learn + Blog everyday for 366 days! Math, Stats, Data, AI!</h1>
+          <h1>Learn + Blog everyday for 366 days!</h1>
+          <h1>Math, Stats, Data, AI!</h1>
           <h2>Ends Jan 2nd, 2025</h2>
           {{ content }}
         </div>

From 121d59ba4ef3119f551fac08bf89b98f495434d2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 12 Dec 2024 23:03:33 +0900
Subject: [PATCH 535/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 1e7e59fcc..29b01a050 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,7 +47,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
-          <h1>Learn + Blog everyday for 366 days!</h1>
+          <h1>🔥Learn + Blog EVERY DAY for 366 days!💻</h1>
           <h1>Math, Stats, Data, AI!</h1>
           <h2>Ends Jan 2nd, 2025</h2>
           {{ content }}

From 149880a11dd370b98fbed1d553e929ae0bc7d570 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 13 Dec 2024 21:53:51 +0900
Subject: [PATCH 536/638] Create 2024-12-13-day-347.md

---
 _posts/2024-12-13-day-347.md | 148 +++++++++++++++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100644 _posts/2024-12-13-day-347.md

diff --git a/_posts/2024-12-13-day-347.md b/_posts/2024-12-13-day-347.md
new file mode 100644
index 000000000..d5a0057bc
--- /dev/null
+++ b/_posts/2024-12-13-day-347.md
@@ -0,0 +1,148 @@
+---
+layout: post
+title: (Day 347) Some ML metrics + Finishing Airflow 101
+categories: [data-eng,theory,applying-knowledge,mlops,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 347!
+A quick summary of today:
+* checking out The Little Book of ML Metrics
+* stream for ~4.5hrs and finishing the Airflow 101 course
+
+## [The little book of ML metrics](https://github.com/NannyML/The-Little-Book-of-ML-Metrics)
+
+After watching [Vincent's latest video on the probabl channel](https://www.youtube.com/watch?v=3M2Gmuh5mtI) about Precision, Recall and the F1-score I reminded myself that this book of ML metrics is actively being written and I decided to finally check some of it out. At the moment about 90 pages are availabe but for maybe half or so of the metrics there are not descriptions/graphs. For those that are written, there are 1-2 pages for each metrics with nice graphics and simple explanations. Here are some of them. I reviewed some more on stream as for each metric the material is very condensed and already very well put 😆
+
+### Regression metrics
+
+**MAE**
+
+_MAE is one of the most popular regression accuracy metrics. It is calculated as the sum of absolute errors divided by the sample size. It is a scaledependent accuracy measure which means that it uses the same scale as the data being measured._
+
+When ti use MAE?
+
+Pros:
+
+* it provides and easy to understand value since it represents the avg error in the same units as the data
+* it treats under-predictions and over-predictions equally (though might not be desirable in all contexts)
+
+Cons:
+
+* can be biased when the distribution of errors is skewed as it does not account for the direction of the error
+* the absolute value function in its formula is not differentiable at 0 which can pose challenges in optimisation and gradient-based learning algs
+
+<img width="616" alt="image" src="https://github.com/user-attachments/assets/c159c96f-dc82-4d79-b23f-e4910c07b7e3" />
+
+_Did you know that: A forecast method that minimises MAE will lead to forecasts of the median_
+
+**MSE**
+
+Pros:
+
+* it is differentiable, making it useful in optimisation algs that require gradient information
+
+Cons:
+
+* due to squaring of each error term, it is particularly sensitive to outliers; large errors contribute more significantly to the MSE than smaller ones
+* can be hard to interpret as it squared the differences
+
+<img width="634" alt="image" src="https://github.com/user-attachments/assets/a3c52cbd-3a2e-47f7-9dfc-70046550811c" />
+
+_Did you know that: MSE can be written as the sum of the variance and bias of an estimator - meaning if we are dealing with an unbiased estimator, MSE will be equivalent to the variance_
+
+**RMSE**
+
+Pros:
+
+* easy to interpret
+* the squaring of errors ensure that larger errors have a greater impact, if large errors are undesirable
+
+Cons:
+
+* unlike some other metrics like R^2 and MAPE, RMSE does not provide a normalised measure, making it difficult to compare across different datasets without standardisation
+
+**MAPE**
+
+_MAPE is one of the most widely used measures of forecast accuracy due to its scale independence and interoperability. It is derived from the Mean Absolute Error (MAE), adjusted to provide a percentage measure, making it easier to interpret and compare across different datasets and contexts._
+
+_MAPE is useful when the value of the error on it’s own is not as important as its relative magnitude compared to the actual target values. It is not feasible when actual values can be 0. Additionally, MAPE is beneficial when it serves as a meaningful business metric._
+
+Pros:
+
+* scale-independent and easy to interpret
+
+Cons:
+
+* indefined when y_true = 0
+* penalises negative errors more than positive ones
+
+<img width="608" alt="image" src="https://github.com/user-attachments/assets/f1c614e5-30dc-4603-8b2b-20c2cb7692b8" />
+
+**Mean Absolute Scaled Error (MASE)**
+
+_MASE is a scale-independent metric that compares the absolute errors of a forecast model to the absolute errors of a naive forecast. This makes MASE particularly useful for comparing the performance of models across different datasets or when the scale of the target variable changes_
+
+_MASE values less than 1 indicate that the forecast model outperforms the naive forecast, while values greater than 1 indicate the opposite. MASE is unbounded on the positive side, with 0 representing a perfect forecast_
+
+Pros:
+
+* scale-independent, allowing for fair comparisons across datasets
+* provides a clear benchmark for model performance relative to a naive forecast
+* unlike %-based metrics like MAPE, MASE avoids division by 0 issues
+* penalises positive and negative forecast errors equally, and it treats errors in large and small forecasts with equal weight
+
+Cons:
+
+* required defining a suitable naive forecast method
+* may not be as intuitive to interpret as some other metrics like MAE/MAPE
+
+**Mean Directional Accuracy (MDA)**
+
+_MDA is a metric used to evaluate the performance of forecasting models in terms of their ability to correctly predict the direction of change in the target variable, rather than the magnitude of the change._
+
+It ranges from 0 to 1, 0 meaning the model always predicted the opposite direction and 1 indication the model correctl predicted the direction of change for all data points.
+
+Pros:
+
+* provides a simple, interpretable metric that is not affected by the scale of the target variable
+* does not provide any info about the magnitude of errors, only the direction
+
+**Pinball loss**
+
+_Pinball Loss is a metric specifically designed for evaluating quantile regression models. It measures the accuracy of predictions at a specified quantile, such as the median (q = 0.5), and generalizes the concept of absolute loss to asymmetric penalties. This makes it suitable for scenarios where the goal is to predict conditional quantiles of a target variable rather than the mean._
+
+<img width="500" alt="image" src="https://github.com/user-attachments/assets/d36c952c-fdea-4113-9cef-f84204c85a3a" />
+
+It ranges from 0 to +infinity, with 0 indicating a perfect quantile prediction. It asymmetrically penalizes over and under-predictions based on the specified quantile, making it unique among regression metrics.
+
+Pros:
+
+* can target any quantile by adjusting q enabling detailed analysis across the entire distribution
+* asymmetric loss penalties reflect practical priorities, such as minimising over-prediction costs
+
+Cons:
+
+* for reliable results, the model must be well-calibrated for the chosen quantile
+
+<img width="635" alt="image" src="https://github.com/user-attachments/assets/d854e9a2-315d-4d96-8c23-e141d30d804d" />
+
+_Did you know that: Pinball Loss is also called Quantile Loss and is an L1 Loss when q = 0.5._
+
+There are also some classification metrics, and I encourage you to check out the video probabl (link above) and also the book itself (also link above).
+
+## Stream
+
+I streamed twice today
+
+* [early afternoon](https://youtube.com/live/Wojl0mk7AmY) doing the Airflow 101 course
+* [evening](https://youtu.be/1F5tjxtfcjY) finishing the last 3 modules of the Airflow 101 course
+
+Amazing courses - with theory and practice. I am keen on learning more and later I am planning on using airflow as my orchestrator when the time to build a project for Zach Wilson's Jan boot camp comes ~ 
+
+I saw there are also other [airflow and astronomer](https://academy.astronomer.io/page/all-modules#product_airflow) courses which I might do on stream in the coming week ^^
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 81e61f268169b2e073753f2edcff0c4b7bc06fb3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 14 Dec 2024 22:26:55 +0900
Subject: [PATCH 537/638] Create 2024-12-14-day-348.md

---
 _posts/2024-12-14-day-348.md | 334 +++++++++++++++++++++++++++++++++++
 1 file changed, 334 insertions(+)
 create mode 100644 _posts/2024-12-14-day-348.md

diff --git a/_posts/2024-12-14-day-348.md b/_posts/2024-12-14-day-348.md
new file mode 100644
index 000000000..cb2714bfe
--- /dev/null
+++ b/_posts/2024-12-14-day-348.md
@@ -0,0 +1,334 @@
+---
+layout: post
+title: (Day 348) Apache Flink setup and debugging + starting the book Effective Machine Learning Teams 
+categories: [data-eng,theory,applying-knowledge,mlops,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 348!
+A quick summary of today:
+* doing setup for Zach Wilson's week 4 - streaming with kafka and flink
+* starting an amazing new book - Effective Machine Learning Teams
+
+## [Apache flink week setup for Zach Wilson's camp](https://github.com/DataExpert-io/data-engineer-handbook/tree/main/bootcamp/materials/4-apache-flink-training)
+
+Tomorrow when I wake up the lectures + labs will be out but today he released the setup. Thankfully I did it without any hurdles. 
+
+I even decided to look into an extra aggregation job that shows number of hits every page has every 5 mins on Zach's website. It was weird because the job submission was alright - no errors in the docker logs, neither in the flink UI. But I had no agg data in my table. After some playing around I believe the problem was in this function connecting to kafka:
+
+I narrowed it down to two changes in the `aggregation_job.py` file in the `create_processed_events_source_kafka` function:
+
+1. change the pattern to `pattern = "yyyy-MM-dd''T''HH:mm:ss.SSS''Z''"` 
+2. change the create table part of the `sink_ddl` to:
+
+```
+CREATE TABLE {table_name} (
+            url VARCHAR,
+            referrer VARCHAR,
+            user_agent VARCHAR,
+            host VARCHAR,
+            ip VARCHAR,
+            headers VARCHAR,
+            event_time VARCHAR,
+            window_timestamp AS TO_TIMESTAMP(event_time, '{pattern}'),
+            WATERMARK FOR window_timestamp AS window_timestamp - INTERVAL '15' SECOND
+        ) WITH (...)
+```
+
+For 2. it might be a problem with one of the cols, but I cannot see an errors anywhere. Nevertheless, this works 
+
+Here are the tables (I ran the job for a few mins only) I get:
+
+![image](https://github.com/user-attachments/assets/0c23d1db-1b9e-49f9-99a3-46105cb9c373)
+
+and 
+
+![image](https://github.com/user-attachments/assets/dc5a7696-4e24-4309-862a-bb88771f4a4f)
+
+
+## [Spark on Databricks](https://youtu.be/iXVIPQEGZ9Y)
+
+I casually watched this video on Spark on databricks ~ it was nice as for every part the teacher showed how to use the UI - see jobs descriptions, DAGs, info about jobs, partition sizes, etc.
+
+It covered:
+
+* Basics
+* Main components like driver and worker
+* Spark UI
+* Deployment modes
+* RDDs, DataFrames and DataSets
+* Transformations and actions
+* Job, Stages and Tasks
+* Data Shuffling
+* Introduction to optimization
+
+
+## Effective Machine Learning Teams - Chapter 1 - Challenges and Better Paths in Delivering ML Solutions
+
+### ML promises and disappointments
+
+Businesses are applying diverse ML techniques, such as natural language understanding and computer vision, across sectors like healthcare and finance. Demand for ML skills has grown consistently over the past decade, even during the pandemic. However, addressing past challenges in ML is crucial as the field progresses.
+
+### Why ML projects fail
+
+Failure can come in different forms:
+
+* inability to ship an ML-enabled product to production
+* shipping products that customers don’t use
+* deploying defective products that customers don’t trust
+* inability to evolve and improve models in production quickly enough
+
+
+|         Task                                         | **High-effectiveness environment**                     | **Low-effectiveness environment**                  |
+|--------------------------------------------------|--------------------------------------------------------------------------------------|--------------------------|
+| **Testing if code changes worked as expected**   | Automated testing (~ seconds to minutes)              | Manual testing (~ minutes to hours)               |
+| **Testing if ML training pipeline works end to end** | Training smoke test (~ 1 minute)                      | Full model training (~ minutes to hours, depending on the model architecture) |
+| **Getting feedback on code changes**            | Pair programming (~ seconds to minutes)              | Pull request reviews (~ hours to days)            |
+| **Understanding if application is working as expected in production** | Monitoring in production (~ seconds - as it happens)  | Customer complaints (~ days, or longer if not directly reported) |
+
+### Is There a Better Way? How Systems Thinking and Lean Can Help
+
+Relying solely on MLOps practices and platforms is insufficient for effective ML delivery. Similar to how DevOps optimizes software infrastructure but cannot address issues like software design or user experience, MLOps cannot compensate for gaps in software engineering (e.g. testing, design) or product delivery practices (e.g user stories, customer journeys). Success in ML requires a multidisciplinary approach involving product development, software engineering, data, ML, and delivery.
+
+### See the Whole: A Systems Thinking Lens for Effective ML Delivery
+
+_Systems thinking helps us shift our focus from individual parts of a system to relationships and interactions between all the components that constitute a system. Systems thinking gives us mental models and tools for understanding—and eventually changing—structures that are not serving us well, including our mental models and perceptions._
+
+A system consists of 3 things:
+
+* elements: such as data, ML experiments, software engineering, infrastructure and deployment, users and customers, and product design and user experience
+* interconnections: such as cross-functional collaboration and production ML systems creating data for subsequent labeling and retraining
+* a function or purpose of the ML product: such as helping users find the most suitable products
+
+![image](https://github.com/user-attachments/assets/b75edd46-c4c5-422a-96ca-db3e9b78e390)
+
+The components of ML product delivery are inherently interconnected
+
+Systems thinking recognizes that a system’s components are interconnected and that changes in one part of the system can have ripple effects throughout the rest of the system. This means that to truly understand and improve a system, we need to consider the system as a whole and how all its parts work together.
+
+### The Five Disciplines Required for Effective ML Delivery
+
+Lean helps ML teams minimize waste, maximize value, and deliver solutions more effectively by incorporating customer feedback and focusing on iterative improvements. Five essential disciplines—product, delivery, software engineering, data, and ML—are critical to effective ML delivery, with each discipline requiring specific principles and practices to enable rapid feedback and iteration.
+
+Lean principles, originating from Toyota's Production System, provide a structured approach to reduce inefficiencies and enhance outcomes.
+
+The 5 principles of Lean:
+
+![image](https://github.com/user-attachments/assets/4cacf449-e464-437e-8c3c-3cd6d5f88598)
+
+**The first discipline: Product**
+
+1. Discovery:  
+   - a structured process to clarify problems, explore opportunities, and develop solutions
+   - relies on tools like the Lean Canvas, Value Proposition Canvas, and Data Product Canvas to align stakeholders and prioritize customer needs
+   - incorporates customer feedback through techniques like user journey mapping and interviews to evaluate the feasibility and value of ML solutions
+   - discovery is continuous, helping teams adapt as they build, measure, and learn
+
+2. Prototype testing:  
+   - rapid, low-cost testing of ideas with users to validate assumptions before investing in engineering
+   - examples include hand-drawn interfaces, clickable mockups, or "Wizard of Oz" prototypes (manual simulations of automated functions)  
+   - reduces feedback loops from weeks or months to days, enabling fast iteration and early validation of ML product designs
+
+By focusing on customer-centric experimentation and fast feedback, the product discipline helps ML teams avoid building the wrong product and ensures resources are directed toward valuable, impactful solutions.
+
+**The second discipline: Delivery**
+
+1. Vertically sliced work:
+   - avoid horizontal slicing (e.g, separating data pipelines, ML models, and UX), as it delays feedback and risks integration issues
+   - opt for vertical slices—independently shippable units containing end-to-end functionality, providing earlier feedback and demonstrable value at all stages (story, iteration, release)
+
+2. Cross-functional teams:
+   - combine diverse expertise (e.g., data science, engineering, UX) within a team to reduce handoffs, waiting times, and blind spots
+   - enables smaller batch sizes, faster feedback, and better decision-making
+   - challenges like inconsistent solutions across teams can be mitigated via shared practices, platform teams, or communities of practice
+
+3. Ways of working (WoW):
+   - effective processes and tools, including agile ceremonies (e.g., standups, retros), story workflows, and quality assurance
+   - focus on the intent behind practices to improve alignment, collaboration, and flow of value, avoiding mechanical adherence to methods
+
+4. Measuring delivery metrics:
+   - regularly track metrics like iteration velocity, cycle time, and defect rates to monitor delivery health
+   - use the 'four key metrics' (lead time, deployment frequency, MTTR, and change failure rate) to assess software delivery performance, which correlates with business outcomes
+   - metrics foster data-driven planning, continuous improvement, and alignment with delivery goals
+
+Delivery ensures timely, reliable execution by prioritizing vertical work slices, fostering cross-functional collaboration, and adopting metrics-driven processes. These practices help ML teams minimize risks, improve flow, and achieve better outcomes for customers and organizations.
+
+**The third discipline: Engineering**
+
+1. Automated testing
+2. Refactoring
+3. Code editor effectiveness
+4. Continuous delivery for ML
+
+![image](https://github.com/user-attachments/assets/724b1b9d-5241-41db-bfed-f9ae069ba37f)
+
+**The fourth discipline: ML**
+
+1. Framing ML problems
+
+![image](https://github.com/user-attachments/assets/edad9e28-70b3-4245-913e-3f0757883d3a)
+
+2. ML systems design
+3. Responsible AI and ML governance
+
+**The fifth discipline: Data**
+
+1. Closing the data collection loop: effective ML systems must integrate mechanisms to collect and curate predictions in production, enabling the creation of high-quality ground truth for evaluation and retraining. Scaling the labeling process through active learning, self-supervised learning, or weak supervision can address bottlenecks. Where natural labels exist, data pipelines should stream them alongside corresponding features while mitigating risks such as data poisoning and runaway feedback loops, which can entrench bias in subsequent models. Neglecting this process limits opportunities for data-centric model improvements, hampering long-term performance
+
+2. Data security and privacy: safeguarding data requires a multilayered defense-in-depth approach, including encryption, access controls, and adherence to the principle of least privilege. Clear organizational policies on data governance ensure ethical usage and compliance with regulations, while access is restricted to authorized individuals and systems. These measures protect data integrity, privacy, and the ethical standards necessary for sustainable ML practices.
+
+(wow ~ this 1st chapter is so insightful and definitely something to look back at when thinking about projects)
+
+## Effective Machine Learning Teams - Chapter 2 - Product and Delivery Practices for ML Teams
+
+Concepts covered this chapter:
+
+![image](https://github.com/user-attachments/assets/fcf565a5-175e-434a-90fe-e1d9c31a9541)
+
+Based on the British Double Diamond Design process.
+
+### ML product discovery
+
+Effective ML product discovery involves continuous customer engagement and adopting structured approaches to minimize risks and maximize value. Common pitfalls such as underused ML solutions, over-engineered products, endless PoCs, or weak business cases stem from insufficient discovery efforts.
+
+**Discovering product opportunities**
+
+During this phase the customers are at the center. Some notable techniques for surfacing the voice of the customer include personas, customer journey mapping, contextual inquiry, and customer interviews. These are common tools in the field of experience design, and each offers unique insights about users and customers.
+
+**Canvases to define product opportunities**
+
+The value proposition canvas:
+
+![image](https://github.com/user-attachments/assets/bf376065-7d71-4925-b2a2-37e1055d3908)
+
+_Data product canvas_
+
+For ML product discovery, it’s important to assess the value and feasibility of candidate solutions. One helpful tool in this regard is the data product canvas, which provides a framework for connecting the dots between data, ML, and value creation. 
+
+![image](https://github.com/user-attachments/assets/b0128147-040a-4a51-8ef3-b2905e5eebe5)
+
+_Hypothesis canvas_
+
+The Hypothesis Canvas helps us reduce uncertainty as we formulate testable hypotheses, identify objective metrics, and design lightweight experiments to rapidly validate or invalidate ideas. 
+
+![image](https://github.com/user-attachments/assets/ca643173-e92d-4d2f-ab5c-5df47b0008d8)
+
+**Techniques for rapidly designing, delivering, and testing solutions**
+
+* MVPs (or Minimum Loveable Product, Earliest Testable/​Usa⁠ble/Lovable Product)
+* prototypes (on paper, interactive digital, proof of concepts)
+
+**Riskiest assumption test**
+
+* list all large assumptions and identify the riskiest one
+* define a test for the riskiest assumption
+* conduct the test
+
+**Product and Design thinking**
+
+_Design Thinking is a problem-solving methodology widely used in various industries to address complex challenges through a user-centered approach. It involves understanding the needs and experiences of users, redefining problems, and creating innovative solutions. This is an alternative framework to the Double Diamond.Design Thinking is iterative, meaning that it often involves going back and forth between these stages, refining and adjusting as more is learned about the user and the problem. It’s highly collaborative and often involves cross-functional teams to bring different perspectives and expertise to the problem-solving process. This approach is used not only in product design but also in service design, business strategy, and organizational problem-solving._
+
+![image](https://github.com/user-attachments/assets/d4284004-aeae-42ad-9605-e3ca2817a77d)
+
+_Product Thinking builds on Design Thinking by including product management practices to focus on the business or organizational viability of any product. This means developing product design through the product lifecycle from initial intensive investment to a sustainable offering characterized by incremental innovation, and eventually to retirement as market fit wanes. It also means thinking about product commercial models and coordinating people and resources within organizations to deliver the product. Product Thinking is also sometimes contrasted with a solution mindset, which might be said to deliver an ad hoc collection of point solutions, rather than a unified offering that coherently solves customer problems with a good user experience._
+
+
+
+By the end of the Discovery phase, the team and business should align on a product idea (or ideas) that is desirable (meeting customer needs), viable (profitable), and feasible (achievable within available resources and constraints). The insights and evidence gathered during Discovery can support creating a strong business case and securing funding for the product opportunity, if needed.
+
+At this point, teams transition to the next stage: Inception. This phase focuses on defining actionable plans for product scope, technology, delivery planning, and risk management to ensure the team stays on track during delivery. This will be our next area of focus.
+
+### Inception: Setting Teams Up for Success
+
+Inception is the phase where the team and stakeholders align on the product’s vision, scope, objectives, and delivery plan. Its goal is to ensure a shared understanding, providing a foundation to handle changes during delivery. Inception involves activities that help shape, size, and sequence the work needed, typically lasting from a few days to four weeks. It immediately follows Discovery and precedes delivery, ensuring clarity and alignment. 
+
+While Discovery focuses on identifying problems and potential solutions, Inception elaborates on the solution, covering technical aspects and delivery plans. The context from Discovery should carry into Inception and Delivery to maintain alignment and ensure buy-in. 
+
+In cases where phases are separated for budget or funding reasons, Discovery findings should be documented carefully to support a go/no-go decision. If the decision is 'go', Inception and Delivery proceed together. Inception may also focus on achievable delivery scope based on available resources. In rare cases, if the problem is clear and simple, product delivery might start directly at Inception, though skipping Discovery can increase risks.
+
+**How to plan and run an inception**
+
+Designing an Inception agenda depends on how much existing knowledge and the context the team has around the solution to be built.
+
+A minimal inception agenda includes:
+
+* alignment on priorities (e.g vision setting, trade-off sliders)
+* defining the MVP
+* cross-functional requirements
+* solution design and minimum viable architecture
+* path to production
+* risks, assumptions, issues, and dependencies (RAIDs)
+* security threat modeling
+* ethics and responsible technology
+* release planning
+* ways of Working
+
+**User stories: building blocks of an MVP**
+
+A user story is the smallest unit of value that can be independently shipped to production. A user story is the quantum, or building block, of the functionality of the product. Under agile methodologies, it’s also the quantum or building block of the project scope, so that product and project progress are linked. User stories are the vehicle for translating ideas, intents, and context into clear, business-validated acceptance criteria that guide team members—be they developers, data scientists, ML engineers, quality analysts, etc — to know what to build, and how to know when the feature is done.
+
+User stories should be:
+
+* independent: each story should stand alone, independent of other stories. This allows them to be developed, prioritized, and implemented in any order and in parallel
+* negotiable: rather than being a contract, a story is a conversation starter. Details can be discussed and adjusted based on collaboration between the delivery team and the product owner
+* valuable: every story should deliver value to the end users or the business. If it doesn’t, it might not be worth including in the product backlog
+* estimable: the team should be able to estimate the effort required for a story. If a story can’t be estimated because it’s too vague, it’s a hook for the team to clarify the details in the story or decompose the problem further
+* small: stories should be small enough to be completed within one iteration. If they’re too large, they should be split into smaller, more manageable pieces
+* testable: there should be clear criteria to determine when a story is done and working as expected. Without this, it’s hard to know when the work is truly complete
+
+### Product delivery
+
+**Delivery activities**
+
+* iteration planning
+* daily standups
+* story delivery
+* regular showcases
+* retrospectives
+* ongoing risk management
+* architectural decisions
+* continuous discovery
+* dual-track delivery
+
+Diagram of the dual track delivery model:
+
+![image](https://github.com/user-attachments/assets/36ed6497-ada1-4845-8d10-133e193c0109)
+
+**Measuring product and delivery**
+
+Delivery measures:
+
+- track metrics like scope completed per iteration, cycle time, and defect rates to monitor delivery health and adherence to timelines
+- tools like burn-up charts visualize total scope and work completed, enabling early detection of issues like misaligned velocity or scope management
+- velocity (historical and forecasted) helps in iteration planning and stakeholder communication, but it is team-specific and unsuitable for cross-team comparisons
+- focus on calibrating delivery plans using insights from metrics, aligning with the Agile principle of responding to change
+
+Product measures:
+
+- frameworks like pirate metrics (AARRR) and One Metric That Matters (OMTM) guide teams in tracking user engagement, financial performance, and product-specific outcomes
+- metrics should evolve with the product’s maturity and focus on shorter feedback loops to refine priorities
+
+Model measures:
+
+- ML model performance often uses proxies for product goals, like accuracy or F-score for classifiers
+- balancing competing needs (e.g. fraud detection vs. user acquisition) is critical, requiring agreed-upon thresholds and continuous improvement
+- model metrics reflect progress and guide adjustments, blending discovery (experiments) and delivery (implementation)
+
+Discovery measures:
+
+- discovery work is inherently uncertain and focuses on maintaining a portfolio of meaningful, time-bound experiments
+- minimize cycle time for experimentation to accelerate progress and avoid unproductive R&D efforts
+- regularly review experiment outcomes to integrate discoveries into delivery processes
+
+### Conclusion
+
+* instead of finding out in two weeks that a developer or ML practitioner’s solution did not align with business expectations, spend a few hours writing and validating some user stories
+
+* instead of finding out in three months that your team is going to miss a critical release milestone, spend an hour each week measuring the team’s velocity
+
+* instead of finding out in six months that an entire release didn’t improve customer experience, find out critical success factors in two weeks through discovery
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 2260bd98e07ab80e12f7a7d58989496888576230 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 14 Dec 2024 22:42:28 +0900
Subject: [PATCH 538/638] Update default.html

---
 _layouts/default.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index 29b01a050..91fc6687c 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -50,6 +50,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
           <h1>🔥Learn + Blog EVERY DAY for 366 days!💻</h1>
           <h1>Math, Stats, Data, AI!</h1>
           <h2>Ends Jan 2nd, 2025</h2>
+          <br>
           {{ content }}
         </div>
       </div>

From 9a08b83ec295f4682cb12f3bff80216c23c0246b Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 14 Dec 2024 22:44:01 +0900
Subject: [PATCH 539/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 91fc6687c..76af44206 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -50,7 +50,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
           <h1>🔥Learn + Blog EVERY DAY for 366 days!💻</h1>
           <h1>Math, Stats, Data, AI!</h1>
           <h2>Ends Jan 2nd, 2025</h2>
-          <br>
+          <hr>
           {{ content }}
         </div>
       </div>

From 894c3255ec20d7f71996f5f236387e50d5a696f5 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 15 Dec 2024 22:48:11 +0900
Subject: [PATCH 540/638] Create 2024-12-15-day-349.md

---
 _posts/2024-12-15-day-349.md | 718 +++++++++++++++++++++++++++++++++++
 1 file changed, 718 insertions(+)
 create mode 100644 _posts/2024-12-15-day-349.md

diff --git a/_posts/2024-12-15-day-349.md b/_posts/2024-12-15-day-349.md
new file mode 100644
index 000000000..9ecaf6f2c
--- /dev/null
+++ b/_posts/2024-12-15-day-349.md
@@ -0,0 +1,718 @@
+---
+layout: post
+title: (Day 349) Finished the Effective ML teams book
+categories: [mlops,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 349!
+A quick summary of today:
+* Chapter 3 - Effective Dependency Management: Principles and Tools
+* Chapter 4 - Effective Dependency Management in Practice
+* Chapter 5 - Automated Testing: Move Fast Without Breaking Things
+* Chapter 6 - Automated Testing: ML Model Tests
+* Chapter 7 - Supercharging Your Code Editor with Simple Techniques
+* Chapter 8 - Refactoring and Technical Debt Management
+* Chapter 9 - MLOps and Continuous Delivery for ML (CD4ML)
+* Chapter 10/11 - Building Blocks of Effective ML Teams/Organisations
+
+---
+
+## Effective Machine Learning Teams - Chapter 3 - Effective Dependency Management: Principles and Tools
+
+### What if our code could run everywhere, every time?
+
+![image](https://github.com/user-attachments/assets/386a675f-1879-4d8d-a036-9f357a47bbc3)
+
+**A Better Way: Check Out and Go**
+
+Joining a team, running a cmd like `./go.sh` and we are done - this is known as 'check out and go'. By adopting this practice, we get the following benefits:
+
+* faster onboarding
+* saving time and cognitive resources
+* experiment-operation symmetry
+* reproducability and repeatability
+
+**Principles for Effective Dependency Management**
+
+![image](https://github.com/user-attachments/assets/0513cdd7-e104-4b7a-ad47-e5163851adee)
+
+With good dependency management practices, we can create consistent, production-like, and reproducible environments from the get-go
+
+**Tools for Dependency Management**
+
+In a nutshell, to effectively manage the dependencies of a given project, we need to:
+
+* specify OS-level dependencies as code (by using Docker)
+* specify application-level dependencies as code (by using Poetry)
+
+### Docker crash course
+
+This 2nd part of the chapter went a bit more in-depth on the Docker basics. And also a library called batect which seems to have been archived in 2023 and no longer maintained
+
+![image](https://github.com/user-attachments/assets/74176a47-43c6-4ade-a578-9596a166a4e0)
+
+---
+
+## Effective Machine Learning Teams - Chapter 4 - Effective Dependency Management in Practice
+
+### In Context: ML Development Workflow
+
+A good starting point would be:
+
+* a dev image
+* prod API image
+* deployment image - model training
+* deployment image — model web service
+
+Here are common ML development tasks and their associated images, however the specific slicing and differentiation of images will vary depending on the project’s needs
+
+![image](https://github.com/user-attachments/assets/19397581-6f92-4f9b-8f4c-38f2f970d98d)
+
+### Secure Dependency Management
+
+**Remove unnecessary dependencies**
+
+Unnecessary deps:
+
+* enlarge the attack surface area of your project and make it more vulnerable to malicious attackers
+* increase the time needed to build, publish, and pull your images
+* stray dependencies that are installed but never used can make the project confusing and hard to maintain
+
+As a rule of thumb, we should:
+
+* start with base images that are as small as possible
+* remove dependencies that are not used from pyproject.toml
+* exclude development dependencies from the production image
+* automate checks for vulnerabilities
+
+---
+
+## Effective Machine Learning Teams - Chapter 5 - Automated Testing: Move Fast Without Breaking Things
+
+### Automated Tests: The Foundation for Iterating Quickly and Reliably
+
+Automated tests are the essential foundation for products that are easy to maintain and evolve. Without automated tests, changes become error-prone, tedious, and stressful.
+
+![image](https://github.com/user-attachments/assets/fb48843c-4155-4469-a5da-ed91f808a2e1)
+
+Key benefits:
+
+* fast feedback
+* reduced risk of production defects
+* living documentation and self-testing code
+* reduced cognitive load
+* refactoring
+* regulatory compliance
+* improved flow, productivity, and satisfaction
+
+**If Automated Testing Is So Important, Why Aren’t We Doing It?**
+
+* we think writing automated tests slows us down
+* we have CI/CD (but the content matters)
+* we just don’t know how to test ML systems
+
+### Building Blocks for a Comprehensive Test Strategy for ML Systems
+
+**Identifying Components For Testing**
+
+<img width="1026" alt="image" src="https://github.com/user-attachments/assets/6d8fad9a-0844-41d4-a109-90dd2bdbfca0" />
+
+**The ML Systems Test Pyramid**
+
+![image](https://github.com/user-attachments/assets/61229b54-9d8e-4fb2-914b-92be9c6e88c7)
+
+**Characteristics of a Good Test and Pitfalls to Avoid**
+
+* tests should be independent and idempotent
+* tests should fail fast and fail loudly
+* tests should check behavior, not implementation
+
+```python
+# bad example: implementation-focused test
+def test_train_model_returns_a_trained_model(mock):
+   mock.patch("train.training_helpers._fit_model", 
+              return_value=RandomForestClassifier())
+
+   model = train_model()
+
+   assert isinstance(model, RandomForestClassifier)
+
+# good example: behavior-focused test
+def test_train_model_returns_a_trained_model():
+   test_data = load_test_data()
+   model = train_model(test_data)
+   valid_predictions = [0, 1, 2]
+
+   prediction = model.predict(test_data[0])
+
+   assert prediction in valid_predictions
+```
+
+* tests should be runnable in your development environment
+* tests must be part of feature development
+* tests let us 'catch bugs once'
+
+**Structure of a Test**
+
+* a meaningful test name
+* structure of arrange, act, assert (AAA)
+* specific and holistic assertions
+
+```python
+# good example
+def test_convert_keys_to_snake_case_for_keys_with_spaces_or_punctuation(): 
+   # arrange (provide input conditions) 
+   user_details = {"Job Description": "Wizard",
+                   "Work_Address": "Hogwarts Castle",
+  "Current-title": "Headmaster"}
+
+   # act
+   result = convert_keys_to_snake_case(user_details)
+
+   # assert (verify output conditions)
+   assert result == {
+       "job_description": "Wizard",
+       "work_address": "Hogwarts Castle",
+       "current_title": "Headmaster"
+    }
+
+# bad example
+def test_convert_keys_to_snake_case(): 
+    result = convert_keys_to_snake_case({"Job Description": "Wizard",
+                                        "Work Address": "Hogwarts Castle",
+                                        "Current_title": "Headmaster"
+})
+    assert result["job_description"] == "Wizard" 
+    assert result["work_address"] is not None 
+```
+
+### Software Tests
+
+* unit tests
+
+```python
+# unit testing a function that transforms a dataframe
+from pandas._testing import assert_frame_equal 
+
+def test_normalize_columns_returns_a_dataframe_with_values_between_0_and_1():
+   loan_applications = pd.DataFrame({"income": [10, 100, 10000]})
+
+   result = normalize_columns(loan_applications)
+
+   expected = pd.DataFrame({"income": [0, 0.009, 1]})
+   assert_frame_equal(expected, result)
+```
+
+* training smoke tests
+
+```python
+def test_training_smoke_test():
+    data = pd.read_csv("/code/data/train.csv", encoding="utf-8", 
+                       low_memory=False)
+    # group by target column to ensure we train on data 
+    # that contains all target labels:
+	test_data = data.groupby('DEFAULT') \
+	                            .apply(lambda df: df.head(10)) \
+	                            .reset_index(drop=True)
+
+    pipeline = train_model(test_data)
+
+    predictions = pipeline.predict(test_data)
+    valid_predictions = {0, 1}
+    assert valid_predictions.issubset(predictions)
+```
+
+* API tests
+
+```python
+from fastapi.testclient import TestClient
+from precisely import assert_that, is_mapping, \
+   greater_than_or_equal_to, less_than_or_equal_to
+
+from api.app import app
+
+
+client = TestClient(app)
+
+
+def test_root(self):
+   response = client.get("/")
+
+   assert response.status_code == 200
+   assert response.json() == {"message": "hello world"} 
+
+def test_predict_should_return_a_prediction_when_given_a_valid_payload(self): 
+
+   valid_request_payload = {
+       "Changed_Credit_Limit": 0,
+       "Annual_Income": 0,
+       "Monthly_Inhand_Salary": 0,
+       "Age": 0,
+   } 
+
+   response = client.post("/predict/",
+                          headers={"Content-Type": "application/json"},
+                          json=valid_request_payload) 
+
+   assert response.status_code == 200
+   assert_that(response.json(),
+               is_mapping({"prediction": greater_than_or_equal_to(0)
+                                              and less_than_or_equal_to(4),
+                         "request": valid_request_payload}) 
+```
+
+It is a recommended practice to assert on the whole 'elephant' (final object), rather than assertions for each item in it
+
+* post-deployment tests
+
+```python
+import requests
+
+
+class TestPostDeployment:
+   endpoint_url = "https://my-model-api.example.com" 
+
+   def test_root(self): 
+       response = requests.get(self.endpoint_url)
+
+       assert response.status_code == 200
+
+   def test_predict_should_return_a_prediction_when_given_a_valid_payload(self):
+       valid_request_payload = {
+           "Changed_Credit_Limit": 0,
+           "Annual_Income": 0,
+           "Monthly_Inhand_Salary": 0,
+           "Age": 0,
+        }
+
+       response = requests.post(f"{self.endpoint_url}/predict/", 
+                                headers={"Content-Type": "application/json"},
+                                json=valid_request_payload
+                                ) 
+
+       assert response.status_code == 200
+```
+
+---
+
+## Effective Machine Learning Teams - Chapter 6 - Automated Testing: ML Model Tests
+
+### Model Tests
+
+**The Necessity of Model Tests**
+
+Model tests are essential for ensuring both speed and quality in the ML delivery cycle. Without them, teams face a false choice between maintaining production speed and ensuring model quality, often leading to skipped checks and defects detected too late. Automated model tests address silent failures, reduce manual testing, and prevent production issues, allowing ML practitioners to focus on higher-value tasks. Comprehensive testing builds confidence in model performance and enhances efficiency, ultimately saving time and improving outcomes for users.
+
+**Challenges of Testing ML Models**
+
+Four challenges—slow tests, high-volume and high-dimensional data, the need for visual exploration, and unclear definitions of 'good enough'—are challenges that ML practitioners tend to work through and live with.
+
+![image](https://github.com/user-attachments/assets/d83c1166-b68c-4267-8659-8a8ee392b9f4)
+
+**Fitness Functions for ML Models**
+
+Fitness functions are objective, executable functions that can be used to summarize, as a single figure of merit, how close a given design solution is to achieving its set aims. Fitness functions bridge the gap between automated tests (precise) and ML models (fuzzy).
+
+For example, we can define fitness functions that measure the quality or toxicity of our code. If the code gets too convoluted or violates certain code quality rules, the 'code quality' fitness function fails and gives us feedback that a code change has degraded our system beyond our specified tolerance level.
+
+![image](https://github.com/user-attachments/assets/38aa7716-efc3-4533-89ad-15f9133cd39c)
+
+Example fitness functions for an ML model
+
+* metrics tests: the model is fit for production if a given evaluation metric measured using a holdout set is above a specified threshold
+* model fairness tests: the model is fit for production if a given evaluation metric for each key segment—e.g., country—is within X% of each other
+* model API latency tests: the model is fit for production if it can handle N amount of concurrent requests within t seconds
+* model size tests: the model artifact must be below a certain size (e.g., so that we can deploy it easily to embedded devices or mobile devices)
+* training duration tests
+
+**Model Metrics Tests (Global and Stratified)**
+
+In these tests, we calculate model evaluation metrics that measure the aggregate correctness of the model on an extensible validation dataset and test if they meet our expected threshold of what is considered good enough for the model to be released to production. We compute these metrics at the global level and also at the level of important segments of our data (i.e., stratified level).
+
+Example test:
+
+```python
+class TestMetrics:
+   recall_threshold = 0.65
+
+   def test_global_recall_score_should_be_above_specified_threshold(self):
+       # load trained model
+       pipeline = load_model()
+
+       # load test data 
+       data = pd.read_csv("./data/train.csv", encoding="utf-8", low_memory=False)
+       y = data["DEFAULT"]
+       X = data.drop("DEFAULT", axis=1)
+       X_test, X_train, y_test, y_train = train_test_split(X, y, random_state=10)
+
+       # get predictions
+       y_pred = pipeline.predict(X_test)
+
+       # calculate metric
+       recall = recall_score(y_test, y_pred, average="weighted")
+
+       # assert on metric
+       print(f"global recall score: {recall}") 
+       assert recall >= self.recall_threshold
+```
+
+Example of a stratified test:
+
+```python
+class TestMetrics:
+   recall_threshold = 0.65
+
+   def test_stratified_recall_score_should_be_above_specified_threshold(self):
+       pipeline = load_model()
+
+       data = pd.read_csv("./data/train.csv", encoding="utf-8", low_memory=False)
+       strata_col_name = "OCCUPATION_TYPE" 
+       stratas = data["OCCUPATION_TYPE"].unique().tolist()
+       y = data["DEFAULT"]
+       X = data.drop("DEFAULT", axis=1)
+       X_test, X_train, y_test, y_train = train_test_split(X, y, random_state=10)
+
+       # get predictions and metric for each strata 
+       recall_scores = []
+       for strata in stratas:
+           X_test_stratified = X_test[X_test[strata_col_name] == strata]
+           y_test_stratified = y_test[y_test.index.isin(X_test_stratified.index)]
+           y_pred_stratified = pipeline.predict(X_test_stratified)
+
+           # calculate metric
+           recall_for_single_strata = recall_score(y_test_stratified, 
+                                                   y_pred_stratified, 
+                                                   average="weighted")
+           print(f"{strata}: recall score: {recall_for_single_strata}")
+
+           recall_scores.append(recall_for_single_strata)
+
+       assert all(recall > self.recall_threshold for recall in recall_scores)
+```
+
+**Behavioral Tests**
+
+Behavioral tests complement metrics tests by allowing us to enumerate specific—and potentially out-of-sample—scenarios under which to test our model. Behavioral testing (also known as black-box testing) has its roots in software engineering and is concerned with testing various capabilities of a system by 'validating the input-output behavior, without any knowledge of the internal structure'.
+
+General approach for defining behavioral tests in ML:
+
+1. define or generate test samples. You can start with just one or two examples and use data-generation techniques to scale to more examples, if that provides value
+2. use these test samples to generate predictions from a trained model
+3. verify that the model’s behavior matches your expectations
+
+Example tests:
+
+* invariance tests where we apply label-preserving perturbations to the input data and expect the model’s prediction to remain the same
+* directional expectation tests that are like invariance tests, except that we expect the prediction to change in a specific direction when we perturb the input data
+* minimum functionality tests contains a set of simple examples and corresponding labels that are used to verify the model’s behavior in a particular scenario. These tests are similar to unit tests in software engineering and are useful for creating small, targeted testing datasets.
+
+**Testing LLMs: Why and How**
+
+* manual exploratory tests
+* example-based tests
+* benchmark tests
+* property-based tests
+* LLM-as-a-judge-tests
+
+### Essential Complementary Practices for Model Tests
+
+**Error Analysis and Visualization**
+
+![image](https://github.com/user-attachments/assets/73a2231d-8c28-499f-a267-ef7cd221fcc0)
+
+**Learn from Production by Closing the Data Collection Loop**
+
+* minimize training-serving skew
+* use production-like data for testing (and use synthetic data where necessary)
+* close the data collection loop
+
+_When closing data collection loops, we need to identify and mitigate risks of runaway feedback loops, which is the phenomenon where the model learns biases and, through the effect that it has on the real world, perpetuates the bias in the real world and the data that it will be trained on in the future, thereby creating a vicious cycle._
+
+![image](https://github.com/user-attachments/assets/1141889b-9404-4e59-9923-7364d8bc3800)
+
+The rate at which we can improve our models depends on the rate of information flow in the data collection loop.
+
+**Open-Closed Test Design**
+
+The open-closed design principle states that software entities (classes, modules, functions, and so on) should be open for extension, but closed for modification. This is a simple but powerful design principle that helps us write extensible code and minimize the amount of bespoke customization that we need to add for each new piece of functionality. By exposing the test data source and model as a configurable parameter in our test, we can rerun model evaluation tests on any given model, against any given dataset, at any time
+
+![image](https://github.com/user-attachments/assets/7d02aa98-7ccd-4a78-985d-a00763f3df31)
+
+**Exploratory Testing**
+
+When you are stuck in writing automated model tests—which can often be the case, especially in early phases of an ML project—exploratory testing can help you discover what good looks like. When you are not sure what to test in an exploratory test, feedback or complaints from customers and domain specialists are a valuable starting point. While you may have a reflexive aversion to customer complaints, they are nevertheless valuable signals and they also point to a gap in our ML delivery process that needs to be looked into. When an exploratory test shows signs of being repeated or repeatable, you can formulate it as an automated test and reap the benefits of quality, flow, cognitive load, and satisfaction.
+
+**Means to Improve the Model**
+
+_Tests do not improve a product; the improvement is done when you fix the bugs uncovered by the tests—when a model test fails and you’ve undertaken the error analysis to understand how a model arrived at an incorrect outcome, and finally you identify potential options for improving the model._
+
+We can approach model improvement from two angles
+
+* data-centric approach: we can leverage the data collection loop from earlier to create training data that is more representative and of a better quality. We can also consider various feature engineering approaches, such as creating balanced datasets or feature scaling
+* model-centric approach: no, this is not just tuning hyperparameters. We can explore alternative model architectures, ensembles, or even decompose the ML problem into narrower subproblems that are easier to solve
+
+There are times where teams may try both approaches.
+
+**Designing to Minimize the Cost of Failures**
+
+Given that ML models are bound to make wrong predictions some of the time, you need to design the product in a way that reduces risk (risk = likelihood x impact) of a wrong prediction, especially when the stakes are high. Not all errors are created equal, and some mistakes are more costly than others. Once we know the cost, we can apply some cost-sensitive learning techniques:
+
+* addressing data imbalances
+* highlighting costly mistakes
+* evaluating and training against cost-penalization metrics
+* inference-time cost consideration
+
+**Monitoring in Production**
+
+* application monitoring
+* data monitoring
+* model metrics monitoring
+
+**Bringing It All Together**
+
+![image](https://github.com/user-attachments/assets/fbe1bd59-92a7-4bfc-8c35-aaca467b0fdf)
+
+---
+
+## Effective Machine Learning Teams - Chapter 7 - Supercharging Your Code Editor with Simple Techniques
+
+This chapter was just commands and how to setup PyCharm/VScode 😆 ~ nothing extraordinary, but I guess it's good for someone that has not used an IDE before
+
+---
+
+## Effective Machine Learning Teams - Chapter 8 - Refactoring and Technical Debt Management
+
+### Technical Debt: The Sand in Our Gears
+
+Refactoring requires automated tests as a safety harness and benefits greatly from good software design to manage complexities in ML systems. A high-level understanding of components and their collaboration helps create maintainable, readable solutions, avoiding architectural issues like tight coupling. The C4 model can aid in visualizing software architecture.  
+
+Key principles for software design include abstraction, which simplifies complex details into clear interfaces, and maintaining loosely coupled components to enhance flexibility and maintainability. Readable and maintainable code accelerates iteration and experimentation.  
+
+**Refactoring guidelines**:
+
+1. refactoring definition: restructure code without altering behavior using small, incremental changes, keeping the system functional at all times
+2. Two Hats Rule: separate refactoring from adding new functionality to reduce cognitive load and maintain stability
+3. Scout Rule: leave the codebase cleaner than you found it, addressing minor issues or flagging larger ones for future fixes
+4. leverage IDEs: use tools for efficient tasks like renaming variables or extracting functions
+5. avoid premature abstraction: Favor simplicity initially and refine design later to avoid sunk-cost fallacies and overly complex systems
+
+The ultimate goal is a codebase that is organized, functional, and adaptable, avoiding the chaos of poorly designed 'big ball of mud' systems.
+
+### How to Refactor a Notebook (or a Problematic Codebase)
+
+The refactoring cycle:
+
+![image](https://github.com/user-attachments/assets/c47bc100-107b-45c0-b273-12a72b7b48f1)
+
+There was an exercise to refactor a notebook and turn it into an executable python file - [here](https://github.com/davified/refactoring-exercise/blob/main/notebooks/titanic-notebook-refactoring-starter.ipynb)
+
+### Technical Debt Management in the Real World
+
+**Technical Debt Management Techniques**
+
+* make debt visible
+
+![image](https://github.com/user-attachments/assets/2f6e085d-a9fb-4a71-b55f-869df7708e7b)
+
+A technical debt wall helps to make debt visible and helps the team pay down the most important debt as part of ongoing delivery
+
+* the 80/20 rule
+
+For each story card, focus 80% of the time on delivering features and completing the story, and spend 20% of the time paying off some technical debt, so that the team can continue to deliver features at a sustainable and predictable pace.
+
+* make it cheap and safe
+* demonstrate value of paying off technical debt
+
+## Effective Machine Learning Teams - Chapter 9 - MLOps and Continuous Delivery for ML (CD4ML)
+
+CD4ML practices address help teams ensure that changes to their software and ML models are continuously tested and monitored for quality. Any changes in code, data, or ML models that satisfy comprehensive quality checks can be confidently deployed to production at any time
+
+### MLOps: Strengths and Missing Puzzle Pieces
+
+![image](https://github.com/user-attachments/assets/7667f82a-a330-4db2-b4fb-6308c406d371)
+
+CD4ML complements MLOps well by adding a set of principles and practices that help ML practitioners shorten feedback loops and improve reliability of ML systems
+
+**MLOps 101**
+
+A typical MLOps architecture:
+
+![image](https://github.com/user-attachments/assets/4883e151-f1e5-46ef-958d-07390ae24c1a)
+
+Key components:
+
+* scalable training infrastructure
+* CI/CD pipelines (with tests)
+* deployment automation
+* 'as code' everything
+* artifact stores
+* experiment tracking
+* feature stores or feature platforms (with data versioning)
+* monitoring in production
+* scalable data-labeling mechanisms
+
+**Hints That We Missed Something**
+
+* CI/CD pipelines with no tests
+* infrequent model deployments to production or preproduction environments
+* data in production goes to waste
+* X is another team’s responsibility
+
+### Continuous Delivery for Machine Learning
+
+**Benefits of CD4ML**
+
+* shorter cycle times (between an idea and shipping it to customers in production)
+* lower defect rates
+* faster recovery times (should something go wrong in production)
+
+**Continuous Delivery Principles**
+
+* Principle 1: Build quality into the product (test automation, shift left/prioritise on security)
+* Principle 2: Work in small batches (practice pair programming, use version control for all production artifacts, implement CI, apply trunk-based development - the practice of committing code changes to the main branch)
+* Principle 3: Automation—computers perform repetitive tasks, people solve problems (automate dev env setup, automate deployments (minimally to a prod env), monitoring in production)
+* Principle 4: Relentlessly pursue continuous improvement
+* Principle 5: Everyone is responsible
+
+**Building Blocks of CD4ML: Creating a Production-Ready ML System**
+
+<img width="807" alt="image" src="https://github.com/user-attachments/assets/4ceb45f6-f4cb-49a2-a23d-088a7b4ef831" />
+
+### How CD4ML Supports ML Governance and Responsible AI
+
+As ML capabilities grow, so do risks like bias amplification and societal harm, necessitating ML governance and Responsible AI frameworks to ensure ethical and effective use. Responsible AI integrates principles, policies, tools, and processes to guide the design and deployment of AI systems, balancing societal good and business impact.  
+
+CD4ML (Continuous Delivery for Machine Learning) operationalizes these principles by:  
+
+1. enabling iterative improvements: facilitating small, incremental changes to detect and resolve issues early, reducing harm and ensuring timely fixes
+2. maximizing model lifetime value: supporting MLOps to improve model quality, manage risks, and maximize value through automated monitoring and timely updates
+3. defining and enforcing policy-as-code: automating compliance with Responsible AI policies, such as consent validation and bias testing, reducing manual effort while ensuring auditability
+4. encouraging tough questions: fostering a culture where team members can question practices and explore potential harms or failure modes collaboratively
+
+---
+
+## Effective Machine Learning Teams - Chapter 10 - Building Blocks of Effective ML Teams
+
+### Common Challenges Faced by ML Teams 
+
+* unsurfaced conflicting approaches (ie. one person using tensorflow and another pytorch)
+* intra-team silos of specialization
+* lack of shared purpose
+* no sense of progress
+* lack of skill diversity
+* no common ground
+* unsafe to fail, or even to ask
+
+### Effective Team Internals
+
+**Trust as the Foundational Building Block**
+
+![image](https://github.com/user-attachments/assets/79e36fb4-a99e-4708-95dd-e8f2c94acaeb)
+
+Trust as the Foundation
+
+Patrick Lencioni’s The Five Dysfunctions of a Team underscores trust as the essential building block of team success. Without trust, teams struggle with productive conflict, commitment, accountability, and achieving results. Building trust is not a one-time activity; it requires sustained effort, leadership advocacy, and awareness of group dynamics. Challenges like diverse backgrounds, high-stakes work, and imposter syndrome often exacerbate issues in trust-building, particularly in ML teams.
+
+Vulnerability and Leadership
+
+Brené Brown’s Daring Greatly complements Lencioni by emphasizing vulnerability as critical to trust-building. Leaders can model vulnerability by admitting uncertainties and encouraging openness. Thoughtful team-building activities and empathy foster connections, lowering barriers to vulnerability.
+
+Takeaways:
+
+* reflect on management styles in your work context
+* identify your team’s developmental stage and discuss ways to move forward
+* use the Belbin framework to recognize team strengths, default roles, and opportunities to adopt unfulfilled roles for better collaboration
+
+**Communication**
+
+Communication is a key building block in teams, and something that challenges us all at times. In ML teams, where there are many different specializations, communication can be particularly hard. Are you referring to a product 'feature' or a data 'feature'? Although the average person says 'average' did you mean 'mean'? And recall, precision in ML may be more important than accuracy
+
+Communication is another skill we can improve with deliberate attention, guided by frameworks. A typical team in the tech industry has many noisy communication channels to choose from, raising the question of which channel(s) to use for communications. A typical team also has its ups and downs, and trivial and critical matters to raise and resolve, and it’s important that communication works well in all of these scenarios—especially the difficult conversations. 
+
+![image](https://github.com/user-attachments/assets/b17b32a4-32b0-4ae1-925f-627b47cd9b03)
+
+**Diverse Membership**
+
+![image](https://github.com/user-attachments/assets/b3d033df-74dc-40c3-b03f-c4956b2ca067)
+
+**Purposeful, Shared Progress**
+
+Progress in meaningful work is the most significant factor influencing the inner work life of individuals, followed by the presence of supportive systems (catalysts) and the absence of blockers (inhibitors)
+
+### Improving Flow with Engineering Effectiveness
+
+ML teams share similarities with software engineering teams but differ in their reliance on extensive discovery activities and complex data supply chains. Modern data engineering practices aim to streamline these processes using software-defined solutions. Research on Developer Experience (DevEx) highlights sociotechnical factors impacting productivity, categorizing them into three dimensions:
+
+* feedback loops: fast feedback reduces delays and frustrations, enabling quicker iteration. Techniques like automated testing, pair programming, and clear user stories improve feedback cycles. For long-running tasks, minimizing active involvement (duty cycle) and using automation enhances efficiency
+* cognitive load: high cognitive load stems from both essential and incidental complexities, such as messy codebases or unclear documentation. Reducing cognitive load involves eliminating avoidable obstacles, streamlining workflows, and providing user-friendly tools for repetitive tasks
+* flow state: achieving flow state—characterized by deep focus and enjoyment—boosts productivity and innovation. Protecting flow involves minimizing interruptions, maintaining clear goals, and fostering autonomy while aligning individual efforts with team objectives
+
+---
+
+## Effective Machine Learning Teams - Chapter 11 - Effective ML Organizations
+
+### Common Challenges Faced by ML Organizations
+
+* centralized team has too many responsibilities
+* inter-team dependencies: handoffs and waiting
+* duplication of effort in multiple cross-functional teams
+
+### Effective Organizations as Teams of Teams
+
+**The Role of Value-Driven Portfolio Management**
+
+Value-driven portfolio management emphasizes aligning an organization’s portfolio of work with business value and strategy to enhance team effectiveness. Drawing from concepts in Team Topologies and EDGE: Value-Driven Digital Transformation, this approach advocates for self-sufficient, cross-functional teams organized around delivering value. Tools like the Lean Value Tree help structure portfolios dynamically, focusing on strategic goals and evolving execution plans.
+
+**Team Topologies Model**
+
+![image](https://github.com/user-attachments/assets/2807f876-9c45-4f25-bda4-a50a9ea7120d)
+
+**Team Topologies for ML Teams**
+
+* stream-aligned team: ML product team
+
+<img width="797" alt="image" src="https://github.com/user-attachments/assets/47fb3394-1478-4c45-9d4f-ef02b594002f" />
+
+![image](https://github.com/user-attachments/assets/47f5d7d5-4345-490c-8a2d-f221fcf8d939)
+
+* complicated subsystem team: ML domain team
+
+<img width="804" alt="image" src="https://github.com/user-attachments/assets/81983c1a-56d4-4423-933c-d9d48b0e4091" />
+
+![image](https://github.com/user-attachments/assets/de758a86-4a33-4598-b58e-8f9597486ba9)
+
+* platform team: ML and data platform team
+
+<img width="784" alt="image" src="https://github.com/user-attachments/assets/0544f3d8-50c6-43a6-9cf9-7a51e452970a" />
+
+* enabling team: Specialists in some aspect of ML product development
+
+<img width="790" alt="image" src="https://github.com/user-attachments/assets/89da895f-4ce9-4fd5-920b-9a599e64cad5" />
+
+* combining and evolving topologies
+
+![image](https://github.com/user-attachments/assets/5dd7f67e-9886-4963-801a-840c432f109f)
+
+* an example topology
+
+![image](https://github.com/user-attachments/assets/d93c7d58-7835-4055-ba44-17ed47bd22fc)
+
+### Intentional Leadership
+
+* create structures and systems for rffective teams
+* engage stakeholders and coordinate organizational resources
+* cultivate psychological safety
+* champion continuous improvement
+* embrace failure as a learning opportunity
+* build the culture you wish you had
+* encourage teams to play at work
+
+I guess it is understandable that there are many references to management frameworks in these latter chapters and it seems it would be valuable for managers/team builders to understand the concepts.
+
+---
+
+The Kafka/Flink lectures/labs from Zach Wilson's lab were not released today but I expect them tomorrow ^^
+
+That is all for today!
+
+See you tomorrow :)

From d9c02a99fc80d4c386b301a2c8550d466bd83d5f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 16 Dec 2024 22:35:30 +0900
Subject: [PATCH 541/638] Create 2024-12-16-day-350.md

---
 _posts/2024-12-16-day-350.md | 44 ++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 _posts/2024-12-16-day-350.md

diff --git a/_posts/2024-12-16-day-350.md b/_posts/2024-12-16-day-350.md
new file mode 100644
index 000000000..27c9471e4
--- /dev/null
+++ b/_posts/2024-12-16-day-350.md
@@ -0,0 +1,44 @@
+---
+layout: post
+title: (Day 350) Using Airflow for a RAG app
+categories: [mlops,traditional-machine-learning,nlp,data-eng]
+---
+
+# Hello :) Today is Day 350!
+A quick summary of today:
+* covering week 4 - Streaming from Zach Wilson's YT camp
+* streaming an amazing free course on 'GenAI with Apache Airflow'
+
+## [Streaming by Zach Wilson](https://www.youtube.com/watch?v=dzB0HMtMIfI)
+
+![Streaming with Kafka and Flink P1](https://github.com/user-attachments/assets/04dd6e3f-9bf2-4b72-bab4-94fcc8421352)
+![Streaming with Kafka and Flink P2](https://github.com/user-attachments/assets/cdb9e89f-fe01-4f90-8961-1bed77d8a4b5)
+![Streaming with Kafka and Flink P3](https://github.com/user-attachments/assets/cf8c7298-24c4-4de1-85ff-1d64e4ce2f03)
+![Streaming with Kafka and Flink P4](https://github.com/user-attachments/assets/5327a2c9-20fd-42b5-b94b-0670ea1b8416)
+![Streaming with Kafka and Flink P5](https://github.com/user-attachments/assets/db28dc86-16e1-4afe-9d34-7a89dc559dad)
+
+## [Stream - GenAI with Apache Airflow course](https://youtu.be/2VIkA2Fs64s)
+
+I covered [this(https://academy.astronomer.io/introduction-to-genai-with-apache-airflow/1986175] *amazing* free course on Astronomer on using Airflow for a RAG app. It was amazing, and learned a lot about how Airflow works and making a more complex pipeline like:
+
+![image](https://github.com/user-attachments/assets/b912d787-dc02-4a5e-b62d-985b62bef3f4)
+
+The 2nd DAG was just to delete the docs from the db:
+
+![image](https://github.com/user-attachments/assets/0f12ea84-a0c6-47e7-8447-33da952e7681)
+
+The UI looks like:
+
+![image](https://github.com/user-attachments/assets/610f48e1-d58d-4979-82bd-a60e52f56d58)
+
+and:
+
+![image](https://github.com/user-attachments/assets/459f5777-1df0-47b1-b1c3-d392fdb30c7e)
+
+I have thought about an app like that for this blog as well. However, questions like: "what did Ivan study" would have to force a traverse of the whole blog's info, which can just be done with a preset answer 😆 which I will kind of write for the last day. And questions like "what did Ivan study in day X" - well that might be a slight use case, but also can easily be answered even without an LLM in the RAG - just return the summary I write at the start of each post ~ Nevertheless it was a great blog and taught me some new things about Airflow like dynamic (parallel) task execution, branching, and just creating a more complex DAG. 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 29465b4be2bbfe1d2e47e858b7fe96c3705741a3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 18 Dec 2024 00:20:10 +0900
Subject: [PATCH 542/638] Create 2024-12-17-day-351.md

---
 _posts/2024-12-17-day-351.md | 1002 ++++++++++++++++++++++++++++++++++
 1 file changed, 1002 insertions(+)
 create mode 100644 _posts/2024-12-17-day-351.md

diff --git a/_posts/2024-12-17-day-351.md b/_posts/2024-12-17-day-351.md
new file mode 100644
index 000000000..40b9899df
--- /dev/null
+++ b/_posts/2024-12-17-day-351.md
@@ -0,0 +1,1002 @@
+---
+layout: post
+title: (Day 351) Reading Data Engineering with dbt
+categories: [mlops,traditional-machine-learning,nlp,data-eng]
+---
+
+# Hello :) Today is Day 351!
+A quick summary of today:
+* this book is amazing!!! learned a lot about using dbt for SCD
+* [my repo for the code from today](https://github.com/divakaivan/portfolio-tracking-dbt)
+
+---
+
+## [Data Engineering with dbt](https://learning.oreilly.com/library/view/data-engineering-with/9781803246284/) - Chapter 1/2 - The Basics of SQL to Transform Data/Setting up Snowflake and dbt cloud
+
+The book starts by introducing SQL, its usage and some key words like table, queries, ddl, etc. 
+
+Then I had to create a snowflake account so that I can follow the book examples.
+
+<img width="708" alt="image" src="https://github.com/user-attachments/assets/c9dcf8f9-b827-43f3-99d6-ce5e7871424b" />
+
+To set up users, a warehouse and a db I ran the below queries:
+
+```sql
+/**
+ * Chapter 1 - Section 3: Set up initial users, roles and database in Snowflake
+ */
+
+-- **Create and grant your first role
+USE ROLE USERADMIN;
+
+CREATE ROLE DBT_EXECUTOR_ROLE
+    COMMENT = 'Role for the users running DBT models';
+
+--DROP ROLE DBT_EXECUTOR_ROLE;  -- if you want to drop and redo ;)
+
+GRANT ROLE DBT_EXECUTOR_ROLE TO USER divakaivan;
+SHOW GRANTS TO USER divakaivan;
+
+-- ** Grant the privileges to create a database
+USE ROLE SYSADMIN;
+GRANT CREATE DATABASE ON ACCOUNT
+  TO ROLE DBT_EXECUTOR_ROLE;
+
+GRANT USAGE ON WAREHOUSE COMPUTE_WH
+  TO ROLE DBT_EXECUTOR_ROLE;
+  
+GRANT CREATE DATABASE ON ACCOUNT TO ROLE DBT_EXECUTOR_ROLE;
+
+GRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE DBT_EXECUTOR_ROLE;
+
+SHOW GRANTS TO ROLE DBT_EXECUTOR_ROLE;
+--SHOW GRANTS TO ROLE SYSADMIN;
+
+-- ** Create your first database
+USE ROLE DBT_EXECUTOR_ROLE;
+
+CREATE DATABASE DATA_ENG_DBT;
+SHOW Databases;
+
+-- configuring the default warehouse
+USE ROLE SYSADMIN;
+ALTER WAREHOUSE "COMPUTE_WH" SET
+    WAREHOUSE_SIZE = 'XSMALL'
+    AUTO_SUSPEND = 60
+    AUTO_RESUME = TRUE
+    COMMENT = 'Default Warehouse';
+
+-- ** Create a user for the dbt application
+CREATE USER IF NOT EXISTS DBT_EXECUTOR
+    COMMENT = 'User running DBT commands'
+    PASSWORD = 'pick_a_password'
+    DEFAULT_WAREHOUSE = 'COMPUTE_WH'
+    DEFAULT_ROLE = 'DBT_EXECUTOR_ROLE';
+
+-- assign the executor role to the dbt user
+GRANT ROLE DBT_EXECUTOR_ROLE TO USER DBT_EXECUTOR;
+
+-- Switch back to operational role we should be always working with
+USE ROLE DBT_EXECUTOR_ROLE;
+```
+
+**Next - query syntax**
+
+The full code is [here](https://github.com/PacktPublishing/Data-engineering-with-dbt/blob/main/Chapter_01/01_04_Query_syntax.sql) but it's basically just basic SQL query examples.
+
+Some examples:
+
+<img width="1349" alt="image" src="https://github.com/user-attachments/assets/26830aeb-72ce-4da0-ad81-ee87ecb231c3" />
+
+<img width="1344" alt="image" src="https://github.com/user-attachments/assets/f029fb77-df94-4e56-9217-6728deecf27b" />
+
+**Next - some JOIN clauses**
+
+[Here](https://github.com/PacktPublishing/Data-engineering-with-dbt/blob/main/Chapter_01/01_05_JOIN_clause.sql)
+
+**Finally - some window functions**
+
+```sql
+SELECT
+    O_ORDERKEY,
+    O_CUSTKEY,
+    O_ORDERDATE,
+    O_TOTALPRICE,
+    avg(O_TOTALPRICE) over(partition by O_ORDERDATE) as daily_avg,
+    sum(O_TOTALPRICE) over(partition by O_ORDERDATE) as daily_total,
+    sum(O_TOTALPRICE) over(partition by
+        DATE_TRUNC(MONTH, O_ORDERDATE)) as monthly_total,
+    O_TOTALPRICE / daily_avg * 100 as avg_pct,
+    O_TOTALPRICE / daily_total * 100 as day_pct,
+    O_TOTALPRICE / monthly_total * 100 as month_pct
+FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."ORDERS"
+QUALIFY row_number() over(partition by O_ORDERDATE
+                          order by O_TOTALPRICE DESC) <= 5
+order by O_ORDERDATE, O_TOTALPRICE desc;
+```
+
+**Setting up dbt cloud and connecting it to Snowflake**
+
+Faced some issues here with providing the exact credentials, but thankfully the dbt docs had my back.
+
+---
+
+## Chapter 3 - Data Modeling for Data Engineering
+
+### Data modelling
+
+Attributes of DM:
+
+* entity: a concept that we want to analyze, a thing of interest that we want to collect data about, such as a car, an order, or a page visit
+* attribute: a property of an entity for which we want to store values, such as a car plate, a customer, or the total amount of a transaction
+* relationship: a connection between two entities that are related to each other and captures how they are related, such as the ownership relation between a car and its owner, or the purchasing relation between a customer and a product
+
+Data modeling is all about conveying information so that those that work with the modeled data can better understand it and each other.
+
+Data modeling is generally used in two main contexts:
+
+* to describe what data should be generated or collected by an application or in a project
+
+In this case, we build the data model before having the data.
+
+* to describe some data that already exists
+
+In this case, we reverse-engineer the data model from the available data.
+
+A good practice is to explicitly design this new data model before coding the transformations, drawing at least one model that represents the business concepts that are the goal of the integration and will form the refined targets of our transformations. This model can be at a high level of abstraction, describing the business and only the main entities, or at a greater level of detail, already representing the technical design.
+
+### Conceptual, logical, and physical data models
+
+* conceptual data model: this is the most abstract model, defining what will be in the domain of the project, providing the general scope
+
+![image](https://github.com/user-attachments/assets/57b28497-46f0-4a0c-83ce-01078c14e451)
+
+* logical data model: provides much greater detail, defining what the data will look like
+
+![image](https://github.com/user-attachments/assets/7be965b7-f5c0-4bcf-95cb-72b0579b827e)
+
+* physical data model: the most detailed model, describing exactly how the data will be stored in the database
+
+### Entity-Relationship modeling
+
+![image](https://github.com/user-attachments/assets/90f6b887-9799-438a-a114-e778ff492a55)
+
+* entity: represented as a box with the name of the entity inside
+
+If attributes are shown, the entity name is at the top and often in bold or bigger and visually separated from the attributes
+
+* attribute: represented by the attribute’s name inside the box of the entity it belongs, with one name per line
+* relationship: represented by a line joining the entities involved in the relationship and optionally the name of the relationship close to the line
+
+**Cardinality**
+
+* 1-to-1
+* 1-to-many
+* many-to-many
+
+**Time perspective**
+
+The time perspective significantly influences E-R models, though it is often an implicit and underexplained factor. E-R models capture constraints or business rules of a system, and the system's time perspective affects entity semantics, applicable rules, and the model's structure.
+
+For operational applications, relationships like Driver-Taxi are often viewed as one-to-one at a specific time (e.g. a taxi needs a driver, and a driver can drive one taxi at a time). However, both entities make sense independently when inactive (a taxi not in use or a driver on a break). Time-based entities (shifts or contracts) are sometimes embedded to track relationships over time.
+
+In data engineering projects, a lifetime perspective is typically adopted, showing relationships over the lifespan of entities. For example, over time, a driver may drive multiple taxis, and a taxi may have multiple drivers, resulting in a many-to-many relationship. Such relationships require an intermediate table (e.g., Work Period) to store periods when changes occur
+
+### Modeling use cases and patterns
+
+**Header-detail use case**
+
+As an example, it is used in invoices and orders, and almost anywhere else where there is one document with multiple lines of detail inside. The common document info goes into the header entity with the detail lines represented by one or more weak entities depending on the header. A weak entity is an entity that does not have any meaning without the header entity, such as an order line without an order header.
+
+**Hierarchical relationships**
+
+Hierarchical relationships are very common and generally are a special case of one-to-many relationships. They represent data structures that look like a tree.
+
+The common pattern is that one instance depends on another higher/bigger instance of the same entity, unless it is itself one of the instances at the highest/biggest level.
+
+![image](https://github.com/user-attachments/assets/26f77986-0540-40a7-85eb-1a3af0a4b64b)
+
+![image](https://github.com/user-attachments/assets/a7055748-212c-4d2c-841c-4af376369cfb)
+
+![image](https://github.com/user-attachments/assets/c1c4edbf-155b-403d-bf3e-9a77059ef872)
+
+**Forecasts and actuals**
+
+![image](https://github.com/user-attachments/assets/6ffbeb7f-d7c6-4caf-8a5c-f5b2cf98df82)
+
+Budgeting and forecasting involve predicting figures and comparing them with actuals. A one-to-many relationship exists, with one actual linked to multiple forecasts. Reporting models often generalize Figures into subclasses (actuals and forecasts) and use a single table to simplify comparisons. Actuals are copied for each forecast using joins, relying on reliable forecast dates and business logic.
+
+### Common problems in data models
+
+**Fan trap**
+
+The fan trap is a very common problem that can happen every time you have a join in a one-to-many relationship. It is not a problem of the relationship, but of how you might use it.
+
+The fan trap problem causes the calculations done on measures joined from the one side of the one-to-many relationship to be wrong.
+
+This is only a problem if you use a measure that is on the one side of a one-to-many relationship, when grouping/working at the granularity of the entities on the many side. This happens because the join will duplicate the measures to match the cardinality of the many side.
+
+**Chasm trap**
+
+A chasm trap is an unmanaged and often unintentional many-to-many relationship. Because it is unmanaged, it becomes an unintended partial Cartesian product: the rows on one side will be cross-joined with the occurrences of the same key on the other side, de facto making a per-key Cartesian product.
+
+_Chasm trap and a correct many-to-many relationship model_
+
+![image](https://github.com/user-attachments/assets/4d96f95e-64b2-44ac-9aa8-2606e22a0dbe)
+
+### Modeling styles and architectures
+
+**Kimball method or dimensional modeling or star schema**
+
+![image](https://github.com/user-attachments/assets/d826a004-849e-40f3-8e2c-f8dde44ba0ce)
+
+Limitations:
+
+* measures in dimensions: while Kimball advises against including measures in dimensions to avoid fan traps, practical needs like storing important metrics (e.g., price in a product dimension) often necessitate this, requiring caution to handle fan traps correctly
+
+* no fact-to-fact connections: Kimball schemas restrict facts to only connect with dimensions, which limits analyses involving multiple facts (e.g. orders and deliveries). Alternative approaches like USS or custom SQL are needed for such scenarios
+
+* limited enterprise view: the Kimball approach focuses on reporting specific business processes rather than providing a comprehensive enterprise data view, which may not meet all reporting needs
+
+* business rules dependency: the loading and storage processes are tightly integrated with business rules, as conformed dimensions and facts serve both as transformation goals and primary data storage
+
+**Unified start schema**
+
+The USS is an extension of the Kimball star schema that addresses some of its limitations, such as the fan trap and the impossibility to query more than one fact at a time without the risk of a chasm trap.
+
+![image](https://github.com/user-attachments/assets/61176264-cd4f-44b5-8421-3401fde23c11)
+
+**Inmon design style**
+
+The Inmon design style, created by Bill Inmon, emphasizes building an enterprise-wide, integrated, and interoperable data warehouse as a single source of truth for an organization. Its core idea is to model business concepts (entities and processes) and normalize the data into Third Normal Form (3NF) to minimize redundancy and capture complex relationships.
+
+Advantages
+
+* resilience: adaptable to changes in business processes or source systems.
+* comprehensive: captures rich concepts and relationships for broad reporting capabilities
+
+Drawbacks
+
+* complexity: requires highly skilled modelers and significant management commitment
+* auditability: original data is hard to trace due to heavy transformations
+* effort-intensive: maintenance and updates for new reporting needs are resource-heavy
+
+**Data vault**
+
+![image](https://github.com/user-attachments/assets/ede71cdf-04d2-4496-8946-26a9ab7ccc02)
+
+DV architecture:
+
+![image](https://github.com/user-attachments/assets/79965dde-aa47-481e-b150-bdfb7f777791)
+
+**Data mesh**
+
+Data mesh is a modern data architecture approach aimed at solving data ownership issues by decentralizing data management. Instead of a centralized data platform, it organizes data into domain-specific platforms, each owned and maintained by business units familiar with the data's semantics and use cases. Inspired by microservice architecture and DevOps practices, data mesh emphasizes 'data products' — well-defined interfaces, applications, or data marts enabling seamless interaction across domains.  
+
+While it aligns business and IT goals, improves data quality, and empowers teams with end-to-end data ownership, it introduces complexity and requires advanced skills in managing distributed systems and APIs. Adoption is challenging in less tech-savvy industries, but insights like treating data as products and assigning data ownership to business units can still be applied. Tools like dbt facilitate collaboration and documentation in this framework.
+
+**The Pragmatic Data Platform - PDP**
+
+PDP combines the best aspects of various data modeling styles and architectures to create a modern, efficient data platform while minimizing complexity and expertise requirements.  
+PDP adopts key principles from Data Vault (DV) and Inmon, such as:  
+- separating ingestion from business rules
+- using insert-only ingestion of full history
+- organizing data around business concepts and clear BK definitions
+
+However, it avoids DV modeling due to its complexity, accepting a trade-off of reduced resilience for simplicity. Data delivery is typically through data marts with star schemas but can also include wide, denormalized tables for specific use cases like ML/AI.  
+PDP emphasizes practical solutions, balancing advanced techniques (like using hashes) with accessibility to a broader audience. This pragmatic approach leverages the full capabilities of dbt while simplifying implementation.
+
+---
+
+## Chapter 4 - Analytics Engineering as the New Core of Data Engineering
+
+### The data life cycle
+
+1. data creation
+2. data movement and storage
+3. data transformation
+4. access to reports
+5. data write back
+
+### Understanding the modern data stack
+
+**Traditional**
+
+* ETL
+* limitations of OLTP systems
+* rigid transformation logic
+* data gaps
+* siloed workflows
+* DQ issues
+
+**Modern**
+
+* analytical data warehouses in the cloud, such as BigQuery, Redshift, and Snowflake, are more and more common and becoming an affordable option for most data teams
+* data movement/integration tools, such as Stitch and Fivetran, make it very easy to move the data from traditional databases and common business platforms to the cloud data warehouse
+* modern BI tools start to provide improved abilities for self-service reporting that power users can exploit
+
+### The roles in the modern data stack and the analytics engineer
+
+* data engineer: the 'traditional' DE, armed with scripting/coding and infrastructure knowledge, is still very much valuable, especially if not all your sources can be covered by modern data-loading tools. Even in that case, complex scheduling, security, and a myriad of other topics might require the expertise and focus of a data engineer
+* analytics engineer: a new figure that is firmly centered around knowing and transforming raw data into useful information. They are less concerned with data movement pipeline development and are instead focused on making the data available, usable, and trusted
+* data analyst: this is an improved version of the traditional data analyst, as now they can understand and even help with the construction of data models and certainly have an unprecedented ability to investigate a wider array of datasets
+
+### DataOps – software engineering best practices for data
+
+* version control
+* quality assirance
+* modularity of the code base
+* dev vs prod environments
+* designing for maintainability
+
+---
+
+## Chapter 5 - Transforming Data with dbt
+
+### Layers of a modern data platform
+
+![image](https://github.com/user-attachments/assets/22439cc7-5c47-48f6-9fbe-c77cc424b870)
+
+The implementation effort and cost over time in a warehousing project
+
+![image](https://github.com/user-attachments/assets/ec42db5c-19fe-41a2-9208-165c18535963)
+
+### The project
+
+The book will guide me to build a simple stock tracking platform
+
+**The initial data model and glossary**
+
+This sample project is about being able to track the value of investments spread across multiple portfolios and analyze them in the ways we prefer. We will start simple, just tracking the current value of the portfolio, and then add more functionalities along the next sections and chapters.
+
+The following conceptual data model illustrates the main entities and their relations, as you could gather from the initial explanation of the use case.
+
+![image](https://github.com/user-attachments/assets/48a3a882-af6d-48d9-9384-6b75be3f29ad)
+
+* Position: a position tracks the amount of a specific security hold in a portfolio and comes with a few measures besides the amount of stock owned, such as the cost of the security hold in the position
+* Portfolio: a portfolio is a container of positions, usually associated with a trading or custody account at a broker
+* Security: a security is an investment instrument that can be bought and sold on a specific market. Securities exist in multiple types, including bonds, stock, options, and many others
+* Quote: a security quote is a set of values, such as a bid and an ask price, that describe the market offering for the stock at a moment in time or a set of values that describe the market offering for the stock during an interval of time:
+  * Daily quote: a daily quote is a set of values, such as open, close, max, and min prices, that describe the market offering for the stock during a trading day.
+* Calendar: the calendar represents the time dimension at the desired resolution, often a day, providing useful precalculated attributes for all the items in the calendar
+
+### Setup
+
+* [my dbt project repo](https://github.com/divakaivan/portfolio-tracking-dbt)
+* snowflake db
+
+```sql
+USE ROLE DBT_EXECUTOR_ROLE;
+CREATE DATABASE PORTFOLIO_TRACKING
+    COMMENT = 'DB for the portfolio tracking project';
+```
+
+* set up dbt cloud 
+
+<img width="809" alt="image" src="https://github.com/user-attachments/assets/2392be4c-82c9-4d47-ade7-9349fab25752" />
+
+### Defining data sources and providing reference data
+
+Setup snowflake:
+
+```sql
+USE ROLE DBT_EXECUTOR_ROLE;
+
+-- ** 1 ** Create the schema for the source data
+CREATE SCHEMA PORTFOLIO_TRACKING.SOURCE_DATA;
+
+-- ** 2 ** Create the landing table for ABC_BANK data
+CREATE OR REPLACE TABLE PORTFOLIO_TRACKING.SOURCE_DATA.ABC_BANK_POSITION (
+     accountID	    TEXT,
+     symbol	        TEXT,
+     description	TEXT,
+     exchange	    TEXT,
+     report_date	DATE,
+     quantity	    NUMBER(38,0),
+     cost_base	    NUMBER(38,5),
+     position_value	NUMBER(38,5),
+     currency	    TEXT
+);
+
+-- ** 3 ** Create the file format to load the data for ABC_BANK
+CREATE FILE FORMAT "PORTFOLIO_TRACKING"."SOURCE_DATA".ABC_BANK_CSV_FILE_FORMAT
+    TYPE = 'CSV'
+        COMPRESSION = 'AUTO'
+        FIELD_DELIMITER = ','
+        RECORD_DELIMITER = '\n'
+        SKIP_HEADER = 1
+        FIELD_OPTIONALLY_ENCLOSED_BY = '\042'
+        TRIM_SPACE = FALSE
+        ERROR_ON_COLUMN_COUNT_MISMATCH = TRUE
+        ESCAPE = 'NONE'
+        ESCAPE_UNENCLOSED_FIELD = '\134'
+        DATE_FORMAT = 'AUTO'
+        TIMESTAMP_FORMAT = 'AUTO'
+        NULL_IF = ('\\N')
+;
+```
+
+Next, I inserted the given csv file manually:
+
+<img width="1255" alt="image" src="https://github.com/user-attachments/assets/395eda55-45e0-4623-a3d4-b31260cc99b5" />
+
+Next, creating a source for the table just inserted:
+
+```
+version: 2
+
+sources:
+  - name: abc_bank
+    database: PORTFOLIO_TRACKING
+    schema: SOURCE_DATA
+    tables:
+      - name: ABC_BANK_POSITION
+```
+
+And ~
+
+<img width="1156" alt="image" src="https://github.com/user-attachments/assets/ca8991bb-89bd-475e-981f-5b334565a5a1" />
+
+It works ~ the database, schema, table names, connections are good :) 
+
+### Writing a basic model
+
+```sql
+SELECT
+    *, 
+    POSITION_VALUE - COST_BASE as UNREALIZED_PROFIT,
+    ROUND(UNREALIZED_PROFIT / COST_BASE, 5)*100 as UNREALIZED_PROFIT_PCT
+FROM {{ source('abc_bank', 'ABC_BANK_POSITION') }}
+```
+
+And it appeared in snowflake after `dbt run`:
+
+<img width="1223" alt="image" src="https://github.com/user-attachments/assets/b7874faf-2173-4225-bfea-a3e173d89b7e" />
+
+Finally the dbt_project.yml is updated to include the different layers of the architecture:
+
+```
+models:
+  portfolio_tracking:
+    +materialized: view
+    staging:    # Data storage layer
+      +materialized: view
+      +schema: STAGING
+    refined:    # Refined data layer
+      +materialized: view
+      +schema: REFINED
+    marts:      # Data delivery layer
+      portfolio:     # one specific data mart
+        +materialized: table
+        +schema: MART_PORTFOLIO
+snapshots:
+  portfolio_tracking:
+    +target_schema: SNAPSHOTS
+```
+
+Generating views or tables for the different layers is mostly a performance choice, as they are functionally equivalent. Tables are slower to create and take up space on storage, but they are much quicker to query.
+
+Therefore, it makes sense to materialize models that are queried multiple times during the ELT as tables so that the calculations are performed only once, as well as for models queried by end users in interactive mode so that the query executes in the shortest time possible.
+
+We configured the refined layer to be generated as tables; that is a sensible choice when we do not have a data mart layer yet and the refined layer is also our delivery layer.
+
+The delivery layer is usually materialized at tables, but a BI tool that stores the data from your data mart layer internally is used, such as Power BI - it is a sensible choice to generate the layer as views, because each view will be queried once to load the data in the tool.
+
+**Some tests**
+
+```
+version: 2
+
+sources:
+
+  - name: abc_bank
+
+    database: PORTFOLIO_TRACKING
+    schema: SOURCE_DATA
+
+    tables:
+      - name: ABC_BANK_POSITION
+        description: The landing table holding the data imported from the CSV extracted by ABC Bank.
+        columns:
+          - name: ACCOUNTID
+            description: The code for the account at the ABC Bank.
+            tests:
+              - not_null
+          - name: SYMBOL
+            description: The symbol of the security for the ABC Bank.
+            tests:
+              - unique
+              - not_null
+          - name: DESCRIPTION
+            description: The name of the security at the ABC Bank.
+            tests:
+              - not_null
+          - name: EXCHANGE
+            description: The short name or code of the exchange where the security is traded.
+            tests:
+              - not_null
+          - name: REPORT_DATE
+            description: >
+              The date of the portfolio report extracted by ABC Bank.
+              We consider this position to be the effective from this date forward, until a change is seen.
+            tests:
+              - not_null
+          - name: QUANTITY
+            description: The number of secuities we hold in the portfolio at ABC Bank at the Report Date.
+            tests:
+              - not_null
+          - name: COST_BASE
+            description: The cost we paid attributable to the secuities we hold in the portfolio at ABC Bank at the Report Date.
+            tests:
+              - not_null
+          - name: POSITION_VALUE
+            description: The value of the secuities we hold in the portfolio at ABC Bank acording to market quotation on the Report Date.
+            tests:
+              - not_null
+          - name: CURRENCY
+            description: The currency for the monetary amounts of the position.
+            tests:
+              - not_null
+```
+
+Added a test for the 2 new columns in the POSITION_ABC_BANK table:
+
+```
+version: 2
+models:
+  - name: POSITION_ABC_BANK
+    description: The positions we have in the ABC Bank portfolio.
+    columns:
+      - name: UNREALIZED_PROFIT
+        description: The unrealized profit on the position.
+        tests:
+          - not_null
+      - name: UNREALIZED_PROFIT_PCT
+        description: >
+          The unrealized profit percentage on the position.
+        tests:
+          - not_null
+```
+
+<img width="951" alt="image" src="https://github.com/user-attachments/assets/a2ce8cbc-2d24-4fcd-a82d-e2e625d00138" />
+
+**Docs**
+
+And of course finally, we can see the amazing docs generated 
+
+<img width="1041" alt="image" src="https://github.com/user-attachments/assets/15f678e3-c492-40be-be30-31e5cb189837" />
+
+---
+
+## Chapter 6 - Writing maintainable code
+
+### Writing code for humans
+
+The other most important things that we can do to keep complexity in check are
+
+* reduce the need to re-read, applying the Least Surprise Principle (LSP)
+* make the code that you write easy to re-read/understand
+* keep the models that we write focused and limited in dimension
+* keep the dependencies ordered and going only in one direction, from one layer to the next in the architecture
+* when you see an improvement, refactor your models to keep them current
+
+### Creating the architectural layers
+
+![image](https://github.com/user-attachments/assets/00dc67b8-ca74-4266-a7da-cc5d84f67f88)
+
+**Creating the Staging layer**
+
+Goals and contents of the staging models: to adapt external data to how we want to see and use it in our project. This includes preparing it for change tracking and for storage in the most usable way for our future needs.
+
+* pick the desired columns, usually all of them
+* rename the columns to follow our conventions and use our project-designated names
+* manage data types, performing conversions to desired data types
+* apply “hard rules” with non-destructive transformations, such as time-zone conversions
+* add the default record but only for dimensions; it is not needed for facts/events
+* add the desired/available metadata, based on the input specifics
+* make keys and change tracking explicit by adding hash-based keys and diff fields
+
+A general pattern for your staging models
+
+```sql
+WITH
+src_data as ( … ),
+default_record as ( … ),
+with_default_record as(
+    SELECT * FROM src_data
+    UNION ALL
+    SELECT * FROM default_record
+),
+hashed as ( … )
+SELECT * FROM hashed
+```
+
+This is the final first staging model after following the above goals:
+
+```sql
+WITH
+
+src_data as (
+    SELECT
+        ACCOUNTID         as ACCOUNT_CODE     -- TEXT
+         , SYMBOL            as SECURITY_CODE    -- TEXT
+         , DESCRIPTION       as SECURITY_NAME    -- TEXT
+         , EXCHANGE          as EXCHANGE_CODE    -- TEXT
+         , REPORT_DATE       as REPORT_DATE      -- DATE
+         , QUANTITY          as QUANTITY         -- NUMBER
+         , COST_BASE         as COST_BASE        -- NUMBER
+         , POSITION_VALUE    as POSITION_VALUE   -- NUMBER
+         , CURRENCY          as CURRENCY_CODE    -- TEXT
+
+         , 'SOURCE_DATA.ABC_BANK_POSITION' as RECORD_SOURCE
+
+    FROM {{ source('abc_bank', 'ABC_BANK_POSITION') }}
+),
+
+hashed as (
+    SELECT
+          {{ dbt_utils.surrogate_key([ 'ACCOUNT_CODE', 'SECURITY_CODE']) }} as POSITION_HKEY
+        , {{ dbt_utils.surrogate_key([ 'ACCOUNT_CODE', 'SECURITY_CODE',
+                'SECURITY_NAME', 'EXCHANGE_CODE', 'REPORT_DATE',
+                'QUANTITY', 'COST_BASE', 'POSITION_VALUE', 'CURRENCY_CODE'
+                ]) }} as POSITION_HDIFF
+        , *
+        , '{{ run_started_at }}'::timestamp as LOAD_TS_UTC
+    FROM src_data
+)
+
+SELECT * FROM hashed
+```
+
+Metadata examples include: load_ts, record_source, extraction_time, creation_time, effective_from
+
+<img width="801" alt="image" src="https://github.com/user-attachments/assets/9bf5622b-f3f3-43cf-93a1-9ea1891561e2" />
+
+_HASH_KEY: This is a single field with all the columns that make up the primary key of the entity chained together;_
+
+_HASH_DIFF: This is a single field with all the fields that we want to test for changes chained together._
+
+_The HASH_DIFF field is a purely technical field that can be considered a version identifier, but be aware that it is perfectly normal for an entity to go back to the same set of values it had at some point in the past. Therefore, it might very well be that we have two equal hash diff values separated in the history table by one or more different ones._
+
+**Goals and contents of the refined layer**
+
+![image](https://github.com/user-attachments/assets/994133bd-b4ad-4397-a1c4-c90376bc47df)
+
+The goal of the refined layer is clear – take the 'adapted' data served from the staging layer and by applying business rules, including combining multiple sources, produce 'refined' data that can be used in the data marts to provide useful information for the platform users.
+
+The fact is that 'business rule' is a very general term and it can mean something as simple as a sum or product, or it can involve the use of multiple models and quite complicated logic.
+
+Pretty much every operation that changes or combines data to produce other data is a business rule
+
+Types of models in this refined layer:
+
+* intermediate transformation (TR): a catch-all type for models that you create for intermediate calculations that do not represent a proper business concept or that cannot exist by themselves, in the sense that you will not expect other developers to use such a model without knowing the specific caveats
+* refined model (REF): the type for models that represent a recognizable business concept, at any level, with the level needing to be clear from the name if it is not for general use. We expect that anyone needing this concept at that level should reuse this model and not build their own
+* mapping model (MAP): the type for models that support conversion from one reference system to another, by providing the mapping for keys in one system to the equivalent key in the other system
+* Master Data Dimension (MDD) model: this is the type for models that implement dimensions provided directly from the master data management system
+* aggregated (AGG) model: this is the type for models that contain aggregated or summarized data
+* pivoted (PIVOT) model: This is the type for models that contain pivoted data. This is a specialized model type that you can use if you happen to deal with pivoted data. It is listed here to make clear that you can create new model types according to your needs. The goal should always be to provide clear communication of what the model does
+
+**Creating the first data mart**
+
+![image](https://github.com/user-attachments/assets/a5ff6def-5f2b-48c4-8067-f579cce118df)
+
+The general plan is to have one data mart for each domain that will use the platform’s data so that we can provide the best dataset for each of them, using the right names, the correct depth and granularity of the data, and making available only the concepts that matter for that domain.
+
+### Saving history is crucial
+
+When working with dbt, we have two main avenues to pick up and save the changes to the instances of our entities:
+
+* using the standard functionality of dbt `snapshots`: this allows us to easily capture changes as a Slowly Changing Dimension of type 2 (SCD2) without writing any code for that purpose, as the logic is implemented by dbt itself using merge or delete/insert statements. If we get reliable full exports from the source, the snapshot can also capture deletions.
+
+This works very well for normal-size datasets (up to millions of rows), and it’s the simplest way to go, but it comes with its own cons, so we generally prefer to store changes using insert only history
+
+One of the major cons of snapshots is that they are global objects, so you only have one copy and one version active for all environments. Another one is that you cannot preview or test the SELECT query that feeds the data to the snapshot.
+
+The 2nd option is:
+
+* using incremental models to save the new or changed data from each run by applying set-based operations to capture changes in insert-only mode, as this is the most effective way to manage and store even huge amounts of data, and it can be tailored to work in all cases, even when you have multiple sources for a single table, such as a near-real-time feed and a periodic control feed.
+
+**Saving history using snapshots**
+
+A snapshot is a dbt-managed table that will store the data from a SELECT statement in the form of a slowing changing dimension.
+
+A snapshot is created by the special snapshot block that you create in a .sql file in dbt.
+
+Snapshots are not normal models, in the sense that they are not run with the `dbt run` command, but they must be run with the specific `dbt snapshot` command or with the catch-all `dbt build` command, and they are stored in a snapshots folder.
+
+_It is important to note that a snapshot table is a global object, shared by all environments that use the same database, including all developer environments. This poses substantial limitations for teamwork because when you change a snapshot, all the environments sharing the snapshot need to be updated at the same time._
+
+The general layout of a snapshot definition:
+
+```
+{% snapshot snapshot_name %}
+{{ config(…) }}
+select … from {{ source / ref }}
+{% endsnapshot %}
+```
+
+Central to the operations of a snapshot are the following concepts:
+
+* **identity** or how to distinguish between two different instances of one entity, versus two versions of the same instance. Identity is expressed by a unicity constraint that is often called a primary key, natural key, or a similar name.
+
+* **a strategy to identify changes** – that is, how you decide whether two rows with the same key are the same row or two different versions of that specific instance of the entity.
+  * there is the *timestamp* strategy which is simpler and quicker when we have a field like `updated_at` or a monothonically increasing `ID` field - and in such cases we can just compare the current ID/updated_at to decide
+  * the *check* strategy is the one that you use when you do not have a good high-watermark indicator, and instead, you must look at the data to check whether each row presented by the SELECT statement is the same or a different instance from the most recent one stored in the snapshot under the specified unique key. With this strategy, you must configure the columns to be checked to decide whether two rows are the same instance or not, using the check_cols configuration. You do not have to list all columns from SELECT but only the ones you want to compare; in any case, all columns from SELECT will be stored in the snapshot.
+
+The intended usage of snapshots is as starting point of your ETL process to capture and store the source data, so that the general ETL would follow the following pattern:
+
+1. run the snapshots command to capture the source data – dbt snapshot
+2. run the models to read the data out of the snapshots and calculate the desired transformations – dbt run
+3. run the tests to verify the results – dbt test
+
+Running a snapshot will cause the following:
+
+* the creation of a snapshot table from the result of the SELECT statement, which happens the first time that you run the snapshot
+* the snapshot table to be updated, with the changes coming from the SELECT statement relating to the contents of the snapshot table, which happens any other time that you run the snapshot after the first
+
+
+In data processing, a snapshot captures data at a specific point in time. When working with snapshots, we have the option to either read the source data unchanged and apply transformations from the STG model after the snapshot, or we can choose to include the transformations directly in the snapshot. The first option requires re-running the transformations every time we access the data, which can be inefficient. The second option, which involves incorporating the transformations into the snapshot, ensures that the transformations are applied once, and the resulting data is stored in the correct format, including any additional data calculated (like dates extracted from filenames).
+
+However, putting the transformations from the STG model directly into the snapshot can make it more challenging to build and maintain the ELT process. This is because we lose the ability to preview the results of your SELECT query inside the snapshot block, which can make development more cumbersome. This lack of previewing results can create tension with the goal of maintaining easy-to-manage models, especially when the STG layer handles most of the data transformation logic.
+
+One way to address this issue is by declaring the STG model as an 'ephemeral model'. An ephemeral model doesn’t create any persistent tables or views but instead defines a named query. This allows us to preview the results of the query before including it in the snapshot, making it easier to maintain the ELT process while still ensuring that the snapshot runs efficiently with the necessary transformations.
+
+**Layout of a snapshot tables**
+
+The snapshot table will contain the following metadata fields in addition to the fields produced by the SELECT statement
+
+* dbt_valid_from
+* dbt_valid_to
+* dbt_scd_id: a unique key generated for each row in the snapshot. This is internal to dbt
+* dbt_updated_at: the timestamp when the row was inserted. This is internal to dbt
+
+Here are some snapshot best practices:
+
+* snapshot source data as the first thing in your ETL, declaring a source as appropriate, and make the rest of the ETL depend only on the snapshot
+* include all the source columns in your snapshot, as you cannot go back in time and backfill information that you do not have, and even if you have it, the backfilling of data is a complicated and time-consuming task
+* do not add any business logic in the query that feeds data to the snapshot; use only hard rules that do not change the data. If you apply soft business rules and they will change, most probably you will not be able to go back to the original data and apply the new business rule, and even if it’s possible, it would be a complex and slow manual process
+* avoid joins in your SELECT query, as it is not always easy to establish when the result of a join is a new version, and even more difficult to absorb the changes into two tables. Snapshot the two tables to be joined independently, and then do the join in the downstream models where you apply the business logic
+
+**Multiple ways to take into use the snapshots**
+
+* a snapshot as a first step: Taking a snapshot as the first thing in the flow is what dbt suggests, but it means either preserving the source names by using the trivial `SELECT * FROM {{ source(…) }}` query or writing a query that is not simple to preview/debug
+* a snapshot after the STG model: Taking the snapshot out of an STG model solves the maintainability issue, but it is a bit clumsy if you use a normal model, as you need first to run only the STG models for the snapshots, take the snapshot, and then run all the models except the snapshot ones. The clumsiness is removed if you use ephemereal models for the STG models, as you do not need to run them before
+
+**Creating a snapshot in the project**
+
+The below is put in the snapshots folder in an sql file
+
+```sql
+{% snapshot SNSH_ABC_BANK_POSITION %}
+{{
+    config(
+      unique_key= 'POSITION_HKEY',
+      strategy='check',
+      check_cols=['POSITION_HDIFF'],
+      invalidate_hard_deletes=True,
+    )
+}}
+select * from {{ ref('STG_ABC_BANK_POSITION') }}
+{% endsnapshot %}
+```
+
+We use the POSITION_HKEY which was created in the STG_ABC_BANK_POSITION. Here is the data lineage
+
+<img width="1099" alt="image" src="https://github.com/user-attachments/assets/39ebab13-b6c0-4bd4-a715-3f7ac86c9d88" />
+
+Also, before executing `dbt snapshot` I dropped the existing STG_ABC_BANK_POSITION view as otherwise, in the future, people on the project might think it’s still connected to the STG model and even use it. And I added `{{ config(materialized='ephemeral') }}` to the top of the model file to fix the issue of not being able to preview or test that query in isolation - we change the model to ephermal materialization and use it to feed the content of the STG view to the snapshot. 
+
+After running `dbt snapshot`, in snowflake we can see:
+
+<img width="1262" alt="image" src="https://github.com/user-attachments/assets/c04fff7f-7e94-4b42-8d85-cdf89e52a2cd" />
+
+It's so cool seeing this outside of Zach Wilson's bootcamp where I learned about SCD just 3 weeks ago.
+
+**Connecting the REF layer with the snapshot**
+
+The REF_POSITION_ABC_BANK model that we wrote before is still reading data from the STG model that we made ephemeral. It still works, but it’s now time to make it read from the snapshot so that our ELT will keep working even if the external source is not available.
+
+This is the current one:
+
+```sql
+SELECT
+    *, 
+    POSITION_VALUE - COST_BASE as UNREALIZED_PROFIT,
+    ROUND(UNREALIZED_PROFIT / COST_BASE, 5) * 100 as UNREALIZED_PROFIT_PCT
+FROM {{ ref('STG_ABC_BANK_POSITION') }}
+```
+
+If we just change the ref to the snapshot, we would read everything out of the history – that is, all the versions that you encountered in the history of the snapshot which is not what we want.
+
+To report on the current portfolio, we want to read only the active positions.
+
+*To achieve this, we need to select from the snapshot only the rows that have not been closed – that is, where the DBT_VALID_TO field is null – as the old versions are the ones with a timestamp in this column.*
+
+It should be:
+
+```sql
+WITH
+current_from_snapshot as (
+    SELECT *
+    FROM {{ ref('SNSH_ABC_BANK_POSITION') }}
+    WHERE DBT_VALID_TO is null
+)
+SELECT
+    *
+    , POSITION_VALUE - COST_BASE as UNREALIZED_PROFIT
+    , ROUND(UNREALIZED_PROFIT / COST_BASE, 5)*100
+         as UNREALIZED_PROFIT_PCT
+FROM current_from_snapshot
+```
+
+All is good:
+
+<img width="1446" alt="image" src="https://github.com/user-attachments/assets/09d52ecd-430e-4ecb-bc17-0ebf6cc62cca" />
+
+🥳
+
+Here is the lineage at the end of this chapter:
+
+<img width="857" alt="image" src="https://github.com/user-attachments/assets/ead88d0c-2494-41b8-98bf-dc6fb4b1965f" />
+
+## Chapter 7 - Working with Dimensional Data
+
+They give a definition about dimensional data - descriptive data that provides human-readable information for an entity, such as the name or country of a customer or account
+
+### Adding dimensional data
+
+![image](https://github.com/user-attachments/assets/a864f148-b78b-481f-aeb2-7e17dee44b8f)
+
+### Loading the data of the first dimension
+
+**Creating and loading a CSV as a seed**
+
+<img width="1087" alt="image" src="https://github.com/user-attachments/assets/1384917a-d2dc-4a58-a2d9-22cd82521d48" />
+
+We need to configure them in the project yml:
+
+```
+seeds:
+  +schema: seed_data
+```
+
+Then we can run `dbt seed` to load the data
+
+We can add a column using a post-hook:
+
+```
+seeds:
+  +schema: seed_data
+  portfolio_tracking:
+    ABC_Bank_SECURITY_INFO:
+      +post-hook:
+        - "UPDATE {{ this }} SET LOAD_TS = '{{ run_started_at }}' WHERE LOAD_TS is null"
+      +column_types:
+        LOAD_TS: TIMESTAMP
+```
+
+And also add the column name LOAD_TS and no values in the seed csv
+
+<img width="531" alt="image" src="https://github.com/user-attachments/assets/3338cc39-435e-4829-8a21-435073bbf627" />
+
+And load_ts becomes the timestamp when the dbt seed is run
+
+<img width="1162" alt="image" src="https://github.com/user-attachments/assets/694da637-6850-4c8a-b467-b73149dd3a54" />
+
+(the time zone is 9hrs behind - Europe)
+
+### Building the STG model for the first dimension
+
+**Creating an STG model for the security dimension**
+
+```sql
+{{ config(materialized='ephemeral') }}
+
+WITH
+src_data as (
+    SELECT
+        SECURITY_CODE     as SECURITY_CODE    -- TEXT
+         , SECURITY_NAME     as SECURITY_NAME    -- TEXT
+         , SECTOR            as SECTOR_NAME      -- TEXT
+         , INDUSTRY          as INDUSTRY_NAME    -- TEXT
+         , COUNTRY           as COUNTRY_CODE     -- TEXT
+         , EXCHANGE          as EXCHANGE_CODE    -- TEXT
+         , LOAD_TS           as LOAD_TS          -- TIMESTAMP_NTZ
+
+         , 'SEED.ABC_Bank_SECURITY_INFO' as RECORD_SOURCE
+
+    FROM {{ source('seeds', 'ABC_Bank_SECURITY_INFO') }}
+ ),
+
+default_record as (
+    SELECT
+        '-1'      as SECURITY_CODE
+        , 'Missing' as SECURITY_NAME
+        , 'Missing' as SECTOR_NAME
+        , 'Missing' as INDUSTRY_NAME
+        , '-1'      as COUNTRY_CODE
+        , '-1'      as EXCHANGE_CODE
+        , '2020-01-01' as LOAD_TS_UTC
+        , 'Missing' as RECORD_SOURCE
+),
+
+with_default_record as(
+    SELECT * FROM src_data
+    UNION ALL
+    SELECT * FROM default_record
+),
+
+hashed as (
+    SELECT
+        concat_ws('|', SECURITY_CODE) as SECURITY_HKEY
+        , concat_ws('|', SECURITY_CODE, SECURITY_NAME, SECTOR_NAME,
+                         INDUSTRY_NAME, COUNTRY_CODE, EXCHANGE_CODE ) as SECURITY_HDIFF
+
+        , * EXCLUDE LOAD_TS
+        , LOAD_TS as LOAD_TS_UTC
+    FROM with_default_record
+)
+SELECT * FROM hashed
+```
+
+It is always important to think about what's included in the HDIFF key.
+
+We have the default records to help in case there is missing info or incomplete data. 
+
+### Saving history for the dimensional data
+
+Created `SNSH_ABC_BANK_SECURITY_INFO.sql` in the snapshots folder.
+
+```sql
+{% snapshot SNSH_ABC_BANK_SECURITY_INFO %}
+
+{{
+    config(
+      unique_key= 'SECURITY_HKEY',
+
+      strategy='check',
+      check_cols=['SECURITY_HDIFF'],
+    )
+}}
+
+select * from {{ ref('STG_ABC_BANK_SECURITY_INFO') }}
+
+{% endsnapshot %}
+```
+
+The invalidate_hard_deletes parameter is removed, as generally for dimensions, it is more important to keep old entries than to capture deletions, as these old entries are referenced by old facts. Eventually, we might be interested to know whether an entry is still active or not. In that case, we would enable the parameter to track hard deletions, but then keep both the active and deleted rows in the REF model derived from the snapshot.
+
+### Building the REF layer with the dimensional data
+
+Created `REF_ABC_BANK_SECURITY_INFO.sql` in the models/refined folder
+
+```sql
+WITH
+    current_from_snapshot as (
+        SELECT * EXCLUDE (DBT_SCD_ID, DBT_UPDATED_AT, DBT_VALID_FROM, DBT_VALID_TO)
+        FROM {{ ref('SNSH_ABC_BANK_SECURITY_INFO') }}
+WHERE DBT_VALID_TO is null
+    )
+SELECT *
+FROM current_from_snapshot
+```
+
+### Adding the dimensional data to the data mart
+
+Created `DIM_SECURITY.sql` in the models/marts/portfolio folder
+
+```sql
+SELECT * FROM {{ ref('REF_ABC_BANK_SECURITY_INFO') }}
+```
+
+Running dbt build after all this:
+
+<img width="948" alt="image" src="https://github.com/user-attachments/assets/eccc0b37-de7f-4258-bdec-267b490859e0" />
+
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 010a0d049ecea8a2646599e1c5059afc0ac6d38c Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 18 Dec 2024 00:24:06 +0900
Subject: [PATCH 543/638] Update 2024-12-17-day-351.md

---
 _posts/2024-12-17-day-351.md | 62 +++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 30 deletions(-)

diff --git a/_posts/2024-12-17-day-351.md b/_posts/2024-12-17-day-351.md
index 40b9899df..8556c4ac3 100644
--- a/_posts/2024-12-17-day-351.md
+++ b/_posts/2024-12-17-day-351.md
@@ -11,6 +11,8 @@ A quick summary of today:
 
 ---
 
+Some of the jinja might look wrong, but I had to put it like that because of some weird formatting from my blog.
+
 ## [Data Engineering with dbt](https://learning.oreilly.com/library/view/data-engineering-with/9781803246284/) - Chapter 1/2 - The Basics of SQL to Transform Data/Setting up Snowflake and dbt cloud
 
 The book starts by introducing SQL, its usage and some key words like table, queries, ddl, etc. 
@@ -21,7 +23,7 @@ Then I had to create a snowflake account so that I can follow the book examples.
 
 To set up users, a warehouse and a db I ran the below queries:
 
-```sql
+```
 /**
  * Chapter 1 - Section 3: Set up initial users, roles and database in Snowflake
  */
@@ -96,7 +98,7 @@ Some examples:
 
 **Finally - some window functions**
 
-```sql
+```
 SELECT
     O_ORDERKEY,
     O_CUSTKEY,
@@ -446,7 +448,7 @@ SELECT
     *, 
     POSITION_VALUE - COST_BASE as UNREALIZED_PROFIT,
     ROUND(UNREALIZED_PROFIT / COST_BASE, 5)*100 as UNREALIZED_PROFIT_PCT
-FROM {{ source('abc_bank', 'ABC_BANK_POSITION') }}
+FROM { source('abc_bank', 'ABC_BANK_POSITION') }
 ```
 
 And it appeared in snowflake after `dbt run`:
@@ -630,18 +632,18 @@ src_data as (
 
          , 'SOURCE_DATA.ABC_BANK_POSITION' as RECORD_SOURCE
 
-    FROM {{ source('abc_bank', 'ABC_BANK_POSITION') }}
+    FROM { source('abc_bank', 'ABC_BANK_POSITION') }}
 ),
 
 hashed as (
     SELECT
-          {{ dbt_utils.surrogate_key([ 'ACCOUNT_CODE', 'SECURITY_CODE']) }} as POSITION_HKEY
-        , {{ dbt_utils.surrogate_key([ 'ACCOUNT_CODE', 'SECURITY_CODE',
+          { dbt_utils.surrogate_key([ 'ACCOUNT_CODE', 'SECURITY_CODE']) }} as POSITION_HKEY
+        , { dbt_utils.surrogate_key([ 'ACCOUNT_CODE', 'SECURITY_CODE',
                 'SECURITY_NAME', 'EXCHANGE_CODE', 'REPORT_DATE',
                 'QUANTITY', 'COST_BASE', 'POSITION_VALUE', 'CURRENCY_CODE'
                 ]) }} as POSITION_HDIFF
         , *
-        , '{{ run_started_at }}'::timestamp as LOAD_TS_UTC
+        , '{ run_started_at }}'::timestamp as LOAD_TS_UTC
     FROM src_data
 )
 
@@ -710,10 +712,10 @@ _It is important to note that a snapshot table is a global object, shared by all
 The general layout of a snapshot definition:
 
 ```
-{% snapshot snapshot_name %}
-{{ config(…) }}
-select … from {{ source / ref }}
-{% endsnapshot %}
+% snapshot snapshot_name %}
+{config(…) }
+select … from { source / ref }
+{% endsnapshot %
 ```
 
 Central to the operations of a snapshot are the following concepts:
@@ -760,7 +762,7 @@ Here are some snapshot best practices:
 
 **Multiple ways to take into use the snapshots**
 
-* a snapshot as a first step: Taking a snapshot as the first thing in the flow is what dbt suggests, but it means either preserving the source names by using the trivial `SELECT * FROM {{ source(…) }}` query or writing a query that is not simple to preview/debug
+* a snapshot as a first step: Taking a snapshot as the first thing in the flow is what dbt suggests, but it means either preserving the source names by using the trivial `SELECT * FROM { source(…) }}` query or writing a query that is not simple to preview/debug
 * a snapshot after the STG model: Taking the snapshot out of an STG model solves the maintainability issue, but it is a bit clumsy if you use a normal model, as you need first to run only the STG models for the snapshots, take the snapshot, and then run all the models except the snapshot ones. The clumsiness is removed if you use ephemereal models for the STG models, as you do not need to run them before
 
 **Creating a snapshot in the project**
@@ -768,8 +770,8 @@ Here are some snapshot best practices:
 The below is put in the snapshots folder in an sql file
 
 ```sql
-{% snapshot SNSH_ABC_BANK_POSITION %}
-{{
+{% snapshot SNSH_ABC_BANK_POSITION 
+{
     config(
       unique_key= 'POSITION_HKEY',
       strategy='check',
@@ -777,15 +779,15 @@ The below is put in the snapshots folder in an sql file
       invalidate_hard_deletes=True,
     )
 }}
-select * from {{ ref('STG_ABC_BANK_POSITION') }}
-{% endsnapshot %}
+select * from { ref('STG_ABC_BANK_POSITION') }
+{% endsnapshot 
 ```
 
 We use the POSITION_HKEY which was created in the STG_ABC_BANK_POSITION. Here is the data lineage
 
 <img width="1099" alt="image" src="https://github.com/user-attachments/assets/39ebab13-b6c0-4bd4-a715-3f7ac86c9d88" />
 
-Also, before executing `dbt snapshot` I dropped the existing STG_ABC_BANK_POSITION view as otherwise, in the future, people on the project might think it’s still connected to the STG model and even use it. And I added `{{ config(materialized='ephemeral') }}` to the top of the model file to fix the issue of not being able to preview or test that query in isolation - we change the model to ephermal materialization and use it to feed the content of the STG view to the snapshot. 
+Also, before executing `dbt snapshot` I dropped the existing STG_ABC_BANK_POSITION view as otherwise, in the future, people on the project might think it’s still connected to the STG model and even use it. And I added `{config(materialized='ephemeral') }` to the top of the model file to fix the issue of not being able to preview or test that query in isolation - we change the model to ephermal materialization and use it to feed the content of the STG view to the snapshot. 
 
 After running `dbt snapshot`, in snowflake we can see:
 
@@ -799,12 +801,12 @@ The REF_POSITION_ABC_BANK model that we wrote before is still reading data from
 
 This is the current one:
 
-```sql
+```
 SELECT
     *, 
     POSITION_VALUE - COST_BASE as UNREALIZED_PROFIT,
     ROUND(UNREALIZED_PROFIT / COST_BASE, 5) * 100 as UNREALIZED_PROFIT_PCT
-FROM {{ ref('STG_ABC_BANK_POSITION') }}
+FROM { ref('STG_ABC_BANK_POSITION') }
 ```
 
 If we just change the ref to the snapshot, we would read everything out of the history – that is, all the versions that you encountered in the history of the snapshot which is not what we want.
@@ -819,7 +821,7 @@ It should be:
 WITH
 current_from_snapshot as (
     SELECT *
-    FROM {{ ref('SNSH_ABC_BANK_POSITION') }}
+    FROM { ref('SNSH_ABC_BANK_POSITION') }
     WHERE DBT_VALID_TO is null
 )
 SELECT
@@ -871,7 +873,7 @@ seeds:
   portfolio_tracking:
     ABC_Bank_SECURITY_INFO:
       +post-hook:
-        - "UPDATE {{ this }} SET LOAD_TS = '{{ run_started_at }}' WHERE LOAD_TS is null"
+        - "UPDATE { this } SET LOAD_TS = '{ run_started_at }' WHERE LOAD_TS is null"
       +column_types:
         LOAD_TS: TIMESTAMP
 ```
@@ -891,7 +893,7 @@ And load_ts becomes the timestamp when the dbt seed is run
 **Creating an STG model for the security dimension**
 
 ```sql
-{{ config(materialized='ephemeral') }}
+{ config(materialized='ephemeral') }
 
 WITH
 src_data as (
@@ -906,7 +908,7 @@ src_data as (
 
          , 'SEED.ABC_Bank_SECURITY_INFO' as RECORD_SOURCE
 
-    FROM {{ source('seeds', 'ABC_Bank_SECURITY_INFO') }}
+    FROM { source('seeds', 'ABC_Bank_SECURITY_INFO') }
  ),
 
 default_record as (
@@ -949,20 +951,20 @@ We have the default records to help in case there is missing info or incomplete
 Created `SNSH_ABC_BANK_SECURITY_INFO.sql` in the snapshots folder.
 
 ```sql
-{% snapshot SNSH_ABC_BANK_SECURITY_INFO %}
+ snapshot SNSH_ABC_BANK_SECURITY_INFO %}
 
-{{
+{
     config(
       unique_key= 'SECURITY_HKEY',
 
       strategy='check',
       check_cols=['SECURITY_HDIFF'],
     )
-}}
+}
 
-select * from {{ ref('STG_ABC_BANK_SECURITY_INFO') }}
+select * from { ref('STG_ABC_BANK_SECURITY_INFO') }
 
-{% endsnapshot %}
+% endsnapshot %
 ```
 
 The invalidate_hard_deletes parameter is removed, as generally for dimensions, it is more important to keep old entries than to capture deletions, as these old entries are referenced by old facts. Eventually, we might be interested to know whether an entry is still active or not. In that case, we would enable the parameter to track hard deletions, but then keep both the active and deleted rows in the REF model derived from the snapshot.
@@ -975,7 +977,7 @@ Created `REF_ABC_BANK_SECURITY_INFO.sql` in the models/refined folder
 WITH
     current_from_snapshot as (
         SELECT * EXCLUDE (DBT_SCD_ID, DBT_UPDATED_AT, DBT_VALID_FROM, DBT_VALID_TO)
-        FROM {{ ref('SNSH_ABC_BANK_SECURITY_INFO') }}
+        FROM { ref('SNSH_ABC_BANK_SECURITY_INFO') }
 WHERE DBT_VALID_TO is null
     )
 SELECT *
@@ -987,7 +989,7 @@ FROM current_from_snapshot
 Created `DIM_SECURITY.sql` in the models/marts/portfolio folder
 
 ```sql
-SELECT * FROM {{ ref('REF_ABC_BANK_SECURITY_INFO') }}
+SELECT * FROM { ref('REF_ABC_BANK_SECURITY_INFO') }
 ```
 
 Running dbt build after all this:

From a1b51a6f45cdfcf95841853c06894504311f7e20 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 18 Dec 2024 00:26:37 +0900
Subject: [PATCH 544/638] Update 2024-12-17-day-351.md

---
 _posts/2024-12-17-day-351.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/_posts/2024-12-17-day-351.md b/_posts/2024-12-17-day-351.md
index 8556c4ac3..43f65ef58 100644
--- a/_posts/2024-12-17-day-351.md
+++ b/_posts/2024-12-17-day-351.md
@@ -712,10 +712,10 @@ _It is important to note that a snapshot table is a global object, shared by all
 The general layout of a snapshot definition:
 
 ```
-% snapshot snapshot_name %}
+% snapshot snapshot_name %
 {config(…) }
 select … from { source / ref }
-{% endsnapshot %
+{ endsnapshot %
 ```
 
 Central to the operations of a snapshot are the following concepts:
@@ -770,7 +770,7 @@ Here are some snapshot best practices:
 The below is put in the snapshots folder in an sql file
 
 ```sql
-{% snapshot SNSH_ABC_BANK_POSITION 
+{ snapshot SNSH_ABC_BANK_POSITION 
 {
     config(
       unique_key= 'POSITION_HKEY',
@@ -780,7 +780,7 @@ The below is put in the snapshots folder in an sql file
     )
 }}
 select * from { ref('STG_ABC_BANK_POSITION') }
-{% endsnapshot 
+{ endsnapshot 
 ```
 
 We use the POSITION_HKEY which was created in the STG_ABC_BANK_POSITION. Here is the data lineage

From 021c53441f684e7449089dfd80ed356803a95879 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 18 Dec 2024 21:00:50 +0900
Subject: [PATCH 545/638] Create 2024-12-18-day-352.md

---
 _posts/2024-12-18-day-352.md | 422 +++++++++++++++++++++++++++++++++++
 1 file changed, 422 insertions(+)
 create mode 100644 _posts/2024-12-18-day-352.md

diff --git a/_posts/2024-12-18-day-352.md b/_posts/2024-12-18-day-352.md
new file mode 100644
index 000000000..218daafd2
--- /dev/null
+++ b/_posts/2024-12-18-day-352.md
@@ -0,0 +1,422 @@
+---
+layout: post
+title: (Day 352) SCD type 2 exercises + some dbt tests
+categories: [data-eng,theory,applying-knowledge]
+---
+
+# Hello :) Today is Day 352!
+A quick summary of today:
+* did some of the SCD type 2 exercises in the DE with dbt book
+* learned more about dbt tests
+
+## DE with dbt - Chapter 7 exercises
+
+At the end of the chapter there are some optional exercises to add extra dimensions to the model ~ so I did. And it was great because I got to practice SCD type 2 by myself in dbt and had to think about whether I need scd type 2 for a dimension, what values should I track and need scd type 2 ~ 
+
+This is the data lineage at the end:
+
+![image](https://github.com/user-attachments/assets/23c6eb0f-ed5a-4e1e-a560-b7b7175747ae)
+
+![image](https://github.com/user-attachments/assets/acd9a68c-e399-4840-92f6-ece317832d0e)
+
+
+## DE with dbt - Chapter 8 - Delivering Consistency in Your Data
+
+### Keeping consistency by reusing code – macros
+
+Using macros, we can apply the DRY and KISS software engineering principles, reducing duplication of code, and making complex tasks simple, allowing general solutions such as patterns to be widely adopted, as the complexity can be hidden away from most coders.
+
+**How to write a macro**
+
+In the macro folder, we can write something like
+
+<img width="609" alt="image" src="https://github.com/user-attachments/assets/9c5685f1-fb34-4849-88d7-78ea3f22afb6" />
+
+The macro will be run before the query is sent to the database to produce the final text to be sent to the database by running the logic in the macro.
+
+**Refactoring the 'current' CTE into a macro**
+
+Currently it is:
+
+```sql
+WITH
+current_from_snapshot as (
+    SELECT *
+    FROM { ref('SNSH_ABC_BANK_POSITION') }
+    WHERE DBT_VALID_TO is null
+)
+SELECT
+    *
+    , POSITION_VALUE - COST_BASE as UNREALIZED_PROFIT
+    , ROUND(UNREALIZED_PROFIT / COST_BASE, 5)*100
+         as UNREALIZED_PROFIT_PCT
+FROM current_from_snapshot
+```
+
+To make this into a macro, in the macro folder, I created a `current_from_snapshot.sql` file
+
+<img width="456" alt="image" src="https://github.com/user-attachments/assets/fd8ee4a4-9deb-45bf-ad8e-32e4031551be" />
+
+And in the files where we use the current_from_snapshot CTE format (like in the above sql snippet), we replace it with:
+
+<img width="410" alt="image" src="https://github.com/user-attachments/assets/4a116c0f-fbe3-4dd4-a85e-62311cc3050d" />
+
+**Fixing data loaded from our CSV file**
+
+Because of formating, the report date in one of the tables is messed up (ignore the first 3 lines for now):
+
+<img width="893" alt="image" src="https://github.com/user-attachments/assets/21bbdb32-62b3-4903-a1a3-16002c51f8e5" />
+
+We can create a macro:
+
+<img width="477" alt="image" src="https://github.com/user-attachments/assets/9ea6e7ac-d2cf-4cf1-9b57-e0297b4a1c79" />
+
+And apply it to the column we want to fix:
+
+`{{to_21st_century_date('REPORT_DATE')}}       as REPORT_DATE`
+
+And then when we run dbt build on that table and its downstream, we can see it got fixed. Also in the snapshot we see 3 new rows where the REPORT_DATE is correct and also for the previous incorrect 3 REPORT_DATE values now we have a value in the DBT_VALID_TO because they were updated and the REPORT_DATE is one of the cols we keep track of.
+
+<img width="463" alt="image" src="https://github.com/user-attachments/assets/c033927d-1207-4742-b962-c4f441c161de" />
+
+### Building on the shoulders of giants – dbt packages
+
+**dbt_utils package**
+
+* `expression_is_true`: this is a model level test that asserts that the SQL predicate (and expression evaluating to true or false) that you provide is true for all the records in the table
+* `not_empty_string`: a column-level test that verifies that there are no empty strings ('') in a column, after eventually trimming the content of the column
+
+Example:
+
+```
+      - name: column_name
+        tests:
+          - not_null
+          - dbt_utils.not_empty_string
+```
+
+* `mutually_exclusive_ranges`: model-level macro that tests the rows create correct date ranges when partitioned by a column, allowing you to verify that ranges do not overlap, have or don’t have gaps, and that the range size is greater than zero
+* `get_column_values`: this macro loads the values from a specific column of a model as a Python list, without having to deal with Agate tables
+* `get_query_results_as_dict`: this macro loads the results from a SQL query as a Python dictionary so that you do not have to deal with Agate tables
+* `date_spine`: generates the SQL code to generate a continuous sequence of dates in an interval, according to a desired granularity (day, month, etc)
+* `union_relations`:  allows you to unite two or more relations (tables or views), aligning fields with the same name and adding null columns for the missing ones, making it simple to UNION tables that are similar but do not have the same columns in the same order
+
+**Codegen package**
+
+* the following code will generate the YAML for a source file to import the relations from a schema in the selected database: `{{ codegen.generate_source('<schema>',database_name = '<database>') }}`
+* the following code can be used to generate the YAML to describe and test a model: `{{ codegen.generate_model_yaml(model_name='<model>') }}`
+
+**dbt_expectations package**
+
+This package extends the testing abilities of dbt by defining many test macros, inspired by the Great Expectations package that tests Python programs
+
+The available tests go from simple ones such as `expect_table_row_count_to_equal_other_table` to more complex ones that can test the values of your data, such as `expect_column_values_to_be_within_n_moving_stdevs`
+
+**The dbt_external_tables package**
+
+This package allows you to create external tables as sources in a database-independent way, by configuring the tables that you want in YAML
+
+**The metrics package**
+
+This dbt-lab-created package uses the metrics definition introduced in dbt V1.0.0 to produce data tables, providing data according to the metrics definition.
+
+This package allows you to materialize the metrics that you have defined and also take advantage of secondary calculations, such as period over period, period to date, rolling, and so on
+
+**dbt_vault package**
+
+This is the first dbt package to help build a data platform according to the Data Vault 2.0 (DV2) principles and is produced by Data Vault
+
+**datavault4dbt package**
+
+This is a package to build a data platform according to the DV2 principles
+
+**dbt Model Usage package**
+
+This dbt package provides tests to let you know whether your models are still relevant to your users. These tests scan your database’s query logs to check whether users are still selecting from the tables that dbt produces. Test failures can let you know when it might be time to retire unused models.
+
+**dbt_profiler package**
+
+This package provides macros to profile the content of your relations, providing many measures for each column, such as the proportion of not null, min, max, and the number of distinct values.
+
+**dbt_artifacts package**
+
+This dbt package builds a data mart to analyze the executions and usage of dbt components, such as models, tests, and exposures, ultimately describing the dbt project where you install the package
+
+---
+
+## DE with dbt - Chapter 9 - Delivering Reliability in Your Data
+
+### Testing to provide reliability
+
+_Tests are going to be your sleep-well-at-night superpower._
+
+**Types of tests**
+
+* singular test: used in a single case; generally only a few of these tests are used to cover the rare cases not covered by the available generic tests
+* generic tests: writing or using a parametrized SQL query and passing the required arguments; you will get most tests from packages, as in general, you will be testing over and over the same kind of assumptions that other people also test in other projects
+
+**Defining a generic test**
+
+To define a generic test, or to override one of the out-of-the-box ones, you can write a test Jinja block, containing the SQL for your test in a SQL file.
+
+Test blocks are very similar to macros, as generic tests used to be macros with a name starting with test_, and they still work pretty much the same as macros – defining a function that takes arguments and whose result is evaluated as a test result.
+
+The SQL for generic tests must be written to return the rows that fail the test, using the same logic like singular tests
+
+Generic tests are applied using YAML files and must always define at least two arguments, model and column_name, which are automatically used to pass to each test invocation the reference to the resource and, eventually, the name of the column on which the test is applied in the YAML file.
+
+Since dbt v1.0.0, we can place these generic tests in either the macros folder or tests/generic
+
+This is a generic test pattern to follow:
+
+<img width="797" alt="image" src="https://github.com/user-attachments/assets/f1956bcb-a934-4072-9be1-24223f0ab240" />
+
+**Writing a generic test**
+
+In the project, we have hash values as keys and to check when the content of a row is changed. The probability of having hash collision – that is, two different sets of values producing the same hash – is really minuscule, and it is very safe to use hashes for such purposes.
+
+We will write a generic test that allows us to check whether we get unlucky and have a collision, so we can manage it and re-hash the table to avoid missing even a single row of data.
+
+In the macros/tests folder, I created a `no_hash_collisions.sql` file with:
+
+<img width="486" alt="image" src="https://github.com/user-attachments/assets/f8fae889-0e1d-43a8-8fbf-cfeb0488d365" />
+
+Then in the snapshots folder, I create a snapshots.yml file for the tests:
+
+```
+version: 2
+snapshots:
+  - name: SNSH_ABC_BANK_POSITION
+    columns:
+      - name: POSITION_HKEY
+        tests:
+          - no_hash_collisions:
+              hashed_fields: "{{ as_sql_list(
+                 ['ACCOUNT_CODE', 'SECURITY_CODE',
+                  'SECURITY_NAME', 'EXCHANGE_CODE',
+                  'REPORT_DATE', 'QUANTITY', 'COST_BASE',
+                  'POSITION_VALUE', 'CURRENCY_CODE'
+                 ] ) }}"
+```
+
+(in the `hashed_fields` param, these are the fields we use for creating the hash key in the model itself so we pass them all here)
+
+The model and column_name parameters are managed automatically by dbt, so we do not need to pass them to the test, but we need to provide a value for any other parameter of the generic test that does not have a default value, such as hashed_fields
+
+Finally, we can run the test with `dbt test -s SNSH_ABC_BANK_POSITION`
+
+**Test configuration patter in yml**
+
+```
+      - <test_name>:
+          <argument_name>: <argument_value>
+          config:
+            where: <string>
+            severity: error | warn
+            error_if: <string>
+            warn_if: <string>
+            fail_calc: <string>
+            store_failures: true | false
+            limit: <integer>
+```
+
+### Testing the right things in the right places
+
+**What do we test?**
+
+* The Primary Key (PK) and the Business Key (BK): Getting the PK/BK right is essential to be able to identify an entity and store their history of changes. They enable us to join facts and dimensions, and the BK is essential to be able to identify the same entity across different systems
+* Foreign Keys (FKs): FKs are the essential element of relational systems, as they store the relations between tables and entities by pointing to the PK/BK of another table that completes the information. In the case of a many-to-many relationship, the FK of each side is stored in the mapping table that defines the relation, instead of being directly inside a table
+* Business rules (BRs): BRs come in many flavors and are the spice that gives most of the value to our data platform. It is impossible to make a complete list of BR types, as they can range from calculations to ways of connecting entities to ways of formatting data
+* Measures: Measures are numeric types of data that can be used for calculations. Some measures are extremely important, as they are needed for calculations, and their absence can make data almost useless, such as a sales order without the amount sold
+* Categories: Categories are textual data that is not purely descriptive, such as a name or any free text, but they are used to group or somehow organize data, such as a type of product, the industry of a customer, or any value that our users use to slice reports, but is not a full fledged dimension
+* Descriptive data: Everything else can pretty much fall under the descriptive data category, which contains things such as names, freeform text, and generally, all kinds of textual data that does not fall into the other category types
+
+**Where to test what?**
+
+The golden rules are :
+
+* test your expectations about inputs on sources or as soon as possible
+* test your expectations about outputs when you deliver them
+* in the intermediate transformations of the ELT pipelines, test the important things that might create a lot of problems or that you do not totally trust to always work as planned, such as BRs or complex joins
+
+Tests on the sources (input layer): 
+
+* test PKs and BKs to be not null and unique
+* test FKs to check that mandatory one-to-many relationships are not null
+* test relationships to check that they don’t have an orphan FK
+* test the source freshness
+
+Tests on the storage layer – the STG and HIST models:
+
+* test that the PK and BK (that you could not test on sources) are not null and unique
+* test that HKEY and HDIFF have no hash collision
+* create all the tests that you would have done in the source layer but that were impractical because they needed some data transformation
+
+Tests on the refined layer: 
+
+* test the results of the BRs that calculate values
+* test the PK/FK after risky or complex joins (to implement the BRs)
+
+Tests on the delivery layer:
+
+* test that the dimensions are well-formed
+* test that facts are well-formed
+* test the values in facts and dimensions
+* test that well-known expected output is produced correctly
+* run statistical, quantitative tests to ensure that data is flowing and in a reasonable range
+
+### Testing our models to ensure good quality
+
+**Testing the uniqueness of composite keys**
+
+### Exercise for myself
+
+This is not in the book - but I will make an exercise for meself to add tests to all models because at the moment only 2 have tests on them. 
+
+---
+
+## DE with dbt - Chapter 10 - Agile Development
+
+### Agile development and collaboration
+
+Manifesto for Agile Software Development:
+
+```
+We are uncovering better ways of developing software
+
+by doing it and helping others do it.
+
+Through this work we have come to value:
+
+Individuals and interactions over processes and tools
+Working software over comprehensive documentation
+Customer collaboration over contract negotiation
+Responding to change over following a plan
+That is, while there is value in the items on the right,
+
+we value the items on the left more.
+```
+
+### Applying agile to data engineering
+
+* agile is not improvising. Having a plan is important, as it is the understanding that any plan is not set in stone, and it will keep changing
+* pick a direction in collaboration with the people who know the domain (aka business) and the ones who know the technology
+* an important corollary to the previous point is to improve the detail of your goal while you go
+* a general goal is to keep your plan as flexible as possible, taking decisions (especially the ones that are difficult to change) at the last responsible moment so that the decision can be taken using as much knowledge and experience as possible
+* verify the biggest questions at the beginning so that the risk of a big change of direction decreases as you move forward
+* try to cut the work into small work increments, often referred to as a story, that can be completed with little risk of them having to stop
+* don’t try boiling the ocean or getting lost in too many different things. Try to focus the team on the most important topics of the moment so that they can collaborate on the solution
+* start completing and stop starting – focus on closing open work before you start more work
+* agility is not only about flexibility but also about avoiding waste and removing things that are outdated or of little or uncertain value
+
+### Building reports in an agile way
+
+**S1 – designing a light data model for the data mart**
+
+* keep the model simple
+
+**S2 – designing a light data model for the REF layer**
+
+* define REF model from sources
+* dimension enrichment
+* connecting to a fact a snowflaked dimension
+* the application of master data
+* definition of the composite fact
+
+**S3.x – developing with dbt models the pipeline for the XYZ table**
+
+Once the S1 and S2 stories have been completed or are at least in the advanced stage of completion, you will have a list of tables that you need to build in the refined and delivery layers.
+
+**S4 – an acceptance test of the data produced in the data mart**
+
+Two approaches:
+
+* automated acceptance/integration tests
+* manual acceptance test
+
+**S5 – development and verification of the report in the BI application**
+
+This is the final story, where you will use the data mart verified in S4 to build the report using the BI application.
+
+The starting point is the data model for the data mart designed in the S1 story, which was developed with the BI tool needs in mind and will, therefore, provide the correct model for your BI tool, whether it is a star schema or a wide table.
+
+---
+
+## DE with dbt - Chapter 11 - Team Collaboration
+
+### Core collaboration practices
+
+* common code ownership
+* clean code and technical excellence
+* trust, but challenge
+* an open door policy
+* pairing
+* code walkthroughs
+* PRs and code reviews
+
+### Working with Git in dbt Cloud
+
+* using the cloud's git functionality
+* proper branch naming conventions
+* adopting frequent releases
+
+---
+
+## DE with dbt - Chapter 12 - Deployment, Execution, and Documentation Automation
+
+### Creating our deployment automation
+
+This part was to create a dev (QA) and prod environments in the 'Deploy' page of dbt cloud and execute jobs on either ~ 
+
+<img width="1342" alt="image" src="https://github.com/user-attachments/assets/97b9c96f-b3b8-469f-90f1-1b41631b83cd" />
+
+I needed to run dbt seed because one of the dimensions comes directly from a seed csv which does not exist and I was getting an error when doing only dbt build. 
+
+So I made the job run like this:
+
+<img width="1321" alt="image" src="https://github.com/user-attachments/assets/baa2577e-dd96-4693-a2e0-05074d28e5f5" />
+
+Everything in the staging folder is materialised as a view, so I need to load the csv directly so that downstream tasks can run. Nice little error that I faced only because I did not do SCD type 2 on all dimensions. 🥳
+
+### Advanced automation – hooks and run-operations
+
+**Hooks**
+
+The available hooks are as follows:
+
+* pre-hook: the SQL will be run before building the dbt node it is attached to
+* post-hook: the SQL will be run after building the dbt node it is attached to
+* on-run-start: the SQL will be run at the beginning of the dbt command
+* on-run-end: the SQL will be run at the end of the dbt command
+
+While all hooks can access the context available in the model, the on-run-end hook has some extra context variables:
+
+* the schemas variable contains the list of schemas that contain models that have been built by dbt in the current run
+* the database_schemas variable is like the schemas variable but contains tuples (database, schema) in place of schema names
+* the results variable contains a list with a result object for each node that has been built by dbt in the current run. This corresponds to the data that will populate the result’s JSON artifact
+
+**Run-operations**
+
+Allows to activate a macro from the command line or as part of a job
+
+Some examples:
+
+* data ingestion: using macros to configure external tables, or to ingest data from files residing in data lakes
+* migrations: perform the evolutionary changes to your database objects that are not handled by dbt, such as dropping unused tables or managing individual fields
+* post-run operations: once transformations are done, there are extra things that we might want to do, such as managing grants, writing audit information, profiling tables, producing extra documentation, and so on
+
+**Table migrations**
+
+Dbt manages most of the DDL and DML commands needed to create and maintain the schemata, tables, and views, but in some cases we might want to do things that dbt does not do for us
+
+The most common things:
+
+* deleting tables and views that are not in use anymore
+* evolving incremental tables
+* performing bulk operations on data
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From a9c06bc5ff5b3985775e94cab990381be055ba10 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 19 Dec 2024 22:57:17 +0900
Subject: [PATCH 546/638] Create 2024-12-19-day-353.md

---
 _posts/2024-12-19-day-353.md | 372 +++++++++++++++++++++++++++++++++++
 1 file changed, 372 insertions(+)
 create mode 100644 _posts/2024-12-19-day-353.md

diff --git a/_posts/2024-12-19-day-353.md b/_posts/2024-12-19-day-353.md
new file mode 100644
index 000000000..370fc307a
--- /dev/null
+++ b/_posts/2024-12-19-day-353.md
@@ -0,0 +1,372 @@
+---
+layout: post
+title: (Day 353) Started reading - The AI Engineer's Guide to Surviving the EU AI Act
+categories: [data-eng,theory,mlops]
+---
+
+# Hello :) Today is Day 353!
+A quick summary of today:
+* started reading a [book about EU's AI act](https://learning.oreilly.com/library/view/the-ai-engineers/9781098172480/)
+
+## The AI Engineer's Guide to Surviving the EU AI Act - Chapter 1 - Understanding the AI Regulations
+
+_As people, organizations, and the public sector increasingly rely on AI to drive decision-making, the technology must be trustworthy. The EU AI Act aims to provide a legal framework for developing, deploying, and using AI technologies within the EU, emphasizing safety, transparency, and ethical considerations. The EU AI Act is a regulatory framework for artificial intelligence that includes specific requirements for AI systems of different risk categories within the European Union_
+
+### The Motivation for the EU AI Act: Trustworthy AI
+
+As AI integrates into critical areas like human resources, finance, and medicine, ensuring trustworthiness in AI systems is essential. Predictive accuracy alone is insufficient. AI must adhere to principles like fairness, privacy, and accountability to be deemed trustworthy. Trustworthy AI, also known as responsible or ethical AI, encompasses technical requirements (like robustness, explainability, and transparency) and ethical requirements (like fairness and privacy).
+
+Grounded in lawfulness, ethics, and robustness, trustworthy AI ensures compliance with regulations, ethical inclusivity, and risk-aware design. The European Commission’s High-Level Expert Group on AI outlined seven requirements for trustworthy AI:
+
+1. Human agency and oversight
+2. Technical robustness and safety
+3. Privacy and data governance
+4. Transparency
+5. Diversity, non-discrimination and fairness
+6. Societal and environmental well-being
+7. Accountability
+
+![image](https://github.com/user-attachments/assets/58a85f37-b1fe-40ef-8bcf-5ba38906c27d)
+
+**Human Agency and Oversight**
+
+Human oversight ensures humans can monitor, evaluate, and intervene in AI systems. It involves governance processes, transparency, interpretability, and control mechanisms, allowing humans to understand and influence AI decisions, including overriding or stopping the system when necessary.
+
+![image](https://github.com/user-attachments/assets/2c0eae5e-14fa-48ac-960f-be174046f368)
+
+**Technical Robustness and Safety**
+
+AI systems need to be accurate, reliable, and reproducible. They need to have a fall-back plan if something does not work properly.
+
+The author distinguishes robustness on different levels of AI systems, namely data, algorithms, and underlying software systems
+
+In academia, robustness has two main aspects: 
+
+* non-adversarial robustness: refers to how well the model performs when it is given corrupted or altered inputs that may not match the original data distribution. It means that the model can handle unintentional image corruptions, distributional shifts, or changes in the data generating process while maintaining its performance
+* adversarial robustness: refers to the model’s ability to resist adversarial examples, which are intentionally crafted input perturbations designed to fool the model into making incorrect predictions
+
+We can evaluate the robustness based on certain metrics:
+
+* accuracy
+* error rate
+* sensitivity
+* specificity
+
+Or through robustness curves which show how the model’s performance changes as a function of parameters affecting data quality, such as noise level, missing values, outliers, data size, or feature selection
+
+**Privacy and Data Governance**
+
+_AI systems must ensure full respect for privacy and data protection. Since privacy is a fundamental right within the European Union, privacy is a prerequisite to building trustworthy AI systems_
+
+Key elements of data governance:
+
+![image](https://github.com/user-attachments/assets/9e51cf45-322e-423d-8d10-7044241323b0)
+
+Data governance is becoming more important and its ultimate goal is to enhance the trustworthiness of data. To ensure trust in data, three key aspects of data governance must be addressed: discoverability, security, and accountability of data.
+
+* discoverability: refers to the availability of the dataset’s metadata, data provenance (lineage), and domain entities glossary; data quality is also crucial for trust
+* data security and privacy: about protecting data and ensuring adherence to regulations such as GDPR
+* accountability: consider data as a product - a data unit that is valid within a business domain where the domain team has a clear responsibility for this data unit
+
+Example metrics to define and track for privacy and data governance include:
+
+* data encryption levels
+* access control
+* data retention and deletion policies
+* data product usage metric
+
+**Transparency**
+
+* explainability
+* data transparency
+* algorithmic transparency
+* governance transparency
+* communication transparency
+
+![image](https://github.com/user-attachments/assets/dd2d844e-f66c-4df9-8db3-2f9d65388047)
+
+**Diversity, Non-Discrimination and Fairness**
+
+AI systems must mitigate unfair bias. Addressing potential bias should happen at every stage of the AI system development. The EU AI Act ensures that every entity along the AI system’s value chain, from producer to deployer, is responsible for providing users with a fair and ethical experience
+
+There are several metrics used to evaluate fairness in ML systems:
+
+* demographic parity (measures the difference in the probability of receiving a positive outcome between different demographic groups)
+* equal opportunity (measures whether the model has an equal chance of correctly predicting positive outcomes for individuals from different groups who actually deserve a positive outcome)
+* equalised odds (ensures that the model not only has equal opportunity but also equal mistreatment across groups)
+* predictive parity (evaluates whether the precision of the model is consistent for all groups)
+* individual fairness (ensures that similar individuals are treated similarly by the model, based on a predefined similarity metric)
+* counterfactual fairness (evaluates whether changing an individual’s sensitive attribute would change the model’s prediction for that individual)
+
+![image](https://github.com/user-attachments/assets/edb8adf1-97a4-4ab1-8c61-5fa9c5e8dcf6)
+
+**Societal and Environmental Well-being**
+
+AI systems should benefit all human beings. The EU Act is concerned specifically with the societal and environmental aspects of trustworthy AI areas. 
+
+The primary environmental concerns associated with AI include energy consumption, carbon footprint, e-waste, and indirect environmental impact
+
+The amount of computing resources used to train deep learning models has increased 300,000x in six years from 2012 to 2018:
+
+![image](https://github.com/user-attachments/assets/2d24d3b6-1db4-4f6c-a619-0d0354407625)
+
+The proposed 'Green AI' approach considers efficiency as an additional evaluation criterion, together with the usual eval metrics. 'Green AI' efficiency is measured as the number of floating-point operations required to generate a result
+
+The rise of AI has led to job polarization, with high-skilled jobs increasing and low-skilled jobs facing obsolescence. This worsens income inequality and creates challenges for those without access to education and training. 
+
+Lastly, in addition to ethical concerns and job changes, AI systems can negatively impact society at large and democracy through misinformation and surveillance.
+
+**Accountability**
+
+AI accountability means establishing mechanisms for holding AI developers and users accountable for their systems’ impacts along the AI system’s complete development cycle. It implies that information about the AI system’s purpose, design, data, and processes is available to internal and external auditors
+
+Google has a responsible GenAI Toolkit that covers risk and mitigation techniques to address safety, privacy, fairness, and accountability:
+
+![image](https://github.com/user-attachments/assets/d98cccbf-3a9e-4734-8b4f-d7c973567c45)
+
+Several mechanisms should be implemented to create accountability. Here are a few:
+
+* clear responsibility guidelines and processes
+* transparency and explainability
+* human oversight and intervention
+* auditing and evaluation
+* redress and complaint mechanisms
+
+### EU AI Act in a Nutshell
+
+_The EU AI Act is about the adoption of human-centered and trustworthy artificial intelligence. The objective is to guarantee a high level of protection for health, safety, and fundamental rights as outlined in the Charter, including democracy, the rule of law, and environmental protection, against the damaging effects of AI systems in the European Union, all while fostering innovation_
+
+The subject of the EU AI Act is the following:
+
+1. “Harmonized rules for the placing on the market, the putting into service, and the use of AI systems in the Union;
+2. Prohibitions of certain AI practices;
+3. Specific requirements for high-risk AI systems and obligations for operators of such systems;
+4. Harmonized transparency rules for certain AI systems;
+5. Harmonized rules for the placing on the market of general-purpose AI models;
+6. Rules on market monitoring, market surveillance, governance and enforcement;
+7. Measures to support innovation, with a particular focus on SMEs, including startups.”
+
+<img width="1209" alt="image" src="https://github.com/user-attachments/assets/48f96800-fda5-4f00-a0ad-a92d64b384eb" />
+
+[Here](https://artificialintelligenceact.eu/ai-act-explorer/)
+
+**AI Definition**
+
+```
+AI system’ means a machine-based system that is
+
+* designed to operate with varying levels of autonomy and
+* that may exhibit adaptiveness after deployment, and
+* that, for explicit or implicit objectives, infers, from the input it receives,
+* how to generate outputs such as predictions, content, recommendations, or decisions
+* that can influence physical or virtual environments;
+```
+
+The following AI techniques and approaches refer to machine-based systems mentioned in the AI Act:
+
+* ML methods (such as supervised, unsupervised, semi-supervised machine learning)
+* DL methods
+* Reinforcement Learning
+* Logic- and Knowledge-based Methods (such as logic programming, expert systems, inference and deductive engines, reasoning engines)
+* Statistical approaches
+* Bayesian Methods
+* Search and Optimization Approaches
+
+The AI Act provides a definition of the 'general-purpose AI model':
+
+_AI model, including where such an AI model is trained with a large amount of data using self-supervision at scale, that displays significant generality and is capable of competently performing a wide range of distinct tasks regardless of the way the model is placed on the market and that can be integrated into a variety of downstream systems or applications, except AI models that are used for research, development or prototyping activities before they are placed on the market_
+
+**Key Players From Creation to Market Operation**
+
+* provider - a person or organization that develops an AI system or model, or has one developed, and puts it on the market or uses it, whether for payment or for free, under their own name or brand
+* importer - a natural or legal person in the EU who trades an AI system bearing the name or trademark of a person in a non-EU country
+* distributor - a person or company in the supply chain, other than the provider or the importer, that makes an AI system available on the Union market
+* authorized representative - a person or organization in the EU who has been given written permission by an AI system or general-purpose AI model provider to carry out the responsibilities and procedures outlined in this Regulation on their behalf
+* deployer - refers to a person or organization using an AI system for professional activities
+* user - persons impacted by an AI system
+
+![image](https://github.com/user-attachments/assets/833cf82c-0de9-4450-b84f-183d374c3e00)
+
+**Classification of the AI Systems by Risk Levels**
+
+* prohibited
+* high-risk
+* limited-risk
+* minimal risk
+
+![image](https://github.com/user-attachments/assets/50e272e8-03f5-4076-b734-557eee05627d)
+
+**Enforcement and Implementation of the EU AI Act**
+
+![image](https://github.com/user-attachments/assets/d4245682-97a9-478b-afb5-e6cfcae0199d)
+
+**The Full Picture of Compliance**
+
+| Steps                                             | Guiding Questions                                                                                                                                                       |
+|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1. Creating AI System Landscape and AI System Risk Classification | How many AI systems are in place and are intended to be put into production? What risk category do these AI systems belong to?                                        |
+| 2. Compliance Requirements Structuring for AI Systems and GPAI    | What requirements do we need to fulfill?                                                                                                                              |
+| 3. Compliance Operationalization                           | What processes, structures, engineering practices, and roles need to be established to comply with the AI Act?                                                       |
+| 4. Pre-market Compliance Verification                    | What has to be done before placing AI systems on the market and putting them into service? What are internal and external conformity assessments for compliance verification? How do we CE-mark our AI systems? Where should our AI system be registered (database)? |
+| 5. Post-market Continuous Compliance                     | What has to be done to ensure compliance after putting the AI system into service?                                                                                   |
+
+
+Knowing the risk category will shape all the requirements to be fulfilled to become compliant. It’s important to distinguish between obligations for AI systems and for usage General Purpose AI in the systems. For each of the groups, there are different articles in the AI Act with concrete requirements.
+
+**Penalties for EU AI Act Violation**
+
+| Violation Details                                                                                              | Penalties                                                                                                                                                           |
+|---------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Prohibited AI Practices (AI practices listed in Article 5, such as exploiting vulnerabilities, social scoring, real-time biometric identification in public spaces, etc.) | Administrative fines of up to €35 million or 7% of their total worldwide annual turnover for the preceding financial year, whichever is higher.                  |
+| High-Risk AI Systems (non-compliance with requirements for high-risk AI systems under Article 10).             | Administrative fines of up to €30 million or 6% of their total worldwide annual turnover, whichever is higher.                                                   |
+| Other non-compliance with other obligations under the AI Act, apart from Articles 5 and 10.                    | Administrative fines of up to €20 million or 4% of their total worldwide annual turnover, whichever is higher.                                                   |
+| Providing incorrect, incomplete, or misleading information to authorities.                                     | Administrative fines of up to €10 million or 2% of their total worldwide annual turnover, whichever is higher.                                                   |
+
+
+### Existing AI Regulations and Standards
+
+* [UNESCO AI Ethics Recommendations](https://unesdoc.unesco.org/ark:/48223/pf0000381137)
+* [U.S. Executive Order on Trustworthy AI](https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/)
+* [China Generative AI Service Law](https://www.ashurst.com/en/insights/new-generative-ai-measures-in-china/)
+* [NIST AI Risk Management Framework](https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf)
+* [IEEE P2863 - Recommended Practice for Organizational Governance of Artificial Intelligence](https://standards.ieee.org/ieee/2863/10142/)
+
+---
+
+## Chapter 2 - MLOps: A Proactive Compliance Catalyst
+
+### Designing the ML-powered Application
+
+An AI product:
+
+* delivers value
+* incorporates AI technology
+* interacts and adapts
+* operational integration
+
+**Backward Thinking For Designing AI Systems**
+
+It’s important to start an AI endeavor with the business goal in mind because 80-90% of AI initiatives never make it to production. The remaining few that make it is a result of, often, alignment to the business problem, solving a real user’s pain.
+
+'Backward Thinking' provides a strategic framework for scoping, planning, and executing ML projects. Starting with the end in mind helps maximize the chances of delivering a useful ML solution while avoiding common pitfalls around poorly defined objectives, irrelevant features, and misaligned architectures.
+
+Benefits:
+
+* clear scope upfront
+* business-aligned AI products
+* minimum viable data
+* narrowing solution space
+* iterative-incremental dev process
+
+The ML canvas is referenced (again, like in other books):
+
+![image](https://github.com/user-attachments/assets/aef1c74c-05e3-4084-992d-97a4f99bd88c)
+
+(this is a good chapter to reference when filling a ML canvas as it provides in-detail info about what can be good to fill and think about in each section of the canvas)
+
+**Value Proposition for AI Products**
+
+Suggested value proposition statement:
+
+```
+For [target customer]
+
+Who [statement of the need or opportunity]
+
+The [product name] is a [product category]
+
+That [statement of key benefit – compelling reason to buy or use]
+
+Unlike [primary competitive alternative]
+
+Our AI product [statement of primary differentiation]
+```
+
+![image](https://github.com/user-attachments/assets/4be6cf0f-3b3e-44ea-94fb-2edaa22f32fc)
+
+**Business Value Validation Through Metrics**
+
+* business metrics
+* AI product health metrics
+* AI technical metrics
+* system technical metrics
+
+The success of AI products is measured through a holistic system of metrics, encompassing business impacts, product health, and technical performance, as drafted in the Monitoring section of the ML Canvas.
+
+**Prediction Part of the AI System**
+
+* prediction task and heuristic benchmark:
+
+Define the task by specifying inputs, outputs, and problem type. Use heuristic baselines (e.g., rules or statistics) for initial benchmarks instead of simple ML models. For instance, anomaly detection can use the 99th percentile value as a baseline
+
+* workflow integration:
+
+Incorporate predictions into workflows, aligning decisions with predictions to achieve business goals. Use cases illustrate predictions leading to decisions (e.g. credit scoring predicts default probability, decision to reject loans if above a threshold)
+
+* prediction serving:
+
+Specify prediction frequency, volume, and latency needs. Choose between batch (e.g., periodic demand forecasting) or real-time serving (e.g., instant anomaly detection). Optimize serving for cost-efficiency and performance using techniques like REST APIs, Kubernetes, and quantization.
+
+* deployment decision:
+
+Simulate deployment to evaluate system performance on representative test cases. Use meaningful metrics and recent data for realistic assessment. Establish governance, audits, and safety measures to ensure reliable deployment. Combine offline and online evaluations to measure real-world performance and align with business objectives.
+
+**Training Part of the AI System**
+
+The training part of the AI system is responsible for all aspects required for building an ML model, namely data preparation, new data collection, feature engineering, and building and rebuilding ML models. The origin of ML models is data, and here are steps we need to consider
+
+* data landscape
+* data collection
+* model training frequency
+* feature engineering strategy
+
+**Requirements Engineering for ML and alignment with the EU AI Act Requirements**
+
+Using the ML canvas we end up with clearly defined, user-centric ML solutions and promotes cross-functional collaboration between data scientists, ML engineers, and businesses to build production-ready systems that deliver value. 
+
+Each of the ten building blocks of the canvas considers one particular aspect of the ML project, be it training or prediction. In total, we distinguish between six categories of requirements that we need to gather to build an AI product:
+
+* strategic business alignment
+* reusability of ML subsystems
+* retraining frequency of ML models
+* prediction serving mode
+* ML code change frequency
+* what kind of audit needs to be implemented?
+
+With the application of the EU AI Act, the ML Canvas becomes an additional layer of requirements. According to the EU AI Act, we should consider the following requirements for high-risk AI systems:
+
+* high-quality datasets
+* technical documentation and record-keeping
+* transparency and provision of information to users
+* robustness, resilience, accuracy, and security
+* human oversight
+* specific requirements for AI systems in critical infrastructure
+* respect for fundamental rights (privacy, data protection, non-discrimination, and consumer protection laws)
+* post-market monitoring
+
+### Structuring Machine Learning Development Process With [CRISP-ML(Q)](https://arxiv.org/pdf/2003.05155)
+
+![image](https://github.com/user-attachments/assets/e1bc25ab-1a7e-4e7f-aa02-c01287d7f71c)
+
+CRISP-ML(Q) defines six key phases in the ML development process:
+
+1. Business and Data Understanding (determine business objectives, identify data sources, collect initial data, describe data, explore data, verify data quality)
+2. Data Preparation (select, clean, construct, integrate, and format the data)
+3. Modeling (select modeling techniques, generate test design, build model, assess model)
+4. Evaluation (evaluate prediction results, review process, decide on the next steps)
+5. Deployment (plan deployment, plan monitoring and maintenance, produce final report, review project)
+6. Monitoring and Maintenance (monitoring the model, evaluating the model, retraining/updating the model as needed)
+
+Working with CRISP-ML(Q) will help incorporate the requirements of the AI Act for high-risk and limited-risk AI systems. Furthermore, for each task in an ML development phase, CRISP-ML(Q) defines requirements and constraints to identify risks, such as bias, overfitting, or lack of reproducibility, that can impact the success of the ML system. Moreover, CRISP-ML(Q) mandates that AI systems must be thoroughly tested prior to deployment to ensure they perform consistently for their intended purpose and comply with requirements. Importantly, CRISP-ML(Q) requires that you document the entire ML development process, including the risk management measures.
+
+### Understanding MLOps
+
+![image](https://github.com/user-attachments/assets/05e8b195-ca53-4218-89a3-192dfb595dbb)
+
+~ need to finish this chapter tomorrow. Definitely an interesting book so far.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 26227d477b6c606076bd6fa800efcf8e3f2d6844 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 19 Dec 2024 23:07:26 +0900
Subject: [PATCH 547/638] Update 2024-12-19-day-353.md

---
 _posts/2024-12-19-day-353.md | 64 ++++++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 2 deletions(-)

diff --git a/_posts/2024-12-19-day-353.md b/_posts/2024-12-19-day-353.md
index 370fc307a..c360b3c52 100644
--- a/_posts/2024-12-19-day-353.md
+++ b/_posts/2024-12-19-day-353.md
@@ -359,11 +359,71 @@ CRISP-ML(Q) defines six key phases in the ML development process:
 
 Working with CRISP-ML(Q) will help incorporate the requirements of the AI Act for high-risk and limited-risk AI systems. Furthermore, for each task in an ML development phase, CRISP-ML(Q) defines requirements and constraints to identify risks, such as bias, overfitting, or lack of reproducibility, that can impact the success of the ML system. Moreover, CRISP-ML(Q) mandates that AI systems must be thoroughly tested prior to deployment to ensure they perform consistently for their intended purpose and comply with requirements. Importantly, CRISP-ML(Q) requires that you document the entire ML development process, including the risk management measures.
 
-### Understanding MLOps
+### The MLOps graph
 
 ![image](https://github.com/user-attachments/assets/05e8b195-ca53-4218-89a3-192dfb595dbb)
 
-~ need to finish this chapter tomorrow. Definitely an interesting book so far.
+### Defining Technical Components With MLOps Stack Canvas
+
+![image](https://github.com/user-attachments/assets/e2ee1e30-b632-4605-81b0-f869f500cdbd)
+
+**Value Proposition**
+
+By answering the following questions, you can confidently articulate your value proposition:
+
+```
+What are we trying to do for the end-user(s)?
+
+What is the problem?
+
+Why is this an important problem?
+
+Who is our persona? (ML Engineer, Data Scientist, Operation/Business user)
+
+Who owns the models in production?
+```
+
+With the example value proposition, we can identify the key elements such as:
+
+* target customer: data science and ML teams
+* need: efficiently building and deploying ML models at scale
+* product name and category: MLOps Platform, an end-to-end machine learning platform
+* key benefit: rapidly develop and operationalize ML to drive business value
+* competition: other ML platforms that are complex and lack features such as observability and monitoring
+* differentiation: fully managed, intuitive, automated, with experiment tracking, AutoML, and seamless deployment to accelerate the full ML lifecycle
+
+**Data and Code Management**
+
+* data sources and versioning
+* data analysis and experiment management
+* feature store and workflows
+* reflect upon DevOps foundations
+* ML pipe orchestration
+
+**Model Management**
+
+* model registry and model versioning
+* model deployment
+* prediction serving
+* ML model, data, and system monitoring
+
+**Metadata Management**
+
+![image](https://github.com/user-attachments/assets/4ba0f52b-04db-4de2-b205-5196aa0cf79b)
+
+**Three Dilemmas of MLOps**
+
+* tooling: should we purchase, use existing open-source, or develop in-house tools for any of the MLOps components? What are the risks, trade-offs, and impacts of each decision?
+* platforms: should we standardize on a single MLOps platform or create a hybrid solution? What are the risks, trade-offs, and impacts of each decision?
+* skills: what is the cost associated with either hiring or training our own machine learning engineering talent?
+
+### Conclusion
+
+![image](https://github.com/user-attachments/assets/6a2de603-b85e-47cf-82d6-bd00ef8c78c9)
+
+The synergy between MLOps and CRISP-ML(Q). CRISP-ML(Q) defines a structured process, while MLOps provides technical/organizational best practices
+
+Definitely a great book so far.
 
 ---
 

From 0f976beeb89b7017b1820899e9a6a4998ea5fd95 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 20 Dec 2024 23:47:38 +0900
Subject: [PATCH 548/638] Create 2024-12-20-day-354.md

---
 _posts/2024-12-20-day-354.md | 421 +++++++++++++++++++++++++++++++++++
 1 file changed, 421 insertions(+)
 create mode 100644 _posts/2024-12-20-day-354.md

diff --git a/_posts/2024-12-20-day-354.md b/_posts/2024-12-20-day-354.md
new file mode 100644
index 000000000..482773465
--- /dev/null
+++ b/_posts/2024-12-20-day-354.md
@@ -0,0 +1,421 @@
+---
+layout: post
+title: (Day 354) Reading more about the EU AI act
+categories: [data-eng,theory,mlops]
+---
+
+# Hello :) Today is Day 354!
+A quick summary of today:
+* more of The AI Engineer's Guide to Surviving the EU AI Act book
+* PyFlink homework
+
+---
+
+## The AI Engineer's Guide to Surviving the EU AI Act - Chapter 3 - Data and AI Governance and MLOps
+
+### Why Data and AI Governance are crucial in the EU AI Act era
+
+Proper data governance for AI is crucial given some of the disasters that already happened (Amazon's AI recruiter discriminating against women, IBM Watson suggesting incorrect cancer treatments, etc). Comprehensive bias testing across diverse populations is essential, especially in healthcare and education applications. Transparency and explainability in AI systems are not just technical challenges but legal and ethical requirements. Ongoing monitoring and auditing of AI systems in production are necessary to catch and address issues early.
+
+To comply with the AI Act’s requirements, organizations have a few things to think about:
+
+* need robust data and AI governance frameworks in place to meet strict requirements related to data quality, documentation, transparency, human oversight, and risk management
+* effective data and AI governance are essential for identifying, assessing, and mitigating the risks associated with AI systems
+* strong data and AI governance promote responsible and ethical AI development by ensuring data quality, preventing bias, and maintaining human oversight, in line with the AI Act’s aim to foster the development of trustworthy AI that respects EU values and principles
+* well-designed governance enables organizations to innovate in AI while safeguarding privacy and other fundamental rights
+
+### Overview of Data Governance
+
+It includes a data management system informing about:
+
+* sourcing
+* storage
+* labeling
+* access control
+* quality
+
+**Data Governance Definition**
+
+_Data governance is a data management concept focused on ensuring high data quality throughout its lifecycle, implementing data controls supporting business objectives, and enabling organization-wide use. Key focus areas include data availability, usability, consistency, integrity, security, and standards compliance._
+
+**Data Engineering Lifecycle and Data Governance**
+
+![image](https://github.com/user-attachments/assets/c4657940-d159-40ee-b4ab-6acabb2f673b)
+
+**Data Products and Data Contracts**
+
+Data product canvas:
+
+![image](https://github.com/user-attachments/assets/849e1da8-58d3-4c9b-9bca-31829b7cb58f)
+
+Data products consist of 8 building blocks:
+
+1. Domain Name
+2. Data Product Name
+3. Consumer and Use Case
+4. Data Contract (more on this in a moment)
+5. Documentation of data sources
+6. Data Product Architecture, which includes the transformation pipelines for data lineage.
+7. Ubiquitous Language to get shared understanding of all definitions.
+8. Classification of the data product (source-aligned, aggregate, or consumer-aligned)
+
+Key components of data contracts:
+
+* data schema
+* quality standards
+* delivery schedule
+* SLAs
+* security, privacy, and governance
+* change management
+
+### Integrating Data Governance into MLOps
+
+Integrating data governance into the MLOps Stack Canvas ensures machine learning systems are compliant, trustworthy, and sustainable. By embedding governance across each component of the MLOps lifecycle, organizations can manage data responsibly and adhere to regulations, enhancing the overall value of ML projects. 
+
+**Value Proposition**  
+   Incorporate compliance and data governance goals, using metrics like policy compliance rate, regulatory adherence, and cost of poor data quality
+
+**Data Sources & Versioning**  
+   - Catalog data sources by sensitivity and regulatory needs
+   - Implement lineage tracking and sustainable documentation practices  
+   - Use tools like DVC, Apache Atlas, or Delta Lake for versioning and lineage
+
+**Data Analysis & Experimentation**  
+   - Enforce access controls and maintain audit trails  
+   - Ensure compliance with tools like MLflow or Weights & Biases
+
+**Feature Store & Engineering**  
+   - Secure and govern features with metadata and access policies  
+   - Track feature lineage and ensure compliance with tools like Feast or AWS SageMaker feature store
+
+**DevOps Foundations**  
+   - Embed governance checks in CI/CD pipelines  
+   - Automate data quality validations and anomaly detection
+
+**Continuous Integration & Deployment**  
+   - Monitor data drift and automate bias detection using tools like Evidently AI or Fiddler AI
+
+**Model Registry & Versioning**  
+   - Maintain governance metadata in model registries and establish versioning strategies aligned with data policies.
+
+**Model Deployment**  
+   - Deploy models with staged practices to ensure compliance, and monitor input/output data for privacy adherence.
+
+**Prediction Serving**  
+   - Protect sensitive data with encryption and implement data access controls 
+   - Monitor prediction quality, fairness, and compliance
+
+**System Monitoring**  
+    - Track metrics like data quality and model drift.  
+    - Use tools like Prometheus and Grafana for real-time alerts.
+
+**Metadata Management**  
+    - Centralize metadata to ensure lineage, provenance, and compliance tracking 
+    - Leverage platforms like Alation or Apache Atlas for comprehensive management
+
+### Overview of AI Governance
+
+**AI Governance Defined**
+
+_The objective of AI governance is to establish and maintain a framework that ensures AI systems are developed, deployed, and used responsibly, ethically, and in alignment with organizational goals and values. AI governance addresses AI algorithms, decision-making based on AI predictions, security, and data privacy. AI governance is an ecosystem in an enterprise. This ecosystem consists of people, processes, and policies._
+
+**Core Principles of AI Governance**
+
+The core principles for AI governance:
+
+* human-centred design and oversight
+* privacy and data security
+* safety, security, and dependability
+* ethical and responsible practices
+* transparency and explainability
+* accountability, redress, and remedy
+* fairness and inclusiveness
+* reproducability
+* robustness
+* innovation and competition
+
+**AI System Lifecycle and AI Governance**
+
+* business and data understanding
+* data preparation
+* model engineering
+* model evaluation
+* model deployment
+* model monitoring and maintenance
+
+### The integration of AI Governance and MLOps
+
+Incorporating AI governance into the MLOps Stack Canvas ensures ethical, legal, and responsible AI practices throughout the machine learning lifecycle. By addressing risks such as bias, fairness, privacy, and security, this approach enhances compliance, promotes cross-functional collaboration, and ensures higher-quality, trustworthy models.
+
+**Value Proposition**:  
+   - align AI projects with ethical principles and regulations (i.e. EU AI Act, GDPR)
+   - perform risk classifications and ethical impact assessments 
+   - identify affected stakeholders, including vulnerable groups
+
+**Data Sources & Versioning**:  
+   - ensure data sourcing follows ethical guidelines
+   - assess data quality and bias
+   - conduct data ethics reviews
+
+**Data Analysis & Experiment Management**:  
+   - use tools for transparency and accountability (e.g., fairness metrics, XAI techniques)
+   - document experiments for reproducibility and audit trails
+   - implement ethical reviews for design and results
+
+**Feature Store & Engineering**:  
+   - detect and mitigate feature bias  
+   - use privacy-preserving techniques  
+   - ensure feature documentation for governance
+
+**CI/CT/CD Pipelines**:  
+   - add ethical checkpoints and compliance automation  
+   - test for robustness and bias using tools like AI Fairness 360  
+   - incorporate human oversight and gradual rollouts
+
+**Model Registry & Deployment**:  
+   - document ethical metadata and approvals
+   - monitor models for ethical concerns post-deployment  
+   - enable user feedback and explainability
+
+**Monitoring & Metadata Management**:  
+   - continuously track fairness, bias, and data drift  
+   - maintain comprehensive governance documentation with traceability
+   - add regulatory compliance metadata and decision traceability
+
+### AI Compliance vs. AI Governance vs. AI Risk Management
+
+_AI Compliance focuses on adhering to legal, regulatory, and policy standards, ensuring ethical behavior, and promoting transparency through rigorous processes, metrics, and stakeholder engagement._
+
+_AI Governance establishes a framework for ethical and responsible AI development, emphasizing transparency, stakeholder engagement, and balancing innovation with ethical considerations._
+
+_AI Risk Management identifies and mitigates risks associated with AI systems, ensuring robust risk assessment, risk mitigation strategies, and promoting a culture of risk awareness and proactive risk management._
+
+**Well-known examples of AI risks**
+
+* algorithmic bias
+* privacy valuations and social manipulations
+* deepfakes
+* autonomous weapons
+* AI-enabled cyberattacks
+* lack of transparency
+* environmental impact
+* safety risks in critical systems
+* intellectual property issues
+* autonomous vehicle accidents
+
+The NIST AI Risk Management Framework (AI RMF) proposes four core functions that form the foundation for managing risks associated with artificial intelligence systems.
+
+![image](https://github.com/user-attachments/assets/74b3a847-5a70-400f-8ad3-fb5bdb2a6359)
+
+### Emerging Trends in Data and AI Governance
+
+* focus on AI ethics, trust, and enhanced data privacy
+* adoption of AI in data management and proactive compliance automation
+* focus on data lineage and provenance in AI
+* shift left data and AI governance
+* decentralized data governance
+
+---
+
+## Chapter 4 - Tailoring MLOps for Different Risk Levels
+
+### Creating AI System Landscape
+
+_To navigate the EU AI Act, first, one should get an overview of existing AI systems or, broadly speaking, AI use cases and assess whether these AI systems are underlying the legislation or if there are no action points. It is also helpful to distinguish whether you plan to put these AI use cases into production or are just experimentation or proof of concepts. Whenever an AI use case is developed as an internal research project, it can be safely excluded from the compliance obligation._
+
+**Inventory Your AI Systems**
+
+Start by cataloging all AI systems used or developed within your organization by following these steps:
+
+* identify AI applications across all departments
+* include both custom-developed and third-party AI solutions
+* document key details like department, purpose, data used, deployment status, and risk category.
+
+Include:
+
+* basic information (system name, department, purpose and functionality)
+* technical details (data sources, deployment status, vendor information)
+* risk assessment (risk level and justification)
+* additional notes (integration with other systems, responsible team/individual, date of last review)
+
+Example:
+
+![image](https://github.com/user-attachments/assets/18eb7ef0-54be-4aaa-8ab3-7366215ac9d4)
+
+### Application of the EU AI Act
+
+The process of determining the EU AI Act’s relevance to AI Systems and Risk Classification
+
+![image](https://github.com/user-attachments/assets/94619a14-6015-49f9-8005-ee0645d9c410)
+
+### Prohibited AI Practices
+
+The AI Act aims to protect fundamental rights, democracy, and the rule of law by banning AI applications that are incompatible with EU values and rights. In general, the AI Act prohibits AI systems from:
+
+* using subliminal, manipulative, or deceptive techniques to misinterpret behavior and damage informed decision-making, causing significant harm
+* exploiting vulnerabilities related to age, disability, or socio-economic circumstances to distort behavior, causing significant harm
+* utilizing biometric categorization systems to infer sensitive attributes such as race, political opinions, religious beliefs, and sexual orientation
+* conducting social scoring, i.e., evaluating individuals based on social behavior or personal traits, which can lead to discriminative treatment
+* creating or expanding facial recognition databases through untargeted scraping of facial images from the internet or CCTV footage
+* inferring emotions in workplaces or educational institutions, except for medical or safety reasons
+
+### Determining AI Act Obligation
+
+To determine the EU AI Act obligations, there are two fundamental steps in AI systems assessment:
+
+1. Determine if the AI Act applies to any AI Systems from the inventory above by answering the following questions: Do they meet the definition of “AI System”? Are they in the scope of the AI Act? Conduct a risk classification for each AI System. Are they High-Risk or not? Do Transparency obligations apply?
+2. From the inventory above, determine your organization’s role in relation to AI systems. Is your organization an AI system provider or deployer?
+
+**Framework for Classification of AI Systems by Risk Levels**
+
+High-Risk AI Systems. Key areas include:
+
+* biometric identification
+* critical infrastructure
+* education and employment
+* law enforcement and justice
+* migration and border control
+
+Limited-Risk AI Systems: these systems require transparency measures but are not as regulated as high-risk systems (Article 50). Examples include chatbots, emotion recognition systems, and deepfake generation tools. A questionnaire helps determine compliance
+
+Low-Risk AI Systems: pose minimal societal risks. Examples include spam filters, recommendation engines, and basic chatbots. No specific obligations are imposed under the Act unless their context or usage escalates risks.
+
+### Planning the Data Governance, AI Governance, and MLOps for the Compliance with the EU AI Act
+
+![image](https://github.com/user-attachments/assets/e4a03450-a455-4c6a-b7a9-5c9826a09166)
+
+Engineering and MLOps tasks for EU AI Act compliance grow more detailed and intensive as AI projects progress from ideation to maturity. Robust infrastructure, monitoring, and governance are essential to support compliance and system reliability. Key points include:  
+
+- MLOps and compliance integration: MLOps processes like data pipelines and model monitoring directly support EU AI Act requirements
+- data governance: increasingly critical in later stages for ensuring data quality and regulatory compliance
+- human oversight: evolving sophistication to align with EU AI Act mandates for high-risk AI systems
+- risk management: progresses from high-level to continuous, automated assessments
+- automation and scalability: emphasis grows as systems mature
+- MVP stage: focus shifts to audit readiness and conformity assessments for high-risk AI systems
+
+### Emerging Roles in Organizations for EU AI Act Compliance
+
+* the AI ethics specialist
+* the AI internal auditor
+* the legal AI compliance officer or AI regulatory affairs specialist
+* data and AI governance specialist
+* AI risk management specialist
+* AI security specialist
+
+---
+
+## Flink homework for Zach Wilson's YT camp
+
+I decided to do the homework today ~
+
+Task:
+
+Create a Flink job that sessionizes the input data by IP address and host; Use a 5 minute gap
+
+```python
+import os
+from pyflink.datastream import StreamExecutionEnvironment
+from pyflink.table import EnvironmentSettings, StreamTableEnvironment
+from pyflink.table.expressions import lit, col
+from pyflink.table.window import Session
+
+
+def create_aggregated_events_ip_host_sink_postgres(t_env):
+    table_name = 'processed_events_aggregated_hw'
+    sink_ddl = f"""
+        CREATE TABLE {table_name} (
+            event_hour TIMESTAMP(3),
+            ip VARCHAR,
+            host VARCHAR,
+            num_hits BIGINT
+        ) WITH (
+            'connector' = 'jdbc',
+            'url' = '{os.environ.get("POSTGRES_URL")}',
+            'table-name' = '{table_name}',
+            'username' = '{os.environ.get("POSTGRES_USER", "postgres")}',
+            'password' = '{os.environ.get("POSTGRES_PASSWORD", "postgres")}',
+            'driver' = 'org.postgresql.Driver'
+        );
+    """
+    t_env.execute_sql(sink_ddl)
+    return table_name
+
+def create_processed_events_source_kafka(t_env):
+    kafka_key = os.environ.get("KAFKA_WEB_TRAFFIC_KEY", "")
+    kafka_secret = os.environ.get("KAFKA_WEB_TRAFFIC_SECRET", "")
+    table_name = "process_events_kafka_hw"
+    pattern = "yyyy-MM-dd''T''HH:mm:ss.SSS''Z''"
+    sink_ddl = f"""
+        CREATE TABLE {table_name} (
+            url VARCHAR,
+            referrer VARCHAR,
+            user_agent VARCHAR,
+            host VARCHAR,
+            ip VARCHAR,
+            headers VARCHAR,
+            event_time VARCHAR,
+            window_timestamp AS TO_TIMESTAMP(event_time, '{pattern}'),
+            WATERMARK FOR window_timestamp AS window_timestamp - INTERVAL '15' SECOND
+        ) WITH (
+            'connector' = 'kafka',
+            'properties.bootstrap.servers' = '{os.environ.get('KAFKA_URL')}',
+            'topic' = '{os.environ.get('KAFKA_TOPIC')}',
+            'properties.group.id' = '{os.environ.get('KAFKA_GROUP')}',
+            'properties.security.protocol' = 'SASL_SSL',
+            'properties.sasl.mechanism' = 'PLAIN',
+            'properties.sasl.jaas.config' = 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.plain.PlainLoginModule required username=\"{kafka_key}\" password=\"{kafka_secret}\";',
+            'scan.startup.mode' = 'latest-offset',
+            'properties.auto.offset.reset' = 'latest',
+            'format' = 'json'
+        );
+    """
+    t_env.execute_sql(sink_ddl)
+    return table_name
+
+
+def log_aggregation():
+    # Set up the execution environment
+    env = StreamExecutionEnvironment.get_execution_environment()
+    env.enable_checkpointing(10)
+    env.set_parallelism(3)
+
+    # Set up the table environment
+    settings = EnvironmentSettings.new_instance().in_streaming_mode().build()
+    t_env = StreamTableEnvironment.create(env, environment_settings=settings)
+
+    try:
+        # Create Kafka table
+        source_table = create_processed_events_source_kafka(t_env)
+        print('running hw job')
+        aggregated_table = create_aggregated_events_ip_host_sink_postgres(t_env)
+        t_env.from_path(source_table) \
+            .window(Session.with_gap(lit(5).minutes).on(col("window_timestamp")).alias("w")) \
+            .group_by(
+                col("w"),
+                col("host"),
+                col("ip")
+            ) \
+            .select(
+                    col("w").start.alias("event_hour"),
+                    col("ip"),
+                    col("host"),
+                    col("host").count.alias("num_hits")
+            ) \
+            .execute_insert(aggregated_table).wait()
+
+    except Exception as e:
+        print("Writing records from Kafka to JDBC failed:", str(e))
+
+
+if __name__ == '__main__':
+    log_aggregation()
+```
+
+The thing is that the data every student gets will be different because we run the job during different times, but I think my code is the same as some other students from what I have seen talked about on the community discord server ~ We'll see when the submission opens 😆
+
+---
+
+
+That is all for today!
+
+See you tomorrow :)

From e2dad76f350ed67ac0f87c18265a3fd873afdbf2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 22 Dec 2024 00:26:56 +0900
Subject: [PATCH 549/638] Create 2024-12-21-day-355.md

---
 _posts/2024-12-21-day-355.md | 671 +++++++++++++++++++++++++++++++++++
 1 file changed, 671 insertions(+)
 create mode 100644 _posts/2024-12-21-day-355.md

diff --git a/_posts/2024-12-21-day-355.md b/_posts/2024-12-21-day-355.md
new file mode 100644
index 000000000..913cab8a6
--- /dev/null
+++ b/_posts/2024-12-21-day-355.md
@@ -0,0 +1,671 @@
+---
+layout: post
+title: (Day 355) Continued with the DE with dbt book
+categories: [data-eng,theory,mlops]
+---
+
+# Hello :) Today is Day 355!
+A quick summary of today:
+* finished the preview of The AI Engineer's Guide to Surviving the EU AI Act
+* learned about keeping history in dbt
+
+## The AI Engineer's Guide to Surviving the EU AI Act - Chapter 5 - MLOps for High-Risk AI Systems
+
+### AI Act Engineering
+
+The EU AI Act regulates AI systems in the EU to ensure trustworthiness by mitigating risks and safeguarding fundamental rights. Articles 9–15 provide requirements for high-risk AI systems but lack technical implementation guidance. To address this, a quality model for safety-critical AI systems has been proposed, integrating EU AI Act requirements with attributes like ethical integrity, human oversight, and fairness. 
+
+**Key insights**:  
+
+1. Mapping act requirements to quality attributes:  
+   - facilitates operationalization of regulatory requirements  
+   - integrates compliance into existing practices and aligns with international standards  
+
+2. EU AI act engineering:  
+   - a multidisciplinary field combining AI, engineering, data science, law, and ethics  
+   - focuses on risk assessment, transparency, monitoring, auditing, documentation, privacy, and human oversight  
+
+3. Integration with CRISP-ML(Q):  
+   - aligns AI Act requirements with the six CRISP-ML(Q) phases: business and data understanding, data prep, model engineering, model evaluation, model deployment, and monitoring
+   - ensures continuous compliance by addressing risks iteratively throughout the ML lifecycle  
+
+### MLOps and ML Engineering Practices for Achieving Compliance
+
+* establishing a risk management system for the entire lifecycle of the high-risk AI system
+* ensuring robust data governance by validating and testing datasets to guarantee their relevance, representativeness, and accuracy for the intended purpose
+* creating comprehensive technical documentation to demonstrate compliance and provide authorities with the necessary information for assessment
+* designing the hight-risk AI system for aytomatic event recording to identify national-level risks and substantial modifications throughout its lifecycle
+* providing detailed instructions for downstream deployers to ensure compliance
+* incorporating human oversight capabilities into the design of the high-risk AI system for downstream deployers
+* ensuring that the high-risk AI system meets appropriate levels of accuracy, robustness, and cybersecurity
+* establishing a quality management system to guarantee compliance
+
+**Risk management system**
+
+As per Article 9 of the EU AI Act
+
+![image](https://github.com/user-attachments/assets/6c07c1c8-c4e9-4d7a-b287-0a5f10c6e897)
+
+**Checklist for compliance with the EU AI act, article 9, risk management system:**
+
+```
+** Pre-development
+
+Define risk assessment methodologies and tools to identify critical risks based on potential harms (safety, privacy, bias).
+
+Create a risk management plan aligned with EU AI Act requirements.
+
+Set up a centralized risk register.
+
+** Business and Data Understanding
+
+Conduct initial risk identification workshop
+
+Document potential AI risks in the risk register
+
+Perform ethical impact assessment.
+
+Align project goals with organizational values and EU AI Act requirements.
+
+** Data Preparation
+
+Assess data quality, completeness, and representativeness.
+
+Ensure datasets are free from discriminatory bias.
+
+Identify and document data-related risks (e.g., bias, privacy). Assess risks related to data drift and fairness metrics.
+
+Implement data versioning and lineage tracking.
+
+Design and implement data validation tests. Implement data quality processes (e.g., completeness, correctness).
+
+** Modeling
+
+Conduct model vulnerability assessment.
+
+Validate the model architecture for explainability and robustness.
+
+Implement version control for model code and artifacts.
+
+Design and implement unit tests for model components.
+
+Document model architecture and design decisions.
+
+Implement integration tests for the full ML pipeline.
+
+Simulate edge cases and extreme scenarios during model design.
+
+** Evaluation
+
+Execute comprehensive test suites (unit, integration, system).
+
+Perform adversarial testing and document results.
+
+Evaluate performance with business, ethical goals, and the intended purpose of the system (value alignment metrics, e.g., fairness, explainability, robustness, safety, and accountability).
+
+Conduct A/B testing in controlled environments.
+
+Update risk register based on evaluation results.
+
+Engage domain experts to validate evaluation metrics.
+
+** Deployment
+
+Conduct pre-deployment security audit.
+
+Implement a continuous monitoring and alerting system.
+
+Set up automated regression testing.
+
+Establish feedback mechanisms from end-users and stakeholders.
+
+Ensure that risk mitigation measures are operational (e.g., rollback mechanisms).
+
+Automate the deployment pipeline to include quality gate checks.
+
+** Monitoring and Maintenance
+
+Implement automated monitoring for data drift, concept drift, and model performance.
+
+Set up alerts for predefined risk thresholds.
+
+Conduct regular model and system internal audits.
+
+Perform periodic chaos engineering tests.
+
+Regularly update risk register based on operational insights.
+
+Schedule periodic reviews of ethical guidelines and value alignment.
+```
+
+**MLOps and AI Engineering Checklist for Compliance with EU AI Act, Article 10. Data and Data Governance**
+
+```
+** Business and Data Understanding
+
+Establish a data governance framework aligned with EU AI Act requirements.
+
+(Organizational) Define roles and responsibilities for data management.
+
+Create data quality metrics and standards documentation.
+
+Document all data sources and their credibility.
+
+Verify data licensing and usage rights.
+
+Assess data protection requirements.
+
+Validate data source reliability.
+
+Profile existing datasets.
+
+Document known data quality issues.
+
+Assess data completeness requirements.
+
+Evaluate data representativeness.
+
+Check for potential biases.
+
+** Data Preparation
+
+Specify data validation requirements (e.g., schema validation, data type checks, range checks, functional dependency validation, consistency checks, etc.).
+
+Implement data quality checks (e.g., completeness, accuracy, consistency, etc.).
+
+Set up data validation pipelines.
+
+Implement data cleaning procedures.
+
+Implement data lineage tracking.
+
+Validate processed data quality.
+
+Document transformation rules.
+
+Validate transformed data.
+
+Implement data anonymization where required.
+
+Set up data versioning.
+
+Implement privacy-preserving measures in data.
+
+Set up secure data storage.
+
+Implement access controls.
+
+Document security measures.
+
+Set up audit logging.
+
+** Modeling
+
+Validate training data quality.
+
+Document data splitting methodology.
+
+Implement cross-validation strategy.
+
+Track data versions used for training.
+
+Track model versions.
+
+Track model lineage.
+
+Monitor data drift.
+
+Implement reproducibility controls.
+
+Track experiment results.
+
+Implement model validation procedures.
+
+Test model robustness.
+
+Validate model fairness.
+
+** Evaluation
+
+Evaluate model performance.
+
+Assess model fairness and check for biases.
+
+Validate model robustness.
+
+Conduct risk assessment.
+
+Validate security measures.
+
+** Deployment
+
+Define SLAs (Service Level Agreements) for data quality (completeness, accuracy, timeliness, consistency).
+
+Set up quality gates.
+
+Set up alerts.
+
+Set up incident response.
+
+Implement production monitoring.
+
+Implement rollback procedures.
+
+** Monitoring and Maintenance
+
+Monitor data quality metrics.
+
+Track model performance (e.g. processing or inference time, error rates).
+
+Monitor fairness metrics.
+
+Track system health.
+
+Track business critical metrics (e.g., revenue-impacting metrics, customer-facing metrics, regulatory requirements).
+
+Implement update procedures.
+
+Track changes and updates.
+
+Maintain (internal) audit trail.
+```
+
+**MLOps and AI Engineering Checklist for Compliance with EU AI Act, Article 11. Technical Documentation and Article 12. Record-Keeping**
+
+```
+** Business & Data Understanding Phase
+
+Document intended purpose and use cases (Document business objectives and constraints. Specify expected performance metrics)
+
+Establish record-keeping infrastructure (Define data collection scope and methods. Setup metadata tracking systems. Define logging requirements)
+
+Document data sources and specifications (Data provenance records, data quality criteria, privacy and security requirements, data governance procedures)
+
+** Model Training and Operationalization Phase
+
+Document model development environment (Hardware specifications, software dependencies, development tools and versions)
+
+Document training methodology (Model architecture specifications, hyperparameter settings, training algorithms and procedures, feature engineering processes)
+
+Implement pipeline logging (Training infrastructure logs, resource utilization tracking, pipeline execution records)
+
+Implement training logs (Training runs metadata, model performance metrics, data versioning records, feature extraction logs)
+
+** Model Deployment and Serving Phase
+
+Document deployment procedures (Testing protocols, validation methods, release criteria)
+
+Document deployment architecture (Infrastructure requirements, scaling procedures, security measures)
+
+Implement deployment logging (Deployment event logs, configuration changes, version control records, access control logs)
+
+Document operational procedures (Maintenance protocols, update procedures, emergency response plans)
+
+Document serving infrastructure (API specifications, performance requirements, resource allocations, scaling policies)
+
+Implement prediction logging (Inference requests and responses, latency metrics, error logs, user feedback records)
+
+Document monitoring strategy (Monitoring metrics, alert thresholds, response procedures, maintenance schedules)
+
+Document monitoring system (Alert thresholds, performance metrics, system health checks)
+```
+
+**MLOps and AI Engineering Checklist for Compliance with EU AI Act, Article 13. Transparency and Provision of Information to Deployers and Article 14. Human Oversight**
+
+```
+** Business and Data Understanding
+
+User Requirement Gathering: Involve users in defining AI system requirements through interviews, surveys, and workshops.
+
+Expectation Alignment: Ensure user expectations are documented and understood. Stakeholders’ fairness expectations are documented.
+
+Transparency Requirements: Define what information needs to be transparent to users based on their needs.
+
+User Personas and Goals: Create user personas to tailor transparency efforts effectively.
+
+Explainability Goals: Establish interpretability objectives that align with user needs and business goals.
+
+Documentation Standards: Set standards for documenting requirements, data sources, and decisions.
+
+Requirement Documentation: Use templates to document business and data understanding comprehensively.
+
+Scope Definition: Clearly define the AI system’s intended use cases and limitations.
+
+Use-Case Documentation: Develop detailed descriptions of appropriate and inappropriate use cases.
+
+** Data Preparation
+
+Data Relevance Validation: Involve users in validating the relevance and quality of the data collected. Complete the data bias assessment. Log the dataset distributions.
+
+Fairness-enhancing preprocessing applied (if necessary).
+
+Data Catalog Accessibility: Provide users with access to data catalogs and descriptions.
+
+Feature Explainability: Ensure that data features are understandable and meaningful to users.
+
+Feature Selection Documentation: Document the rationale behind feature selection.
+
+Data Source Documentation: Record all data sources, collection methods, and preprocessing steps.
+
+Data Versioning: Implement data version control to track changes over time.
+
+Contextual Data Labeling: Label and tag data to indicate the context in which it is appropriate.
+
+** Modeling
+
+Model Selection Input: Seek user input when selecting models to ensure they meet user needs.
+
+Model Architecture Disclosure: Provide users with understandable information about model structures.
+
+Use of Interpretable Models: Prefer inherently interpretable models where possible.
+
+Explainability Techniques: Implement techniques to explain complex models if used.
+
+Model Documentation: Record model architectures, parameters, and training processes.
+
+Model Cards: Create model cards summarizing key information about each model.
+
+Model Applicability Testing: Validate that models are appropriate for the intended contexts.
+
+Stress Testing: Perform stress tests to assess model performance in various scenarios.
+
+Explainability outputs validated (with SHAP, LIME, or similar tools).
+
+Mechanisms for human override implemented.
+
+** Evaluation
+
+User Testing Sessions: Involve users in testing and evaluating model outputs.
+
+Evaluation Results Sharing: Provide users with accessible reports on evaluation outcomes.
+
+Explain Evaluation Metrics: Use understandable metrics and explain their significance to users.
+
+Evaluation Procedures Documentation: Record all evaluation methods and results thoroughly.
+
+Experiment Tracking: Use tools to track experiments and results systematically.
+
+Contextual Performance Analysis: Evaluate model performance across different contexts and document findings.
+
+** Deployment
+
+User Control Options: Implement features that allow users to control AI interactions (e.g., opt-in/opt-out).
+
+AI Interaction Disclosure: Clearly inform users when they are interacting with an AI system.
+
+Information Accessibility: Provide easy access to information about how the AI system works.
+
+Decision Explanations: Offer explanations for AI decisions in a user-friendly manner.
+
+Deployment Documentation: Record deployment configurations, environments, and versions.
+
+AI System Indicators: Use visual cues or labels to indicate AI-generated content or decisions.
+
+Usage Limitations Display: Clearly display the limitations and appropriate contexts of the AI system.
+
+Alerting systems for fairness and drift configured.
+
+** Monitoring and Maintenance
+
+Feedback Mechanisms: Provide channels for users to submit feedback and report issues.
+
+Update Notifications: Inform users about updates, changes, or issues with the AI system.
+
+Continuous Explainability: Ensure explanations remain accurate and relevant over time and real-time monitoring pipelines implemented.
+
+Maintenance Logs: Keep detailed logs of maintenance activities, updates, and system changes.
+
+Documentation Updates: Regularly update documentation to reflect the system’s current state.
+
+Context Drift Monitoring: Monitor for changes in the operating environment that may affect appropriateness. Conduct regular fairness audits.
+
+Misuse Alerts: Set up alerts for potential misuse or operation outside intended contexts.
+
+Feedback loops are established for continuous learning.
+```
+
+**MLOps and AI Engineering Checklist for Compliance with EU AI Act, Article 15: Accuracy, Robustness, and Cybersecurity**
+
+```
+** Business and Data Understanding
+
+Identify potential data quality issues through data profiling and initial analysis.
+
+Document known data limitations and risks that could impact faultlessness.
+
+Ensure training data covers diverse scenarios, including potential edge cases.
+
+Identify potential robustness requirements based on system objectives and risks (For example, the system must resist common adversarial attacks such as feature perturbation or injection of adversarial examples. Or maybe it should achieve at least 90% accuracy when tested with synthetic adversarial inputs).
+
+** Data Preparation
+
+Validate datasets for accuracy, duplicates, outliers, and completeness using data validation tools (e.g., Great Expectations).
+
+Maintain a clear lineage of all preprocessing steps for traceability.
+
+Perform data augmentation to enrich the dataset with diverse and extreme scenarios.
+
+Test data against adversarial examples to ensure robustness.
+
+Encrypt data in transit and at rest using AES-256 encryption.
+
+Apply access controls to ensure secure handling of sensitive data.
+
+** Model Development
+
+Define and track performance metrics that cover a wide range of evaluation needs, from functional correctness to robustness and cybersecurity, ensuring a comprehensive assessment of AI systems for compliance with the EU AI Act Article 15 (e.g., precision, recall, F1-score, Mean Absolute Error (MAE), Adversarial Robustness Score, Adversarial Attack Success Rate, System Uptime and Thoughput).
+
+Implement unit and integration tests for model components.
+
+Use exception handling in code to manage unexpected inputs or processing errors.
+
+Conduct code reviews and apply static code analysis tools (e.g., SonarQube).
+
+Train models with adversarial training techniques to improve robustness.
+
+Evaluate models under stress testing to determine performance limits.
+
+Use interpretable models or apply explainability tools (e.g., SHAP, LIME) to explain decisions.
+
+Use metadata management tools (e.g., MLflow, Neptune.ai) to track model lineage.
+
+Sign model artifacts to ensure their integrity using secure hash algorithms (e.g., SHA-256).
+
+Protect model training environments from unauthorized access.
+
+Conduct security assessments and penetration testing during development.
+
+Apply defenses against adversarial attacks (e.g., gradient masking).
+
+** Model Evaluation
+
+Validate model outputs against ground truth data.
+
+Test the model’s accuracy using holdout datasets and cross-validation techniques.
+
+Conduct regression testing to ensure the absence of bugs in updated models.
+
+Evaluate model performance against adversarial inputs and out-of-distribution scenarios.
+
+** Deployment
+
+Use CI/CD pipelines to automate deployment testing and validation.
+
+Monitor real-time performance to ensure consistency in production.
+
+Use redundancy mechanisms to mitigate failures in production.
+
+Deploy redundant models or ensembles for fault tolerance.
+
+Secure deployment pipelines using role-based access controls (RBAC) and encrypted communications.
+
+Monitor for cybersecurity threats (using tools like AWS GuardDuty).
+
+Develop and periodically test incident response plans for security breaches.
+
+** Monitoring and Maintenance
+
+Monitor edge-case behavior and retrain models as new data becomes available.
+
+Periodically reassess robustness to handle evolving conditions.
+
+Perform regular security audits to ensure the integrity of deployed systems.
+
+Encrypt backups and secure access to sensitive data and systems.
+
+Continuously monitor for potential attacks and implement mitigations.
+
+Use intrusion detection systems to detect unauthorized access or anomalies.
+```
+
+Wow ~ this chapter is so detailed about the articles in the Act. Definitely a book to refenrece if I end up doing some work that might be related to the Act/the EU. 
+
+---
+
+## DE with dbt - Chapter 13 - Moving Beyond the Basics
+
+### Building for modularity
+
+A quick recap of the Pragmatic Data Platform approach:
+
+![image](https://github.com/user-attachments/assets/3cbe1f7c-5264-4bb5-b4e8-ff10be5c6008)
+
+* storage layer: adapt incoming data to how we want to use it without changing its semantics and store all the data: the good, the bad, and the ugly
+* refined layer: apply master data and implement business rules
+* delivery layer: organize the data to be easy to use, depending on the user
+
+**Modularity in the storage layer**
+
+The storage layer is designed for predictability and modularity, leveraging interchangeable components for flexibility and scalability. Key components include:  
+
+* seed: loads small reference data from csv files, either directly via `ref()` or indirectly using `source()`. Naming conventions like "SEED_xxx" clarify its function
+* source definition: references external data, grouping inputs for easier management and auditability while maintaining table name consistency
+* snapshot: tracks changes in input entities with periodic snapshots. Limitations include shared resources across environments and dependency on full exports for deletion tracking 
+* STG model: integrates components via CTEs. Subcomponents include:  
+   - adaptation: prepares incoming data by fixing data types, renaming, and applying corrections
+   - default record: adds default values for orphan foreign keys  
+   - hashed: calculates keys (HKEY, HDIFF) and metadata columns (e.g, LOAD_TS)  
+* HIST model: an insert-only strategy for tracking entity changes, creating a slowly changing dimension type 2 table. This high-performance default stores both current and historical states effectively. Delete-aware variants handle deletions seamlessly
+* PII model: separates Personally Identifiable Information (PII) for compliance and controlled access
+
+**Modularity in the refined layer**
+
+The refined layer does not have the same level of predictability as the storage layer, but the general playbook is pretty clear and goes trough these stages:
+
+* refined individual system: the initial process is to compose and transform the data from each individual source system, including the use of locally specific business rules, up to an abstraction level comparable with the other systems
+* master data mapping: applying master data transformations to convert the codes from one system to make them interoperable with a wider range of systems
+* master data refined layer: the final process is to compose and transform the results of the master data mapping to produce a set of business concepts, expressed with master data codes, that allow us to calculate company-wide business metrics and apply company business rules
+
+Progressive vs direct MD conversion example:
+
+![image](https://github.com/user-attachments/assets/4fe3e12b-2e52-4b1e-9653-33c37f563184)
+
+There are many different transformations that can be applied, but here is a general overview:
+
+* REF model: the REF model’s role is to represent a refined entity in a certain context, something that you expect to be generally reusable in that context because it represents a concept that is useful and fairly complete in the context
+* AGG model: the AGG model’s role is to represent an aggregated refined model
+* MDD model: the MDD model’s role is to represent a dimension coming directly from master data, therefore a kind of 'authoritative' list of entities
+* MAP model: the MAP model’s role is to represent a mapping table that allows code conversions between one system/context and another
+* TR model: the TR model’s role is to represent an intermediate transformation
+* PIVOT model: the PIVOT model’s role is to obtain models by pivoting other models
+
+**Modularity in the delivery layer**
+
+When we move to the delivery layer, we have one or more data marts. The role of each data mart is to deliver a dataset related to a specific subject, such as marketing, finance, or operations, at the level of a domain that often matches one organization boundary, such as a plant, country, region, or corporation-wide.
+
+Common models found here:
+
+* FACT model: the FACT model’s role is to represent a Kimball-style fact
+* DIM model: the DIM model’s role is to represent a Kimball-style dimension or hierarchy
+* REPORT model: the REPORT model’s role is to represent a table that can power a report
+* FILTER model: this is a specialized model whose role is to make it extremely clear how one or more facts are filtered based on requirements that exist only in the specific data mart where the filter model lives
+
+### Managing identity
+
+* identifying a key/code to be used to identify one instance of an entity
+
+**Types of keys:**
+
+* surrogate keys
+* natural key or business key
+* primary key
+* foreign key
+
+**Main uses of keys**
+
+* tracking and applying changes
+* extracting the current and historical version
+* joining data
+
+### Master Data management
+
+Master Data refers to the core descriptive data of an organization, ensuring consistent identification of the same concept across different units. It often overlaps with organizational dimensions like customer, product, or employee.
+
+**Data for Master Data management**
+
+* Master Data Dimensions (MDDs): this is the dimension that contains the 'gold records' of the business concept. Often, there is not a full dimension but only a list of master data codes, maybe with a name or description for human use
+* Mapping tables (MAPs): this is a table that allows us to convert the codes from one domain into another – for example, converting Italian product codes into EMEA codes
+
+### Saving history at scale
+
+Before, we used dbt's snapshot feature to track changes to the data. But now the HIST method will be used - _they are simple, efficient, flexible, and resilient and they are the best solution, whether you wish to store the changes, if you want to be able to audit the data that you ingest, or even if you just want to have the current data._
+
+Main advantages of this storage pattern:
+
+* HIST models save version changes using insert-only operations and offer the most simple and scalable way to save your data while keeping all the versions that your data has gone through (insert is the most efficient operation on modern cloud databases, while updates or deletes used by Snapshot have poor performances)
+
+The columnar format is quite good at compressing data, especially for columns with low variability, so our versions will be stored effectively
+
+* HIST tables implement idempotent loads which allow to run and re-run our dbt project
+* HIST tables store the changes in a common and flexible way as slowly changing dimensions, allowing to start by using only the current versions, and move to use all the changes in case we need them. There’s no need to reload the data or change how we load/store data if we want to use the stored changes
+* even the simplest implementation of the save_history macro provides full auditability of the changes if we receive one change per key per load
+
+**Understanding the save_history macro**
+
+The normal use of the macro is to create HIST models that read data from our STG models and save it.
+
+Example usage for one of the staging tables:
+
+```
+{ save_history(
+    input_rel = ref('STG_ABC_BANK_POSITION'),
+    key_column = 'POSITION_HKEY',
+    diff_column = 'POSITION_HDIFF',
+) }
+```
+
+Here is the outline of the macro:
+
+<img width="496" alt="image" src="https://github.com/user-attachments/assets/b617c5e9-1707-4043-8324-2b3eac7cc700" />
+
+By not providing a unique_key parameter to the config, we instruct dbt to append all the rows returned by our SQL code to the incremental model.
+
+The  macro’s design is based on the dbt is_incremental() macro to run the right code depending if we already have the underlying table or if we are creating it
+
+_A crucial step of this load pattern is to be sure that, in both the input and the current_from_history, we only have one row for each key. If we have multiple rows, one HDIFF can never match two different ones, so the load will be incorrect. The current_from_history macro satisfies this requirement by design, so we need to ensure that the input is unique by HKEY._
+
+The core of this pattern is to determine what rows from the input need to be inserted into the HIST table. This job is performed by the two versions of the load_from_input CTE – one for the incremental runs and one for the non-incremental ones.
+
+I think this way of adding history uses a different method compared to what I learned in Zach Wilson's bootcamp about implementing SCD type 2. I think that code then was very manual and not designed to use over different pipelines. However, with *this* one it can be used over different pipelines and create a history table. 
+
+**The current_from_history macro**
+
+<img width="807" alt="image" src="https://github.com/user-attachments/assets/4caa5849-dc08-4189-9d0c-76495255e870" />
+
+The heavy lifting is done by the QUALIFY clause, where we partition the data by the desired key and order it by the desired column, in descending order, so that the first row that we keep is the most recent.
+
+Maybe the only interesting param is the qualify_function - it is an optional string that designates the window function used to filter the HIST table
+
+The default value is row_number, which means that the most recent row is returned, based on the key used to partition and the column used to order.
+
+When we use a key with a coarser grain than the data stored in the HIST table, we can use other functions to control what data is retrieved.
+
+As an example, by using the rank() function together with a MONTH(LOAD_TS_UTC) order column, we can retrieve all the changes that were made in the month with the most recent data by each key.
+
+(wow this macro is amazing)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 0bd02ca0e1a2b7fc4c6405fa9e9f817d8579fea0 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 22 Dec 2024 22:24:07 +0900
Subject: [PATCH 550/638] Create 2024-12-22-day-356.md

---
 _posts/2024-12-22-day-356.md | 333 +++++++++++++++++++++++++++++++++++
 1 file changed, 333 insertions(+)
 create mode 100644 _posts/2024-12-22-day-356.md

diff --git a/_posts/2024-12-22-day-356.md b/_posts/2024-12-22-day-356.md
new file mode 100644
index 000000000..f31523df1
--- /dev/null
+++ b/_posts/2024-12-22-day-356.md
@@ -0,0 +1,333 @@
+---
+layout: post
+title: (Day 356) Applying analytical patterns - new content from Zach Wilson's bootcamp
+categories: [data-eng,theory,mlops]
+---
+
+# Hello :) Today is Day 356!
+A quick summary of today:
+* finished the DE with dbt book - overall opinion - amazing!
+* covered lecture 1 + lab 1 from the analytical pattern content of Zach Wilson's bootcamp (+ homework Q1 😆)
+
+## DE with dbt - Chapter 14 - Enhancing Software Quality
+
+### Refactoring and evolving models
+
+_The core of refactoring is to do multiple small changes that keep the project working, test after each small change, and in the end replace the initial code with better code that keeps working as desired._
+
+**Dealing with technical debt**
+
+Technical debt refers to the future cost of transforming a working but imperfect solution into a cleaner, more maintainable one. It is normal for initial solutions to be suboptimal, but teams must manage technical debt to maintain effectiveness and avoid stagnation. Like financial debt, technical debt should be used judiciously and paid back over time to keep it manageable.
+
+The best approach is to refactor code during development, aiming for a 'good enough' version that is sustainable rather than perfect. Refactoring should be part of delivering functionality, not treated as extra work. Small, continuous improvements can help manage technical debt without halting new development. Only in extreme cases should a team pause new work to address excessive debt.
+
+**Tools for refactoring in dbt projects**
+
+* code readability and organization
+* tests
+* [data auditing](https://hub.getdbt.com/dbt-labs/audit_helper/latest/)
+
+### Publishing dependable datasets
+
+* provide clear and consistent naming for columns and tables
+* use a consistent naming convention
+* use correct and predictable data types
+* decide on the set of default records that you want to use and consistently apply them
+* right-size your data marts
+* once you publish a fact or dimension, refrain from renaming it, renaming the columns inside, or changing the data types
+* when you want/need to change your API, do so in a controlled and communicated way
+
+**What shape should you use for your data mart?**
+
+Different uses of the data call for different ways to deliver the dataset for optimal usability.  
+
+- as an example, for most ai/ml workloads, it is preferable to provide a wide table with all the interesting data cleaned up and pre-joined to the desired granularity, while most bi tools expect to have data models provided in kimball style – that is, in the shape of a star schema
+
+- do not think that all cases should use the same data mart; instead, look at what could be the best solution for each use case
+
+- as an example, if you have to power a rest api, it is better to provide tables that are cut along the data that the api is serving. you should try to use hybrid tables from snowflake unistore’s new functionality 
+
+- when you are looking into operational activities and you want to analyze error situations or when processes do not complete as expected or are left halfway, it might be useful to build a few composite facts using full outer joins so that, besides the usual matching data from the normal (inner) join, you can also observe the data that is not matching from both sides  
+
+- if you recall our discussion of the unified star schema from chapter 3, you can leverage the union all operator to build wide tables with many fks built by following the relations in the data to report on seemingly disparate objects that instead have some common fk in their chain of relations 
+
+- in most cases, you will have at least some data mart prepared for power reporting with bi tools, providing a kimball star schema
+
+**Self-completing dimensions**
+
+The self-completing dimension ensures all foreign keys in fact tables have corresponding entries in the dimension table, preventing missing data issues during joins. It handles orphaned keys by adding default rows to the dimension rather than altering fact tables, ensuring data completeness and integrity. This approach is efficient, flexible, and aligns with modern data practices by shifting validation to the BI delivery layer. It simplifies testing, reduces manual intervention, and maintains clean, consistent data models.
+
+This chapter was mostly refactoring, and also understanding the self-completing dimensions macro. All of the code is on [my repo](https://github.com/divakaivan/portfolio-tracking-dbt)
+
+The jinja below may look wrong but I had to remove some of the jinja special characters to due to a rendering issue with my blog's markdown.
+
+```
+%- macro self_completing_dimension(
+    dim_rel,
+    dim_key_column,
+    dim_default_key_value = '-1',
+    rel_columns_to_exclude = [],
+
+    fact_defs = []
+) -
+/* ** Usage notes **
+ * - The primary key has to be the first field in the underlying reference for the dimension
+ */
+
+% do rel_columns_to_exclude.append(dim_key_column) -%
+
+WITH
+dim_base as (
+    SELECT
+          {dim_key_column }
+        , d.* EXCLUDE( { rel_columns_to_exclude | join(', ') } )
+    FROM {{ dim_rel }} as d
+),
+
+fact_key_list as ( 
+    { if fact_defs|length > 0 %   -- If a FACT reference is passed, then check for orphans and add them in the dimension
+
+        {- for fact_model_key in fact_defs }
+        select distinct {{fact_model_key['key']}} as FOREIGN_KEY
+        FROM {{ ref(fact_model_key['model']) }}
+        WHERE {{fact_model_key['key']}} is not null
+            {if not loop.last } union { endif 
+        {%- endfor -%
+
+    {- else %          -- If NO FACT reference is passed, the list of fact keys is just empty.
+    select null as FOREIGN_KEY WHERE false
+    %- endif%
+),
+missing_keys as (
+    SELECT fkl.FOREIGN_KEY 
+    from fact_key_list fkl 
+    left outer join dim_base on dim_base.{{dim_key_column}} = fkl.FOREIGN_KEY
+    where dim_base.{dim_key_column} is null
+),
+default_record as (
+    SELECT *
+    FROM dim_base
+    WHERE {dim_key_column} = '{dim_default_key_value}'
+    LIMIT 1
+),
+dim_missing_entries as (
+    SELECT 
+        mk.FOREIGN_KEY,
+        dr.* EXCLUDE( { dim_key_column } )
+    FROM missing_keys as mk 
+    join default_record dr
+),
+
+dim as (
+    SELECT * FROM dim_base 
+    UNION ALL
+    SELECT * FROM dim_missing_entries 
+)
+
+SELECT * FROM dim
+
+% endmacro %
+```
+
+---
+
+## DE with dbt - Chapter 15 - Patterns for Frequent Use Cases
+
+### Ingestion patterns
+
+* SQL INSERT: the standard SQL INSERT command; it works well for inserting a few rows, but it does not scale (dbt uses the SQL INSERT command to load the rows from the CSV files in the seed folder)
+* EXTERNAL TABLE: in Snowflake, it is possible to define external tables that point to a folder in a container on an external storage service and the table will parse the files according to the provided file format and expression for the fields
+
+A very flexible way to access external data, structured or semi-structured. Snowflake can exploit the schema information stored in semi-structured files, making the creation very simple. The major downside of using external tables is that every time that you query the table, and you do not hit the cache, all the files that back it must be loaded, parsed, and eventually filtered to answer your query. Some improvement can be achieved by adding partitioning based on the file/path name, but they help only if you query with a filter on a partitioning column.
+
+* COPY INTO: the most important for loading data in Snowflake because all high-performance ingestion methods rely on ingesting files through this command in one way or the other
+
+The main advantage of using COPY INTO is that files are parsed only once. A further advantage is that at the end of the command, the data is in a normal Snowflake table and every query that you issue on it takes advantage of micro-partitions and the associated metadata to reduce the data that’s queried. Another advantage is that you could transform the data while loading it, using almost all normal SQL expressions and functions. 
+
+### History patterns
+
+**Storing history with deletions – full load**  
+
+   - Detect deletions by comparing a full export from the source with active entities in the history table 
+   - Effective but costly for large fact tables due to the need for full exports
+   - Alternative: use change data capture (CDC) systems to track deletions
+
+**Storing history with deletions – deletion list**  
+
+   - Avoids full exports by leveraging deletion lists or database features like triggers to track changes 
+   - Scenarios: provided full table payload with deletion info or only the keys of deleted entities
+   - Efficient for large tables, reducing expensive joins and export sizes
+
+**Storing history with multiple versions in the input**  
+
+   - Allows multiple versions in one load by comparing the current version in history with ordered input versions 
+   - Relies on stable sorting criteria (e.g., timestamps) and high watermark columns to manage version order 
+   - Handles late arrivals by adding duplicates if necessary for correctness while ensuring accurate history
+
+**Storing history with PII and GDPR compliance**  
+
+   - Stores anonymized surrogates (SPII) of PII in history tables to ensure GDPR compliance 
+   - Maintains a PII mapping table to convert SPII back to PII when necessary
+   - Implementation varies between full and incremental exports, with hashing applied during staging
+
+**History and schema evolution**  
+
+   - Manages schema changes by freezing old tables and creating new ones for updated schemas  
+   - Uses REF models to merge old and new data, maintaining compatibility and completeness
+   - Adapts static models for non-breaking changes and dynamically unites histories for breaking changes
+
+Overall thoughts about the book ~ amazing !!! It is a source to refer back to when doing a project or thinking about concepts like SCD and how it might be implemented and issues one might face. 
+
+---
+
+## Applying analytical patterns - new content from Zach Wilson's bootcamp
+
+My notes from the 1st lecture (2nd lecture will be uploaded later ~):
+
+![Applying analytical patterns P1](https://github.com/user-attachments/assets/970fe0fb-31ed-4482-9a36-48aff9b26177)
+![Applying analytical patterns P2](https://github.com/user-attachments/assets/25e88ea9-26a1-46e2-9236-3b7397f3639b)
+![Applying analytical patterns P3](https://github.com/user-attachments/assets/8d6d10e9-1d9e-4273-84a2-2bbd2f690d1b)
+
+My code from lab 1:
+
+```sql
+CREATE TABLE users_growth_accounting (
+     user_id TEXT,
+     first_active_date DATE,
+     last_active_date DATE,
+     daily_active_state TEXT,
+     weekly_active_state TEXT,
+     dates_active DATE[],
+     date DATE,
+     PRIMARY KEY (user_id, date)
+ );
+
+-- growth accounting
+INSERT INTO users_growth_accounting
+WITH yesterday AS (
+    SELECT * FROM users_growth_accounting
+    WHERE date = DATE('2023-01-30')
+),
+     today AS (
+         SELECT
+            CAST(user_id AS TEXT) as user_id,
+            DATE_TRUNC('day', event_time::timestamp) as today_date,
+            COUNT(1)
+         FROM events
+         WHERE DATE_TRUNC('day', event_time::timestamp) = DATE('2023-01-31')
+         AND user_id IS NOT NULL
+         GROUP BY user_id, DATE_TRUNC('day', event_time::timestamp)
+     )
+
+         SELECT COALESCE(t.user_id, y.user_id) as user_id,
+                COALESCE(y.first_active_date, t.today_date) AS first_active_date, -- new user
+                COALESCE(t.today_date, y.last_active_date) AS last_active_date,
+                CASE
+                    WHEN y.user_id IS NULL THEN 'New'
+                    WHEN y.last_active_date = t.today_date - Interval '1 day' THEN 'Retained'
+                    WHEN y.last_active_date < t.today_date - Interval '1 day' THEN 'Resurrected'
+                    WHEN t.today_date IS NULL AND y.last_active_date = y.date THEN 'Churned'
+                    ELSE 'Stale'
+                END as daily_active_state,
+                CASE
+                    WHEN y.user_id IS NULL THEN 'New'
+                    WHEN y.last_active_date < t.today_date - Interval '7 day' THEN 'Resurrected'
+                    WHEN t.today_date IS NULL AND y.last_active_date = y.date - interval '7 day' THEN 'Churned'
+                    WHEN COALESCE(t.today_date, y.last_active_date) + INTERVAL '7 day' >= y.date THEN 'Retained'
+                    ELSE 'Stale'
+                END as weekly_active_state,
+                COALESCE(y.dates_active,
+                         ARRAY []::DATE[])
+                    || CASE
+                           WHEN
+                               t.user_id IS NOT NULL
+                               THEN ARRAY [t.today_date]
+                           ELSE ARRAY []::DATE[]
+                    END AS date_list,
+                COALESCE(t.today_date, y.date + Interval '1 day') as date
+         FROM today t
+                  FULL OUTER JOIN yesterday y
+                                  ON t.user_id = y.user_id
+
+-- example usage
+SELECT date, daily_active_state, COUNT(1)
+FROM users_growth_accounting
+GROUP BY date, daily_active_state;
+
+-- look at retention analysis for cohorts (those J-curves from the lecture)
+SELECT
+    EXTRACT(DOW FROM first_active_date) AS dow,
+    date - first_active_date as days_since_first_signup,
+    COUNT(CASE WHEN daily_active_state in ('Retained', 'Resurrected', 'New') THEN 1 END) AS number_active,
+    CAST(COUNT(CASE WHEN daily_active_state in ('Retained', 'Resurrected', 'New') THEN 1 END) AS REAL)/COUNT(1) AS pct_active,
+    COUNT(1)
+FROM users_growth_accounting
+GROUP BY EXTRACT(DOW FROM first_active_date), date - first_active_date;
+```
+
+### With the knowledge from this lab 1 (and also from week 1 where we learned about cumulative table design) I completed Q1 of this week's homework:
+
+Q1: 
+- A query that does state change tracking for `players`
+  - A player entering the league should be `New`
+  - A player leaving the league should be `Retired`
+  - A player staying in the league should be `Continued Playing`
+  - A player that comes out of retirement should be `Returned from Retirement`
+  - A player that stays out of the league should be `Stayed Retired`
+  
+
+```sql
+create type season_state as enum (
+    'New',
+    'Retired',
+    'Continued Playing',
+    'Returned from Retirement',
+    'Stayed Retired'
+);
+
+create table players_state_tracking (
+    player_name text,
+    first_active_season integer,
+    last_active_season integer,
+    season_state season_state,
+    current_season integer,
+    primary key (player_name, current_season)
+ );
+insert into players_state_tracking
+with yesterday as (
+    select
+        player_name,
+        first_active_season,
+        last_active_season,
+        current_season
+    from players_state_tracking
+    where current_season = 1996
+),
+     today as (
+         select
+            player_name,
+            current_season
+         from players
+         where current_season = 1997
+     )
+         select coalesce(t.player_name, y.player_name) as player_name,
+                coalesce(y.first_active_season, t.current_season) as first_active_season,
+                coalesce(t.current_season, y.last_active_season) as last_active_season,
+                case
+                    when y.player_name is null then 'New'
+                    when y.last_active_season = t.current_season - 1 then 'Continued Playing'
+                    when y.last_active_season < t.current_season -1 then 'Returned from Retirement'
+                    when t.current_season is null and y.last_active_season = y.current_season then 'Retired'
+                    else 'Stayed Retired'::season_state
+                end as season_state,
+                coalesce(t.current_season, y.current_season + 1) as current_season
+         from today t
+                  full outer join yesterday y
+                                  on t.player_name = y.player_name
+```
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 8b2ecf70cd3f1fe3a5ef1411d0c16d4a9a11d660 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 22 Dec 2024 22:26:31 +0900
Subject: [PATCH 551/638] Update 2024-12-22-day-356.md

---
 _posts/2024-12-22-day-356.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/_posts/2024-12-22-day-356.md b/_posts/2024-12-22-day-356.md
index f31523df1..c251aa090 100644
--- a/_posts/2024-12-22-day-356.md
+++ b/_posts/2024-12-22-day-356.md
@@ -62,7 +62,7 @@ This chapter was mostly refactoring, and also understanding the self-completing
 The jinja below may look wrong but I had to remove some of the jinja special characters to due to a rendering issue with my blog's markdown.
 
 ```
-%- macro self_completing_dimension(
+- macro self_completing_dimension(
     dim_rel,
     dim_key_column,
     dim_default_key_value = '-1',
@@ -74,7 +74,7 @@ The jinja below may look wrong but I had to remove some of the jinja special cha
  * - The primary key has to be the first field in the underlying reference for the dimension
  */
 
-% do rel_columns_to_exclude.append(dim_key_column) -%
+% do rel_columns_to_exclude.append(dim_key_column) -
 
 WITH
 dim_base as (
@@ -92,16 +92,16 @@ fact_key_list as (
         FROM {{ ref(fact_model_key['model']) }}
         WHERE {{fact_model_key['key']}} is not null
             {if not loop.last } union { endif 
-        {%- endfor -%
+        {- endfor -%
 
-    {- else %          -- If NO FACT reference is passed, the list of fact keys is just empty.
+    {- else           -- If NO FACT reference is passed, the list of fact keys is just empty.
     select null as FOREIGN_KEY WHERE false
     %- endif%
 ),
 missing_keys as (
     SELECT fkl.FOREIGN_KEY 
     from fact_key_list fkl 
-    left outer join dim_base on dim_base.{{dim_key_column}} = fkl.FOREIGN_KEY
+    left outer join dim_base on dim_base.{dim_key_column} = fkl.FOREIGN_KEY
     where dim_base.{dim_key_column} is null
 ),
 default_record as (
@@ -126,7 +126,7 @@ dim as (
 
 SELECT * FROM dim
 
-% endmacro %
+% endmacro 
 ```
 
 ---

From 99edfec8a7d64e5499369e2433495444ccd31f84 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 22 Dec 2024 22:27:12 +0900
Subject: [PATCH 552/638] Update 2024-12-22-day-356.md

---
 _posts/2024-12-22-day-356.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-12-22-day-356.md b/_posts/2024-12-22-day-356.md
index c251aa090..baa17e910 100644
--- a/_posts/2024-12-22-day-356.md
+++ b/_posts/2024-12-22-day-356.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 356) Applying analytical patterns - new content from Zach Wilson's bootcamp
-categories: [data-eng,theory,mlops]
+categories: [data-eng,theory,applying-knowledge]
 ---
 
 # Hello :) Today is Day 356!

From 910e8b706e7e777d9740d18b8ecd9b35cf36948f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:15:43 +0900
Subject: [PATCH 553/638] fix mlop -> mlops tag

---
 _posts/2024-11-17-day-321.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-11-17-day-321.md b/_posts/2024-11-17-day-321.md
index 8907760c1..aacc0e3b2 100644
--- a/_posts/2024-11-17-day-321.md
+++ b/_posts/2024-11-17-day-321.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 321) Slowly changing dimensions
-categories: [data-eng,applying-knowledge,theory,mlop]
+categories: [data-eng,applying-knowledge,theory,mlops]
 ---
 
 # Hello :) Today is Day 321!

From 150fb57c7ba0b02659838cdfffcad8c8e39b977d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:17:01 +0900
Subject: [PATCH 554/638] fix tag

---
 _posts/2024-10-28-day-301.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-28-day-301.md b/_posts/2024-10-28-day-301.md
index 320018b48..792e0eb59 100644
--- a/_posts/2024-10-28-day-301.md
+++ b/_posts/2024-10-28-day-301.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 301) LLM Engineer's Handbook
-categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+categories: [nlp,applying-knowledge,traditional-machine-learning,data-eng]
 ---
 
 # Hello :) Today is Day 301!

From 8db02ed1daef9579b8804d8757e31f519ff6afc6 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:17:15 +0900
Subject: [PATCH 555/638] fix tag

---
 _posts/2024-10-29-day-302.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-29-day-302.md b/_posts/2024-10-29-day-302.md
index 97ce0ddbf..e7e0bbb11 100644
--- a/_posts/2024-10-29-day-302.md
+++ b/_posts/2024-10-29-day-302.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 302) Data engineering pipeline from the LLM Engineer's handbook
-categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+categories: [nlp,applying-knowledge,traditional-machine-learning,data-eng]
 ---
 
 # Hello :) Today is Day 302!

From a27958cb1f80c77be171dc2fd9696cca91011023 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:17:31 +0900
Subject: [PATCH 556/638] fix tag

---
 _posts/2024-10-30-day-303.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-30-day-303.md b/_posts/2024-10-30-day-303.md
index 1f5e651e6..215f95f5d 100644
--- a/_posts/2024-10-30-day-303.md
+++ b/_posts/2024-10-30-day-303.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 303) Using my professor's A6000 GPU for LLM fine-tuning
-categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+categories: [nlp,applying-knowledge,traditional-machine-learning,data-eng]
 ---
 
 # Hello :) Today is Day 303!

From 99457a0d925c47574ec5943e2629385d067e2b62 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:17:44 +0900
Subject: [PATCH 557/638] Update 2024-10-31-day-304.md

---
 _posts/2024-10-31-day-304.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-31-day-304.md b/_posts/2024-10-31-day-304.md
index c81e01773..2103b6577 100644
--- a/_posts/2024-10-31-day-304.md
+++ b/_posts/2024-10-31-day-304.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 304) LLMs + some dbt
-categories: [nlp,applying-knowledge,traditiona-machine-learning,data-eng]
+categories: [nlp,applying-knowledge,traditional-machine-learning,data-eng]
 ---
 
 # Hello :) Today is Day 304!

From 6f1d89d06d03f3fc88c1d9c3e869118afff6801a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:18:28 +0900
Subject: [PATCH 558/638] fix tag

---
 _posts/2024-10-18-day-291.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-10-18-day-291.md b/_posts/2024-10-18-day-291.md
index e68a256c9..d7f712594 100644
--- a/_posts/2024-10-18-day-291.md
+++ b/_posts/2024-10-18-day-291.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 291) Starting Machine Learning for Financial Risk Management with Python + some math for ML
-categories: [mathematics,traditional-machine-learning,theory]
+categories: [math,traditional-machine-learning,theory]
 ---
 
 # Hello :) Today is Day 291!

From 7344f83b338cf169403200337f2d3f87cbde8de7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 21:18:53 +0900
Subject: [PATCH 559/638] fix tag

---
 _posts/2024-12-10-day-344.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-12-10-day-344.md b/_posts/2024-12-10-day-344.md
index 2a746f4e2..be5eaf51b 100644
--- a/_posts/2024-12-10-day-344.md
+++ b/_posts/2024-12-10-day-344.md
@@ -1,7 +1,7 @@
 ---
 layout: post
 title: (Day 344) Checking out the first chapters from Andriy Burkov's new book
-categories: [data-eng,theory,applying-knowledge,nlp,reading-research,mathematics]
+categories: [data-eng,theory,applying-knowledge,nlp,reading-research,math]
 ---
 
 # Hello :) Today is Day 344!

From 95de03c2ea3c8921a36b598f4e26c1b19cd4d39f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 23:12:59 +0900
Subject: [PATCH 560/638] Create 2024-12-23-day-357.md

---
 _posts/2024-12-23-day-357.md | 338 +++++++++++++++++++++++++++++++++++
 1 file changed, 338 insertions(+)
 create mode 100644 _posts/2024-12-23-day-357.md

diff --git a/_posts/2024-12-23-day-357.md b/_posts/2024-12-23-day-357.md
new file mode 100644
index 000000000..ddbb88be8
--- /dev/null
+++ b/_posts/2024-12-23-day-357.md
@@ -0,0 +1,338 @@
+---
+layout: post
+title: (Day 357) MLOps + data drift + starting some summaries for the final blog post
+categories: [mlops,theory,applying-knowledge,reading-research,data-eng]
+---
+
+# Hello :) Today is Day 357!
+A quick summary of today:
+* a bit more of MadeWithML's MLOps course
+* reading a paper - Failing Loudly: An Empirical Study of Methods for Detecting Dataset Shift
+* blog posts word viz
+* starting a random Apache Flink course on udemy
+
+---
+
+## Continued with the [MadeWithML course](https://madewithml.com/#course)
+
+### Distributed Data Processing
+
+Using a single machine is fine when the dataset can fit in memory. But for Big Data we need a distributed system like Ray, Spark, Dask, Modin.
+
+The course chooses to use Ray
+
+![image](https://github.com/user-attachments/assets/c8dad29b-566f-4745-b0d2-c97e8fb98f45)
+
+After importing a dataset with ray, and splitting it, when running the cell below for a second we can see the logical steps in the distributed computation
+
+<img width="1097" alt="image" src="https://github.com/user-attachments/assets/ebfd990f-8062-48b8-80da-060da6569c34" />
+
+(all of the code is on [MadeWithML's repo](https://github.com/GokuMohandas/Made-With-ML))
+
+### Distributed training
+
+**Intuition**
+
+1. Start with a random (chance) model
+2. Develop a rule-based approach using if-else statements, regular expressions, etc
+3. Slowly add complexity by addressing limitations and motivating representations and model architectures
+4. Weigh tradeoffs (performance, latency, size, etc.) between performant baselines
+5. Revisit and iterate on baselines as your dataset grows and new model architectures are developed
+
+**Distributed training**
+
+With the rapid increase is data and model sizes, we need to be able to distribute our training across multiple machines in order to train in a reasonable amount of time. And we want to be able to do this without having to:
+
+* set up a cluster by individually provisioning compute resources
+* writing complex code to distribute our training across multiple machines
+* worry about communication and resource utilization between our different distributed compute resources
+* worry about fault tolerance and recovery from our large training workloads
+
+Thankfully Ray Train helps us with that.
+
+_With distributed training, there will be a head node that's responsible for orchestrating the training process. While the worker nodes that will be responsible for training the model and communicating results back to the head node. From a user's perspective, Ray abstracts away all of this complexity and we can simply define our training functionality with minimal changes to our code (as if we were training on a single machine)._
+
+After following the setup for using one of the BERT models, the training can be seen on Ray's UI:
+
+<img width="1344" alt="image" src="https://github.com/user-attachments/assets/44d41e44-49de-4a95-82e6-2b539d801e86" />
+
+### Experiment Tracking
+
+_Experiment tracking is the process of managing all the different experiments and their components, such as parameters, metrics, models and other artifacts and it enables us to:_
+
+* organize all the necessary components of a specific experiment. It's important to have everything in one place and know where it is so you can use them later
+* reproduce past results (easily) using saved experiments
+* log iterative improvements across time, data, ideas, teams, etc
+
+The prime example - Mlflow - it's amazing and free!
+
+### Hyperparam tuning
+
+Using Ray Tune (others are Optuna, Hyperopt).
+
+I haven't used this lib before so here is some of the setup that looks interesting:
+
+```python
+# Hyperparameters to start with
+initial_params = [{"train_loop_config": {"dropout_p": 0.5, "lr": 1e-4, "lr_factor": 0.8, "lr_patience": 3}}]
+search_alg = HyperOptSearch(points_to_evaluate=initial_params)
+search_alg = ConcurrencyLimiter(search_alg, max_concurrent=2)
+
+# Parameter space
+param_space = {
+    "train_loop_config": {
+        "dropout_p": tune.uniform(0.3, 0.9),
+        "lr": tune.loguniform(1e-5, 5e-4),
+        "lr_factor": tune.uniform(0.1, 0.9),
+        "lr_patience": tune.uniform(1, 10),
+    }
+}
+
+# Scheduler to prune unpromising trials
+scheduler = AsyncHyperBandScheduler(
+    max_t=train_loop_config["num_epochs"],  # max epoch (<time_attr>) per trial
+    grace_period=5,  # min epoch (<time_attr>) per trial
+)
+
+# Finally the tune config
+tune_config = tune.TuneConfig(
+    metric="val_loss",
+    mode="min",
+    search_alg=search_alg,
+    scheduler=scheduler,
+    num_samples=num_runs,
+)
+
+# Tuner
+tuner = Tuner(
+    trainable=trainer,
+    run_config=run_config,
+    param_space=param_space,
+    tune_config=tune_config,
+)
+
+# Tune
+results = tuner.fit()
+```
+
+And we can get a df of the results with `results.get_dataframe()` or get the best with `results.get_best_result(metric="val_loss", mode="min")`
+
+### Evaluation
+
+* the usual - precision, recall, f1
+* fine-grained - the above but per class
+* confusion matrices
+* confidence learning: this is something I have not seen before and it is interesting.
+
+This [cleanlab](https://github.com/cleanlab/cleanlab) package helps with this. With this we can focus on the labeling quality. The package focuses on identifying noisy labels (with calibration), which can then be properly relabeled and used for training. Here is a picture from the cleanlab docs:
+
+![image](https://github.com/user-attachments/assets/e5ceb098-69f0-4d24-b614-ca3917523b05)
+
+* slicing
+
+Slicing breaks a dataset into subsets to evaluate model performance on specific groups, uncovering hidden issues and biases. Key slices include target/predicted classes, features, metadata, and priority groups. It helps ensure fairness and identify underperforming areas.
+
+To address bias (like overgeneralization of CNNs for vision or transformers for NLP), teams can:
+
+```
+  * collect more diverse data
+  * oversample incorrect predictions
+  * mask features contributing to bias
+  * slicing functions define these subsets, enabling targeted improvements for fair and robust models
+```
+
+* interpretability with libs like SHAP and LIME
+* behavioral testing: the process of testing input data and expected outputs while treating the model as a black box
+* online evaluation
+
+A/B test:
+
+![image](https://github.com/user-attachments/assets/74f062d5-e818-4a37-ab14-af22c4918f87)
+
+Canary tests:
+
+![image](https://github.com/user-attachments/assets/9c9dfd7a-c596-4ae7-885a-3b95c9b3c3b9)
+
+Shadow tests:
+
+![image](https://github.com/user-attachments/assets/a3b555b0-9d15-498e-8629-68cb8b3be5b6)
+
+### Serving
+
+When choosing a framework we would want to consider:
+
+* is it Pythonic, do we need to learn another framework to deploy our models
+* is it framework agnostic
+* is (auto)scaling easy
+* can we combine multiple models into the service
+* is it integrated with popular API frameworks like FastAPI
+
+### Logging
+
+_Logging is the process of tracking and recording key events that occur in our applications for the purpose of inspection, debugging, etc. They're a whole lot more powerful than print statements because they allow us to send specific pieces of information to specific locations with custom formatting, shared interfaces, etc. This makes logging a key proponent in being able to surface insightful information from the internal processes of our application._
+
+There are 3 things to know:
+
+* `Logger`: emits the log messages from our application
+* `Handler`: sends log records to a specific location
+* `Formatter`: formats and styles the log records
+
+### Documentation
+
+Found this VScode extension for automatic python docstring generation. Nowadays copilot might help with that as well but its results might not always be consistent and the below package provides a popular format. 
+
+<img width="358" alt="image" src="https://github.com/user-attachments/assets/7a87ac22-fdc4-49a2-933d-c6f8bea9e1f4" />
+
+### Testing
+
+![image](https://github.com/user-attachments/assets/17dced9a-36b3-4eac-826c-86e4e1dc03b0)
+
+There are four majors types of tests:
+
+* unit: tests on individual components that each have a single responsibility (ex. function that filters a list)
+* integration: tests on the combined functionality of individual components (ex. data processing)
+* system: tests on the design of a system for expected outputs given inputs (ex. training, inference, etc)
+* acceptance: tests to verify that requirements have been met, usually referred to as User Acceptance Testing (UAT)
+* regression: tests based on errors we've seen before to ensure new changes don't reintroduce them
+
+**How should we test**
+
+Using the Arrange Act Assert methodology (also mentioned in the Effective Machine Learning Teams book):
+
+* arrange: set up the different inputs to test on
+* act: apply the inputs on the component we want to test
+* assert: confirm that we received the expected output
+
+**What should we test?**
+
+* inputs: data types, format, length, edge cases (min/max, small/large, etc.)
+* outputs: data types, formats, exceptions, intermediary and final outputs
+
+**Best practices**
+
+* atomic: when creating functions and classes, we need to ensure that they have a single responsibility so that we can easily test them. If not, we'll need to split them into more granular components
+* compose: when we create new components, we want to compose tests to validate their functionality. It's a great way to ensure reliability and catch errors early on.
+reuse: we should maintain central repositories where core functionality is tested at the source and reused across many projects. This significantly reduces testing efforts for each new project's code base
+* regression: we want to account for new errors we come across with a regression test so we can ensure we don't reintroduce the same errors in the future
+* coverage: we want to ensure 100% coverage for our codebase. This doesn't mean writing a test for every single line of code but rather accounting for every single line
+* automate: in the event we forget to run our tests before committing to a repository, we want to auto run tests when we make changes to our codebase. We'll learn how to do this locally using pre-commit hooks and remotely via GitHub actions in subsequent lessons
+
+### Monitoring
+
+**System health**
+
+![image](https://github.com/user-attachments/assets/5c8b1bb6-a09b-4b30-8462-16ffe1bcb300)
+
+**Performance**
+
+![image](https://github.com/user-attachments/assets/3f491355-3ccd-4cc9-8362-cda75f1b4704)
+
+**Drift**
+
+* data dirft
+
+![image](https://github.com/user-attachments/assets/ecd98be3-fd92-415b-a63c-04c1b3415889)
+
+* target drift
+* concept drift
+
+![image](https://github.com/user-attachments/assets/235b974f-c61f-4b59-8e50-1899eb6b2636)
+
+**Locating drift**
+
+The constraints we need to consider:
+
+* reference window: the set of points to compare production data distributions with to identify drift
+* test window: the set of points to compare with the reference window to determine if drift has occurred
+
+Honestly, this is such a great resource and definitely one that I will refer to in the future when doing a project!
+
+---
+
+## [Failing Loudly: An Empirical Study of Methods for Detecting Dataset Shift](https://arxiv.org/pdf/1810.11953)
+
+I found this paper from the MadeWithML reference list and decided to have a look ~
+
+The paper addresses the issue of 'silent failures' in ML systems.
+
+The research utilizes a statistical two-sample testing framework to detect shifts but since applying such tests directly to high-dim data is computationally expensive and statistically weak, the authors first use various dimensionality reduction techniques.
+
+<img width="679" alt="image" src="https://github.com/user-attachments/assets/338f8fda-20c2-49ab-9a52-5f374a309cbf" />
+
+* dimensionality reduction: transforming both source (training) and target (new) data into lower-dimensional representations using various methods
+* two-sample testing: applying appropriate statistical hypothesis tests to compare the distributions of the source and target data in the reduced space
+
+**Dimensionality Reduction Techniques**
+
+Different techniques are explored:
+
+* no reduction: a baseline using raw features
+* PCA: a linear method capturing maximum variance
+* Random Projections (SRP): projecting data onto random lower-dimensional subspaces
+* Autoencoders (TAE and UAE): trained (TAE) and untrained (UAE) autoencoders for non-linear representation learning
+* Label Classifiers (BBSDs and BBSDh): inspired by Black Box Shift Detection (BBSD), using softmax outputs (BBSDs) or hard predictions (BBSDh) of a pre-trained label classifier
+* Domain Classifier (Classif): explicitly training a classifier to distinguish between source and target data
+
+**Statistical Hypothesis Tests**
+
+Depending on the data type and dimensionality, different statistical tests are applied:
+
+* multivariate: Maximum Mean Discrepancy (MMD) for multi-dimensional representations
+* univariate: Kolmogorov-Smirnov (KS) test for continuous data and Chi-Squared test for discrete data, applied across individual dimensions with Bonferroni correction
+* Binomial test: assessing the accuracy of the domain classifier against random chance
+
+**Key Findings: Effectiveness of BBSD and Shift Characteristics**
+
+BBSD with Softmax Outputs (BBSDs): across various simulated dataset shifts, BBSDs consistently demonstrated superior performance in detecting shifts, even when the label shift assumption was not met. 
+
+_We note that BBSDs being the best overall method for detecting shift is good news for ML practitioners. When building black-box models with the main purpose of classification, said model can be easily extended to also double as a shift detector._
+
+Certain shifts like Gaussian noise, image transformations, and adversarial examples were easily detected, whereas others like class imbalance (knock-out) proved more challenging.
+
+The ability to detect shifts increased with the intensity of the shift and the proportion of affected data points.
+
+## Creating a wordcloud
+
+I decided to try to make a word cloud of my most used words throughout this blog. I setup the code using the WordCloud python library, and here is what came out with 
+
+<img width="996" alt="image" src="https://github.com/user-attachments/assets/505200e4-03c5-4466-8834-9e6cdf66a9d1" />
+
+I will re-run the code on day 366 to get a final view.
+
+This is the distribution of categories (tags from every post) as well:
+
+<img width="782" alt="image" src="https://github.com/user-attachments/assets/8a95aa3f-78d4-4b24-aab2-9f978bec27d3" />
+
+The thing with these tag categories is that if, say, I studied something to MLOps - then as part of that I might have used some model and then that could also bring a tag like GNN, or traditiona-machine-learning (a tag signifying the type of model used) ~ so it's a bit tricky and makes me wonder if it's worth posting this picture (an updated version) on the final blog post...
+
+
+## [Apache Flink | A Real Time & Hands-On course on Flink](https://www.udemy.com/course/apache-flink-a-real-time-hands-on-course-on-flink/)
+
+I found this course and because of my geolocation the price was ~7GBP and decided to check it out. It's apparently the most covered Flink course on Udemy
+
+It covers:
+
+- introduction to flink  
+- transformation operations of dataset api  
+- datastream api operations  
+- windows in flink  
+- triggers & evictors  
+- watermarks and late elements  
+- state, checkpointing and fault tolerance
+
+Left to cover:
+
+- interacting with real-time data using kafka  
+- case study 1 - twitter data analysis using flink  
+- case study 2 - bank real-time fraud detection  
+- case study 3 - stock data processing in real-time  
+- table & sql api - relational apis of flink  
+- gelly api for graph processing  
+
+As it is content that is 100% behind a paywall, I cannot share anything ~ but I watched about half of it today and I think the free Flink/Streaming courses on Confluent were a bit better, but I will see as the course goes until the end ~ 
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From a8cc0ba1dbf45659931d27b0308de7ee6d963e6f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 23 Dec 2024 23:16:26 +0900
Subject: [PATCH 561/638] add final sentence to 357

---
 _posts/2024-12-23-day-357.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_posts/2024-12-23-day-357.md b/_posts/2024-12-23-day-357.md
index ddbb88be8..0870855cf 100644
--- a/_posts/2024-12-23-day-357.md
+++ b/_posts/2024-12-23-day-357.md
@@ -329,7 +329,7 @@ Left to cover:
 - table & sql api - relational apis of flink  
 - gelly api for graph processing  
 
-As it is content that is 100% behind a paywall, I cannot share anything ~ but I watched about half of it today and I think the free Flink/Streaming courses on Confluent were a bit better, but I will see as the course goes until the end ~ 
+As it is content that is 100% behind a paywall, I cannot share anything ~ but I watched about half of it today and I think the free Flink/Streaming courses on Confluent were a bit better, but I will see as the course goes until the end ~ Also the most exciting part is the case studied 😆 but I am watching all to recap and double-check my knowledge
 
 ---
 

From 556aa8aff4b68410ba8a2f381373ca1aab3ec11f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 24 Dec 2024 20:31:59 +0900
Subject: [PATCH 562/638] Create 2024-12-24-day-358.md

---
 _posts/2024-12-24-day-358.md | 134 +++++++++++++++++++++++++++++++++++
 1 file changed, 134 insertions(+)
 create mode 100644 _posts/2024-12-24-day-358.md

diff --git a/_posts/2024-12-24-day-358.md b/_posts/2024-12-24-day-358.md
new file mode 100644
index 000000000..26541fb09
--- /dev/null
+++ b/_posts/2024-12-24-day-358.md
@@ -0,0 +1,134 @@
+---
+layout: post
+title: (Day 358) Ready to learn MLOps from pros
+categories: [mlops,theory,reading-research]
+---
+
+# Hello :) Today is Day 358!
+A quick summary of today:
+* registered for Maria Vechtomova and Başak Tuğçe Eskili's End-to-End MLOps course
+* reading about AI Infrastructure Alliance
+* read and listened to a podcast about CRISP-ML(Q)
+
+## Registered for Maria Vechtomova and Başak Tuğçe Eskili's End-to-End MLOps course
+
+![image](https://github.com/user-attachments/assets/48e9d466-446a-47df-8d5f-6638632e103f)
+
+It took me ~6 hours to register due to bank restrictions for non-Koreans here in Korea... but thankfully at the end I managed to register. Christmas came 1 day early for me 😆
+
+## [Why We Started the AIIA and What It Means for the Rapid Evolution of the Canonical Stack of Machine Learning](https://ai-infrastructure.org/why-we-started-the-aiia-and-what-it-means-for-the-rapid-evolution-of-the-canonical-stack-of-machine-learning/)
+
+I saved this article from _The AI Engineer's Guide to Surviving the EU AI Act_ book as it seemed interesting ~
+
+The AI Infrastructure Alliance (AIIA) talk about the need for a standardized, accessible AI/ML infrastructure stack similar to the LAMP stack in the web dev world. AIIA aims to foster collaboration amongst companies developing AI/ML infrastructure software to create this standardised/canonical stack. 
+
+One of the goals is to create a blueprint for enterprises to use for their AI/ML workflows. The below is an early version from one of AIIA's working groups:
+
+![image](https://github.com/user-attachments/assets/f3aff2dc-8656-481d-876a-d2f29811886c)
+
+There are already a few companies that provide a platform that covers many of these blocks. 
+
+**What does the architecture of tomorrow look like?**
+
+Many sources cite this architecture by Google MLOps:
+
+![image](https://github.com/user-attachments/assets/1362dda5-62f8-4879-91c8-ae39a557f9e5)
+
+But as the article authors mention - it is missing Data - the key behind AI
+
+_Where are they storing the data?  What kind of system are they using to access that data? How do they control access and version it?
+They don’t include a storage and data versioning layer at all. The diagram picks up at “data extraction” assuming you already have data storage and scaling perfectly handled._
+
+The above have their structure looking like a combination of multiple pipelines. However, models are never 'completed' - they (need to) evolve constantly to match their changing surroundings. 
+
+_There’s a continual learning cycle that happens as new data comes in and the model updates its understanding of the world and the new model gets deployed to production and the old model gets sunsetted._
+
+**A better diagram by Larysa Visengeriyeva**
+
+<img width="996" alt="image" src="https://github.com/user-attachments/assets/fef6ec55-0318-46a5-b8f2-6954a44f4c1d" />
+
+It captures the looping nature of the workflow much better than a linear diagram
+
+**The Future and Beyond for AIIA**
+
+_In the long run we’re looking to deliver the Kubernetes of ML, something that abstracts away all the concepts and communications between different layers of any kind of complex AI/Ml stack people can dream up._
+
+Their goal is to enable plug-and-play and *open-source* functionality for AI/ML systems that are flexible, fast, and agnostic to the underlying technology - making sure we don't end up locked by vendors. 
+
+They don't seem active on [YT](https://www.youtube.com/@ai-infrastructure-alliance/videos), but I subscribed 😆
+
+## [Towards CRISP-ML(Q): A Machine Learning Process Model with Quality Assurance Methodology](https://arxiv.org/pdf/2003.05155)
+
+I saved this paper from the same EU AI Act book as well. In the book the author referenced this CRISP-ML(Q) framework and I wanted to check the paper ~
+
+_Organizations expect to double the number of machine learning (ML) projects within a year._ - the increase in models in various industries demands a standardized process to improve project success rates and efficiency
+
+They talk about the *current* framework - _CRISP-DM focuses on data mining and does not cover the application scenario of ML models inferring real-time decisions over a long period of time_
+
+<img width="821" alt="image" src="https://github.com/user-attachments/assets/8dae036e-cbf9-4238-8f6e-ca8b2f934cc4" />
+
+Here is a comparison between the two:
+
+<img width="366" alt="image" src="https://github.com/user-attachments/assets/4abf37df-c333-4dec-a1c0-7aa4de8d504b" />
+
+CRISP-ML(Q) focuses on a 6-step approach:
+
+1. Business and Data understanding (merged): defining business & ML objectives, data collection, and feasibility assessment
+  * define the scope of the ML application
+  * success criteria (business, ML, economic)
+  * feasibility (applicability, legal constraints, requirements on the app)
+  * data collection (data version control, cost, time)
+  * DQ verification (data description, data requirements, data verification)
+  * review of output docs
+2. Data prep: cleaning, constructing, standardizing, and selecting relevant features
+  * select data
+  * clean data (noise reduction, data imputation)
+  * construct data (feature eng, data augmentation)
+  * standardize data (file format, normalisation)
+3. Modeling: selecting appropriate models, incorporating domain knowledge, training, and assuring reproducibility
+  * literature research on similar problems
+  * define quality measures of the model
+  * model selection
+  * incorporate domain knowledge
+  * training
+  * using unlabaled data and pre-trained models
+  * model compression
+  * ensemble methods
+  * reproducability (model, result, experiment documentation)
+4. Evaluation: validating performance, robustness, explainability, and comparing results to success criteria
+  * validate performance
+  * determine rebustness
+  * increase explainability for ML practitioners and end users
+  * compare results with defined success criteria
+5. Deployment: defining inference hardware, testing under production conditions, ensuring user acceptance, and planning deployment strategy
+  * define inference hardware
+  * model eval under prod conditions
+  * assure user acceptance and usability
+  * min the risks of unforseen errors
+  * deployment strategy
+6. Monitoring and maintenance: monitoring model performance, data drift, hardware degradation, and implementing update procedures
+  * non-stationary data distribution
+  * degradation of hardware
+  * need to update any part of the system when needed if we notice some shift, some change happens
+
+CRISP-ML(Q) aknowledges that MLOps is about building systems that can learn and adapt on the fly - data isn't static. 
+
+Quality assurance methodology is introduced in each phase and task of the process model:
+
+<img width="814" alt="image" src="https://github.com/user-attachments/assets/f83ed2de-f4d8-448f-b83d-cfa5134b5936" />
+
+## Listening to papers in podcast mode
+
+There is a feature in [google's notebook LM](https://notebooklm.google.com/) that takes your data sources (1 paper pdf in this case) and makes a podcast-like talk about it. There's also an interactive mode in beta where you can click join and ask Qs as if you are part of the podcast 
+
+<img width="1506" alt="image" src="https://github.com/user-attachments/assets/7f6fa225-89fd-4439-b090-713d9af38323" />
+
+Absolutely amazing. 'Reading' research has never been easier 😆
+
+---
+
+That is all for today!
+
+Happy Christmas Eve!
+
+See you tomorrow :)

From a11925a7ac9a02816767f9da262b0c8dd05050c8 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 25 Dec 2024 15:08:15 +0900
Subject: [PATCH 563/638] add closing words

---
 _posts/2024-09-27-day-270.md | 24 +++---------------------
 1 file changed, 3 insertions(+), 21 deletions(-)

diff --git a/_posts/2024-09-27-day-270.md b/_posts/2024-09-27-day-270.md
index 467a38967..74385ee14 100644
--- a/_posts/2024-09-27-day-270.md
+++ b/_posts/2024-09-27-day-270.md
@@ -127,26 +127,8 @@ Today I covered:
 
 One thing I was thinking when reading the KL-convergence part was why are we introducing this at the start of the book when the next thing we will talk about is regression in a single neuron. KL-convergence is not really applicable to regression. From a beginner's point of view, reading about KL-convergence so early might be intimidating, and a better time to include it might be when it's actually used, like before variational autoencoders. Introducing it in a more relevant context could help learners better understand its significance and application. But as it is part of Information Theory, it is also understandable to include it in that part of the book. (And I have seen this pattern in other books as well but it's fine, just commenting)
 
+---
 
+That is all for today!
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+See you tomorrow :)

From 67261e9245b2a61fc0a46b2fc2bd8b49530135a3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 25 Dec 2024 15:16:00 +0900
Subject: [PATCH 564/638] add closing remarks

---
 _posts/2024-10-24-day-297.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/_posts/2024-10-24-day-297.md b/_posts/2024-10-24-day-297.md
index ec6964b10..3b8843f44 100644
--- a/_posts/2024-10-24-day-297.md
+++ b/_posts/2024-10-24-day-297.md
@@ -111,3 +111,9 @@ Which returns:
 ```
 
 And apparently for spark ML models the features need to be put in a vector, so that's why this is returned like that. I need to think if this is ok for me and will continue tomorrow.
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 91b9927613b398b6d2777f8276ec6e6305de8edc Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 25 Dec 2024 23:24:52 +0900
Subject: [PATCH 565/638] Create 2024-12-25-day-359.md

---
 _posts/2024-12-25-day-359.md | 384 +++++++++++++++++++++++++++++++++++
 1 file changed, 384 insertions(+)
 create mode 100644 _posts/2024-12-25-day-359.md

diff --git a/_posts/2024-12-25-day-359.md b/_posts/2024-12-25-day-359.md
new file mode 100644
index 000000000..bbab7f993
--- /dev/null
+++ b/_posts/2024-12-25-day-359.md
@@ -0,0 +1,384 @@
+---
+layout: post
+title: (Day 359) Creating a list of books and courses I covered in the last 358 days
+categories: [mlops,theory,applying-knowledge,data-eng,traditional-machine-learning]
+---
+
+# Hello :) Today is Day 359!
+A quick summary of today:
+* applying analytical patterns day 2
+* streamed - looking back at books/courses I read/did over the past year for this blog
+* covered a 1hr course on MLOps on databricks by Maria Vechtomova
+
+## Applying analytical patterns [day 2](https://www.youtube.com/watch?v=7O7DH0ux0x4) (from Zach Wilson's bootcamp)
+
+My notes:
+
+![Applying analytical patterns 2 P1](https://github.com/user-attachments/assets/a9b0657e-a938-4ea2-baa8-843b690eac2d)
+![Applying analytical patterns 2 P2](https://github.com/user-attachments/assets/d8c52528-b5e3-45fc-918c-5b91af3153e2)
+
+Lab code:
+
+```sql
+-- applying analytical patterns lab 2
+-- find how many of the people that go to the signup page end up signing up
+
+with deduped_events as (
+    select
+        user_id,
+        url,
+        event_time,
+        date(event_time) as event_date
+    from events
+    where user_id is not null
+    group by user_id, url, event_time, date(event_time)
+    order by user_id, event_time
+),
+    selfjoined as (
+        select
+            d1.user_id,
+            d1.url,
+            d2.url as destination_url,
+            d1.event_time,
+            d2.event_time
+        from deduped_events d1
+            join deduped_events d2
+                on d1.user_id = d2.user_id
+                and d1.event_date = d2.event_date
+                -- count the visit if it happens on the same day
+                and d2.event_time > d1.event_time
+    ),
+    userlevel as (
+        select
+            user_id,
+            url,
+            count(1) as number_of_hits,
+            sum(case when destination_url = '/api/v1/user' then 1 else 0 end) as converted
+        from selfjoined
+        group by user_id, url
+    )
+select
+    url,
+    sum(number_of_hits) as num_hits,
+    sum(converted) as num_converted,
+    cast(sum(converted) as real) / count(number_of_hits) as pct_converted
+from userlevel
+group by url
+having sum(number_of_hits) > 500; -- filter random url hits
+
+-- look at website events and what type of device is the most common to hist the website
+
+create table device_hits_dashboard as
+with events_augmented as (
+    select coalesce(d.os_type, '(overall)') as os_type,
+           coalesce(d.device_type, '(overall)') as device_type,
+           coalesce(d.browser_type, '(overall)') as browser_type,
+           url,
+           user_id
+    from events e
+        join devices d on e.device_id = d.device_id
+)
+
+select
+       case
+           when grouping(os_type) = 0
+               and grouping(device_type) = 0
+               and grouping(browser_type) = 0
+               then 'os_type__device_type__browser'
+           when grouping(browser_type) = 0 then 'browser_type'
+           when grouping(device_type) = 0 then 'device_type'
+           when grouping(os_type) = 0 then 'os_type'
+       end as aggregation_level,
+       coalesce(os_type, '(overall)') as os_type,
+       coalesce(device_type, '(overall)') as device_type,
+       coalesce(browser_type, '(overall)') as browser_type,
+       count(1) as number_of_hits
+from events_augmented
+group by grouping sets (
+        (browser_type, device_type, os_type),
+        (browser_type),
+        (os_type),
+        (device_type)
+    )
+order by count(1) desc;
+
+-- example usage (it's pre-aggregated)
+select * from device_hits_dashboard
+where aggregation_level = 'os_type__device_type__browser';
+```
+
+## [Streamed](https://www.youtube.com/watch?v=kkp5kad64io&t=461s)
+
+I went over 358 days worth of posts and collected a list of books and courses 
+
+I will update it with the books/courses I read/do until the end of day 366 and maybe add books I will definitely read after the blog ends. I plan to post each part on linkedin on 2nd of Jan, or maybe do like a top 5/10 of books/courses.
+
+```
+## Book
+
+Mathematics for Machine learning
+Deep Learning by MIT 
+Build a LLM from scratch
+An Introduction to Statistical Learning
+Foundations and Linear Models part from Probabilistic Machine Learning: An Introduction
+What are embeddings? (Vicki Boykis)
+Dive into Deep Learning
+Grokking Machine Learning
+Outlier Detection in Python
+A Primer For The Mathematics Of Financial Engineering
+Design a Machine Learning System (From Scratch)
+Multivariate Statistical Methods: A Primer
+Effective Data Science Infrastructure
+MLOps blog: https://ml-ops.org/
+Fundamentals of Data Engineering
+Introducing MLOps
+Deep Learning at Scale
+Machine Learning Algorithms in Depth
+Databricks Certified Associate Developer for Apache Spark Using Python
+Streaming Databases
+The Little Book of Deep Learning
+The Elements of Statistical Learning
+Deep Learning (Bishop and Bishop)
+The Matrix Calculus You Need For Deep Learning (arxiv paper)
+Graph Algorithms for Data Science
+Before Machine Learning book series
+The 100 page ML book
+MLE (Andriy Burkov)
+Financial Data Engineering
+Machine Learning for Financial Risk Management with Python
+Optimization Algorithms
+Financial calculus (Baxter and Rennie)
+Data Analysis with Python and PySpark
+LLM Engineer's Handbook
+scikit-learn User Guide
+Designing Machine Learning Systems
+Designing Data-Intensive Applications
+Data Quality Fundamentals
+Apache Iceberg: The Definitive Guide
+Learning Spark 2nd ed.
+Data Pipelines Pocket Reference
+Stream Processing with Apache Flink
+The 100 page LM book
+The Little Book of ML Metrics
+Effective Machine Learning Teams
+Data Engineering with dbt
+The AI Engineer's Guide to Surviving the EU AI Act
+
+## Course
+
+caltech - ML CS 156 by Prof. Abu-Mostafa: https://youtu.be/mbyG85GZ0PI?list=PLD63A284B7615313A
+Statistics with Python in Codecademy: https://www.codecademy.com/learn/paths/master-statistics-with-python
+Microsoft Beginner ML course: https://github.com/microsoft/ML-For-Beginners
+ML course on Kaggle Learn: https://www.kaggle.com/learn
+Machine Learning specialization by Andrew Ng: https://www.coursera.org/specializations/machine-learning-introduction
+DeepLearning.AI TensorFlow Certfication Specialization course: https://www.coursera.org/learn/introduction-tensorflow?specialization=tensorflow-in-practice
+Andrej Karpathy's NN series: https://www.youtube.com/watch?v=VMj-3S1tku0&list=PLAqhIrjkxbuWI23v9cThsA9GvCAUhRvKZ&ab_channel=AndrejKarpathy
+DL Specialization on Coursera: https://www.coursera.org/specializations/deep-learning
+NLP Specialization on Coursera: https://www.coursera.org/specializations/natural-language-processing
+TensorFlow: Data and Deployment Specialization: https://www.coursera.org/specializations/tensorflow-data-and-deployment
+Tensorflow Advanced techniques specialization: https://www.coursera.org/specializations/tensorflow-advanced-techniques
+IBM's DL with PyTorch: https://www.coursera.org/learn/deep-neural-networks-with-pytorch
+CS231n Winter 2016: https://youtu.be/NfnWJUyUJYU?list=PLkt2uSq6rBVctENoVBg1TpCC7OQi31AlC
+Aladdin Persson's YT videos: https://www.youtube.com/@AladdinPersson/playlists
+AI 504 by Prof. Choi: https://www.youtube.com/watch?v=TG3OJQjN-l4&list=PLLENHvsRRLjAmAjc8mV0f9C6i8Gh308SS&ab_channel=EdwardChoi
+GAN specialization on Coursera: https://www.coursera.org/specializations/generative-adversarial-networks-gans
+AI503 Math for AI from KAIST: https://alinlab.kaist.ac.kr/ai503_2021.html
+Started Stanford University's cs231n: https://cs231n.github.io/
+Becoming a backprop ninja by Andrej Karpathy (deserves a separate mention): https://youtu.be/q8SA3rM6ckI
+Stanford's CS224N: https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1234/
+11-711 Advanced NLP by CMU: https://phontron.com/class/anlp2024/
+ACL 2023 talk on retrieval-based LMs: https://acl2023-retrieval-lm.github.io/
+Local Retrieval Augmented Generation (RAG) from Scratch: https://youtu.be/qN_2fnOPY-M
+XCS224W ML with Graphs by Stanford University: https://online.stanford.edu/courses/xcs224w-machine-learning-graphs
+GCP's Digital Leader Learning Path: https://www.cloudskillsboost.google/paths/9
+Open Source Models with Hugging Face: https://www.deeplearning.ai/short-courses/open-source-models-hugging-face/
+AI for Global Goals OxML Summer School series: https://www.oxfordml.school/
+Stanford's CS246: Mining Massive Datasets: https://web.stanford.edu/class/cs246/
+Stanford's CS109 with Prof. Chris Piech: https://www.youtube.com/playlist?list=PLoROMvodv4rOpr_A7B9SriE_iZmkanvUg
+MLOps Zoomcamp by DataTalksClub: https://github.com/DataTalksClub/mlops-zoomcamp
+LLM Zoomcamp by DataTalksClub: https://github.com/DataTalksClub/llm-zoomcamp
+Darshil Parmar's DE projects: https://www.youtube.com/@DarshilParmar/playlists
+Data Eng Zoomcamp by DataTalksClub: https://github.com/DataTalksClub/data-engineering-zoomcamp
+Stock Market Analysis Zoomcamp: https://github.com/DataTalksClub/stock-markets-analytics-zoomcamp
+EvidentlyAI's observability course: https://www.evidentlyai.com/ml-observability-course
+Intro to LangGraph by LangChain: https://academy.langchain.com/courses/intro-to-langgraph
+Neo4j's free certification courses: https://graphacademy.neo4j.com/categories/
+DeepLearning.AI Data Engineering Professional Certificate
+The Complete Mathematics of Neural Networks and Deep Learning: https://youtu.be/Ixl3nykKG9M
+ Credit Risk Modelling in Python: https://www.udemy.com/course/credit-risk-modeling-in-python/
+dbt fundamentals: https://learn.getdbt.com/courses/dbt-fundamentals
+scikit-learn's MOOC: https://inria.github.io/scikit-learn-mooc/
+Zach Wilson's free DE boot camp: https://techcreator.io/
+Free streaming courses on Confluent: https://training.confluent.io/channeldetail/apache-kafka-fundamentals-and-accreditation
+Apache Iceberg/Lakehouse course on Dremio University: https://university.dremio.com/
+Airflow courses by Astronomer: https://academy.astronomer.io/
+MadeWithML's MLOps course: https://madewithml.com/
+
+## People
+
+Andrew Ng
+Andrej Karpathy
+Andriy Burkov
+Aladdin Persson
+Alexey Grigorev
+Vincent D. Warmerdam (probabl)
+Zach Wilson
+```
+
+## [MLOps course on LinkedIn learning by Maria Vechtomova](https://www.linkedin.com/learning/mlops-with-databricks/reduce-time-to-production-with-mlops)
+
+I saw that Maria Vechtomova posted a 'free' course on MLOps on linkedin learning so I signed up for a 30-day free trial on the website and decided to check it out ~
+
+Learning objectives:
+
+* explain main components and principles required to deploy machine learning models to production on Databricks
+* identify how to use experiment tracking system, model registry, feature engineering, model/feature serving, and other features required to deploy ML applications
+* articulate how to package your Python code using best practices and deploy your project using Databricks Asset Bundles.
+* review how to monitor your ML applications
+
+### 1. MLOps components and principles
+
+Interesting trend:
+
+<img width="765" alt="image" src="https://github.com/user-attachments/assets/0a7c80f9-589a-4bbf-a1b5-6626c171db8d" />
+
+DS became popular and soon after MLOps. Not surprising, I guess. Companies wanted to build models and then soon after they realised they need a more consistent way of developing, delivering, and maintaining them. 
+
+The minimal set of tools for MLOps:
+
+* version control
+* CI/CD
+* orchestration
+* model registry
+* container registry
+* compute
+* serving
+* monitoring
+* data version control
+* other (feature store, labeling tool, exp. tracking, responsible AI, vector db, llm evaluation, llm frameworks, model hub)
+
+There are many different options for each of the above
+
+* reuse the tools available in your organization
+* only add tools if needed
+* the more tools you have, the more complex migrations you'll have to go through
+
+**Principles**
+
+Real business impact is involved if ML model predictions are unavailable or of bad quality. The goal of MLOps is to minimise the risk of things going wrong
+
+Four main principles:
+
+* documentation (business goals and KPIs, data definition, ML system architecture, the choice of ML models)
+* code quality (review processes, code quality checks, unit and integration tests)
+* traceability and reproducability (for a model this is the ease of finding: corresponding code and commits on git, the infrastructure used for training and serving, ML model artifacts, what data wasa used to train the model; reproducability: the same model with the same data and code can be generated, rollback is easy)
+* monitoring and alerting (infrastructure health and costs, application health, response time and response codes, model eval and business metrics, data and model drift)
+
+The first two are related to people and processes, while the last two are related to tools. 
+
+**Databricks for MLOps**
+
+It offers:
+
+* orchestration
+* model registry
+* compute
+* serving
+* vector search
+* exp. tracking
+* prompt engineering
+* feature engineering
+* monitoring
+
+### 2. MLflow
+
+Offered a managed version by databricks
+
+I could not follow the code exactly as I do not have a proper databricks subscription. I will create one for her course at the end of Jan.
+
+But that did not stop me from running the code locally with a local mlflow server 😆.
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/c6570ce8-5c95-414c-a643-e42998d72833" />
+
+(I cloned the repo and started running files locally). But I faced some issues with pyspark as the env was done all with uv and I kept getting `RuntimeError: Only remote Spark sessions using Databricks Connect are supported. Use DatabricksSession.builder to create a remote Spark session instead.` I guess I could try to do the whole thing locally ~ I just continued watching at this point. She even says that some parts don't work well in vs code so it's better to run them in databricks ~ 😆 So as most of the time ~ I will focus on the ideas rather than the exact implementation/code
+
+The code for all of it is on [github](https://github.com/LinkedInLearning/mlops-with-databricks-2019584)
+
+### 3. Feature engineering
+
+Databricks feature engineering concepts
+
+* `FeatureTable`: a table stored as a Delta table in Unity catalog that contains ML model features
+* `FeatureFunction`: a class that specifies a Python UDF in Unity catalog
+* `FeatureLookup`: a class that specifies how to look up features in the feature table
+* `TrainingSet`: a class that defines how to construct a train set using a feature table, FeatureFunction, and FeatureLookup
+* `FeatureSpec`: feature specification defined as a list containing instances of FeatureLookup and FeatureFunction
+
+Feature engineering client functionality
+
+* create, update, and delete feature tables and feature specifications
+* create a training dataset
+* log an MLflow model packaged with information on how a training set was built
+
+### 4. Feature and Model serving
+
+<img width="607" alt="image" src="https://github.com/user-attachments/assets/b55a366f-d04f-4c47-ac6a-7aedf1a8c974" />
+
+<img width="634" alt="image" src="https://github.com/user-attachments/assets/7b6a9534-7beb-427a-9dc3-97c7f3ea7111" />
+
+**When to use feature serving**
+
+* predictions are precomputed and need to be accessed at the moment of request
+* a simple function that does not require a model artifact must be computed at the moment of request
+
+**When to use model serving**
+
+* there is a model artifact
+* optionally, some features must be retrieved at the moment of request
+
+### 5. Deploy apps using Databricks Asset Bundles
+
+Ways to deploy a databricks job:
+
+* terraform
+* databricks APIs
+* databricks asset bundles (DABs) (recommended way as it manages dependencies and files required to be manually handles if using the first two options)
+
+DAB is deployed by defining a pipeline in a yaml file where we essentially define a job, its schedule, tags, where to run it (compute), and its tasks + there are extra things we can specify ~
+
+This is how it looks like on Maria's databricks:
+
+<img width="1510" alt="image" src="https://github.com/user-attachments/assets/58088418-910d-42e3-9ae0-753586854002" />
+
+### 6. Monitoring using Inference Tables and Lakehouse Monitoring
+
+**What must be monitored**
+
+* application and infrastructure health
+* application performance and costs
+* model and data drift
+* business value
+* for some use cases, fairness and bias
+
+Monitoring can be done again in the `databricks.yml` file as a job.
+
+**With Lakehouse monitoring we can**
+
+* monitor data drift
+* monitor model drift (if the GT is provided)
+
+**Lakehouse Monitoring components**
+
+* profile metrics table (Delta table with summary stats for each slicing window)
+* drift metrics table (Delta table with with stats and different drift detection metrics)
+* customisable dashboard
+
+Overall ~ awesome 🔥
+
+---
+
+That is all for today!
+
+Merry Christmas! 
+
+See you tomorrow :)

From f26d059304d68d1e6c71ffca464c234a3cf1b2f7 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 26 Dec 2024 22:09:46 +0900
Subject: [PATCH 566/638] Create 2024-12-26-day-360.md

---
 _posts/2024-12-26-day-360.md | 452 +++++++++++++++++++++++++++++++++++
 1 file changed, 452 insertions(+)
 create mode 100644 _posts/2024-12-26-day-360.md

diff --git a/_posts/2024-12-26-day-360.md b/_posts/2024-12-26-day-360.md
new file mode 100644
index 000000000..b5105c9ef
--- /dev/null
+++ b/_posts/2024-12-26-day-360.md
@@ -0,0 +1,452 @@
+---
+layout: post
+title: (Day 360) New book - Architecting Data and Machine Learning Platforms + some bootcamp homework
+categories: [mlops,theory,applying-knowledge,data-eng]
+---
+
+# Hello :) Today is Day 360!
+A quick summary of today:
+* read chapters 1 and 2 of Architecting Data and Machine Learning Platforms
+* did Q2(grouping sets) and Q3(window functions) from Zach Wilson's applying analytical patterns homework
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 1 - Modernizing Your Data Platform: An Introductory Overview
+
+### The data lifecycle
+
+**The journey**
+
+* collect
+* store
+* process/transform
+* analyse/visualise
+* activate (decisions can be made)
+
+![image](https://github.com/user-attachments/assets/4c9b006f-eec0-4918-993d-ccddbdbc5b99)
+
+### Limitations of traditional approaches
+
+Traditional data ecosystems often consist of isolated, task-specific solutions, leading to data silos that hinder insight generation and efficient collaboration across business units. Custom-built solutions exacerbate challenges in scalability, business continuity, disaster recovery, privacy, and data auditing. A unified data platform, particularly a cloud-based one, addresses these limitations by enabling consistent, scalable, and reliable analytics and machine learning across organizational data. Leveraging managed services minimizes infrastructure maintenance, allowing organizations to focus on business objectives.
+
+**Antipattern: breakding down silos through ETL**
+
+![image](https://github.com/user-attachments/assets/ce0176ca-3fac-4cfd-a75f-c6b086384be9)
+
+Organizations often use ETL tools to address data silos by transferring and transforming data into centralized analytics stores (data warehouses or data lakes). However, relying on ETL poses significant challenges:
+
+* data quality issues: etl processes may extract incorrect data due to insufficient understanding by the tool's creators
+* latency: etl introduces delays, making data stale, though streaming etl mitigates this issue
+* bottlenecks: etl creation and maintenance require skilled data engineers, slowing down scalability
+* high maintenance costs: routine troubleshooting, infrastructure updates, and schema changes increase operational overhead
+* data gaps: errors often lead to downtime and incomplete records
+* governance issues: redundant and divergent etl processes result in inconsistent data
+* inefficiency and environmental impact: always-on infrastructure increases costs and carbon footprint
+
+**Antipattern: centralisation of control**
+
+Organizations attempting to address data silos by centralizing control under IT face significant challenges:
+
+* IT limitations: managing diverse technologies across multiple storage systems (on-premises and cloud) strains IT capabilities and resources. Defining consistent security, governance, and auditing across systems is complex. Access requests grow with the number of source and target systems, creating scalability issues.
+
+* analytics challenges: accessing the right data becomes difficult and costly, often leading to unnecessary ETL tasks. Pre-prepared data may lack recent updates or comprehensive details, limiting analytic flexibility. Data governance can restrict data access, reducing analytic agility
+
+* business limitations: ensuring reliable and quality data access is a struggle. Organizations must balance data quality with the extent of data provided. IT's limited understanding of business needs can slow innovation, leading to bottlenecks and delays
+
+**Antipattern: data marts and hadoop**
+
+**Antipattern: Data Marts and Hadoop - Summary**
+
+To address challenges with siloed, centrally managed systems, organizations adopted two problematic approaches: data marts and ungoverned data lakes.  
+
+1. Data marts:  
+   - extracted subsets of enterprise data into structured models for specific workloads (e.g retail, banking)  
+   - enabled business users to derive insights (e.g revenue trends, user behavior)  
+   - relied on costly, on-premises technologies with scalability limits, requiring expensive upgrades for larger datasets  
+   - it struggled with infrastructure scaling, talent shortages, and rising costs  
+
+2. Data lakes via hadoop:  
+   - introduced distributed processing (horizontal scaling) on low-cost servers, making advanced analytics affordable 
+   - served as central repositories for raw data, enabling large-scale analytics and data science
+   - required extensive knowledge of hadoop ecosystems (e.g., hive, spark, presto) for access, adding complexity
+   - became ungoverned, leading to poor data quality and low user adoption due to skills gaps and unclear data structures
+
+Both approaches struggled to deliver consistent roi due to high costs, maintenance challenges, and difficulties in managing data quality and accessibility.
+
+### Creating a unified analytics platform
+
+**Cloud instead of on-premise**
+
+* reduce cost by taking advantage of new pricing models (pay-per-use model)
+* speed up innovation by taking advantage of best-of-breed technologies
+* scale on-premises resources using a “bursting” approach
+* plan for business continuity and disaster recovery by storing data in multiple zones and regions
+* manage disaster recovery automatically using fully managed services
+* centralized governance and access management
+* increased productivity and reduced operational costs
+* greater data sharing across the organization
+* extended access by different personas
+* reduced latency of accessing data
+
+**Drawbacks of data marts and data lakes**
+
+* management challenges: difficult to manage and expensive to scale, especially legacy systems like Teradata and Netezza
+* lack of modern capabilities: require additional work to integrate AI/ML or real-time features
+* limited access: analysts lack direct access to raw data and data owners, restricting hypothesis testing and deeper insights
+
+
+* storage and cost issues: unpredictable data production makes storage planning difficult, and provisioning for peak computational demand is expensive
+* maintenance overhead: engineers spend significant time on upkeep, reducing time for feature development, with high total cost of ownership often yielding negative ROI
+* governance problems: inconsistent security models create silos, limiting data and model sharing across teams
+* skill gap: business users lack programming expertise to leverage data lakes, while technical users remain disconnected from business objectives, leading to missed opportunities for insights and growth
+
+**Lakehouses**
+
+![image](https://github.com/user-attachments/assets/fa1ccb0f-1826-4b6c-8c78-55cf38715070)
+
+They offer:
+
+* decoupling of storage and compute that enable inespensive, virtually unlimited, and seamlessly scalable storage; stateless, resilient compute; ACID-compliant storage operations; a logical db storage model, rather than a physical
+* data governance
+* support for data analysis via the native integration with BI tools
+* native support of the typical multiversion approach of a data lake approach
+* data storage and management via open formats like parquet and Iceberg
+* support for different data types in the structured or unstructured format
+* streaming capabilities with the ability to handle real-time analysis of the data
+* enablement of a diverse set of applications varying BI to ML
+
+**Data mesh**
+
+![image](https://github.com/user-attachments/assets/c49f8244-e6cd-49cd-a80a-7c26d64992a2)
+
+Offers:
+
+* domain responsibility
+* on-demand solutions
+* organisational alignment
+
+
+Challenges: managing complex architectures that span multiple environments requires careful planning and execution
+
+### Hybrid cloud
+
+**Reasons why hybrid is necessary**
+
+* data residency regulations
+* legacy investments
+* transition
+* want to meet specific needs
+* benefit from combining different clouds
+
+**Challenges**
+
+* governance
+* access control
+* workload interoperability
+* data movement
+* skill sets
+* economics
+
+### Cloud for AI
+
+It is important to ensure that data platforms will be future proof in being capable of supporting AI use cases given its rise.
+
+AI requires vast data to train models, with more data leading to better performance. Large-scale AI models, such as image classification and language models, need high-quality data and custom hardware (GPUs, TPUs) to process it. Cloud platforms provide the compute power and resources necessary for significant AI breakthroughs.
+
+Cloud makes AI more accessible through:
+
+* ML APIs (pre-built models)
+* customisable ML models (AutoML)
+* simpler ML (i.e. BigQuery enable ML model training using SQL)
+* real-time processing architecture
+* ML lifecycle tools
+
+### Core principles
+
+The authors suggest 5 key design principles for a data analytics stack:
+
+* delivery serverless analytics, not infrastructure
+* embed end-to-end ML
+* empower analytics across the entire data lifecycle
+* enable open source software tech
+* build for growth
+
+## [Nobel Minds 2024](https://www.youtube.com/watch?v=1tELlYbO_U8&pp=ygUTMjAyNCBub2JlbCBsb3JlYXRlcw%3D%3D)
+
+I saw that the 2024 Nobel laureates had a talk and that Geoffrey Hinton and Demis Hassabis attended (not all laureates attended) and decided to watch. Hinton kept talking about being careful with AI and bad actors and that with AI the rich can keep getting richer - I was expecting him to name drop OpenAI or even Sam Altman 😆 but he didn't. Even thought AI is the most popular topic, I am glad the non-AI laureates got to talk about their own research as well - economics and genetics. 
+
+## Applying analytical patterns homework Q2 and Q3
+
+### Q2 - grouping sets
+
+```
+- A query that uses `GROUPING SETS` to do efficient aggregations of `game_details` data
+  - Aggregate this dataset along the following dimensions
+    - player and team
+      - Answer questions like who scored the most points playing for one team?
+    - player and season
+      - Answer questions like who scored the most points in one season?
+    - team
+      - Answer questions like which team has won the most games?
+```
+
+My answer (so far)
+
+```sql
+create table hw_2_table as
+with game_data as (
+    select
+        gd.player_name as player_name,
+        gd.team_abbreviation as team_abbreviation,
+        g.season as season,
+        gd.game_id as game_id,
+        gd.pts as player_pts,
+        case
+            when (gd.team_id = g.home_team_id and home_team_wins = 1)
+                     or (gd.team_id <> g.home_team_id and home_team_wins = 0) then 1 else 0
+        end as won_game
+    from game_details gd
+        left join games g
+            on gd.game_id = g.game_id
+), grouped_data as (
+    select
+        coalesce(cast(player_name as text), '(overall)') as player_name,
+        coalesce(cast(team_abbreviation as text), '(overall)') as team_abbreviation,
+        coalesce(cast(season as text), '(overall)') as season,
+        sum(player_pts) as player_total_pts,
+        count(distinct (case when won_game = 1 then game_id end)) as team_total_wins,
+        case
+            when grouping(player_name, team_abbreviation) = 0 then 'player__team'
+            when grouping(player_name, season) = 0 then 'player__season'
+            when grouping(team_abbreviation) = 0 then 'team'
+        end as agg_level
+    from game_data
+    group by grouping sets (
+            (player_name, team_abbreviation),
+            (player_name, season),
+            (team_abbreviation)
+        )
+)
+select * from grouped_data;
+
+-- Who scored the most points playing for one team? -- Giannis Antetokounmpo 15591 points with MIL
+select
+    player_name,
+    team_abbreviation,
+    player_total_pts
+from hw_2_table
+where agg_level = 'player__team'
+order by player_total_pts desc
+limit 1;
+-- Who scored the most points in one season? -- James Harden 3247 points in 2018
+select
+    player_name,
+    player_total_pts,
+    season
+from hw_2_table
+where agg_level = 'player__season'
+order by player_total_pts desc
+limit 1;
+-- Which team has won the most games? -- GSW 445 games
+select
+    team_abbreviation,
+    team_total_wins
+from hw_2_table
+where agg_level = 'team'
+order by team_total_wins desc
+limit 1;
+```
+
+### Q3 - window functions
+
+The question:
+
+```
+- A query that uses window functions on `game_details` to find out the following things:
+  - What is the most games a team has won in a 90 game stretch? 
+  - How many games in a row did LeBron James score over 10 points a game?
+```
+
+My answer (so far)
+
+```sql
+-- What is the most games a team has won in a 90 game stretch? -- 79
+with game_data as (
+    select
+        gd.player_name as player_name,
+        gd.team_abbreviation as team_abbreviation,
+        g.season as season,
+        gd.game_id as game_id,
+        g.game_date_est as game_date_est,
+        gd.pts as player_pts,
+        case
+            when (gd.team_id = g.home_team_id and home_team_wins = 1)
+                     or (gd.team_id <> g.home_team_id and home_team_wins = 0) then 1 else 0
+        end as won_game
+    from game_details gd
+        left join games g
+            on gd.game_id = g.game_id
+), wins_last_90 as (
+    select
+        team_abbreviation,
+        sum(won_game) over (order by game_date_est rows between current row and 89 following) as wins_last_90_games
+    from game_data
+)
+select
+    team_abbreviation,
+    max(wins_last_90_games) as wins_last_90_games
+from wins_last_90
+group by team_abbreviation
+order by wins_last_90_games desc
+limit 1;
+
+-- How many games in a row did LeBron James score over 10 points a game? -- 62
+with game_data as (
+    select
+        gd.player_name as player_name,
+        g.game_date_est as game_date_est,
+        case
+            when gd.pts > 10 then 1 else 0
+        end as scored_over_10
+    from game_details gd
+        left join games g
+            on gd.game_id = g.game_id
+    where gd.player_name = 'LeBron James'
+), prev_game as (
+    select
+        *,
+        lag(scored_over_10, 1) over (order by game_date_est) as prev_scored_over_10
+    from game_data
+), with_streak as (
+    select
+        *,
+        sum(case when prev_scored_over_10 = scored_over_10 then 0 else 1 end) over (order by game_date_est) as streak_identifier
+    from prev_game
+), with_final as (
+    select
+        *,
+        sum(scored_over_10) over (partition by streak_identifier order by game_date_est) as max_consecutive_scored_over_10
+    from with_streak
+    order by game_date_est
+)
+select
+    player_name,
+    max(max_consecutive_scored_over_10) as consecutive_scored_over_10
+from with_final
+group by player_name
+order by consecutive_scored_over_10 desc;
+```
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 2 - Strategic Steps to Innovate with Data
+
+![image](https://github.com/user-attachments/assets/650d6b7f-0508-429f-b38f-885da697eca0)
+
+### Step 1: Strategy and Planning
+
+Aligning data strategy with business goals, identifying stakeholders, and establishing a change management process are crucial for success.
+
+* business-driven goals: moving beyond cost-saving, the focus should be on how data can drive key business decisions and transformations. It is important to identify the decisions that will be made using the data and the metrics by which you can know the transformation has been successful
+* stakeholder identification: engaging both with IT and business unit leaders ensures alignment of incentives and long-term support for the project
+* change management: effectively cascading and communicating the approach throughout the organization is crucial to gain buy-in from end users
+
+### Step 2: Reduce Total Cost of Ownership by Adopting a Cloud Approach
+
+Migrating to the cloud offers significant cost benefits through automation, right-sizing resources, and leveraging serverless capabilities.
+
+* cost savings: cloud providers leverage economies of scale and automation to offer lower infrastructure costs
+* right-sizing and autoscaling: scaling resources based on demand and usage eliminates overprovisioning and optimizes costs
+* serverless workloads: further cost optimization can be achieved by using serverless computing for specific tasks
+
+### Step 3: Break Down Silos
+
+Balancing decentralization and data accessibility is crucial - centralizing data storage while maintaining domain-specific ownership and leveraging cloud IAM for access control fosters collaboration and insights.
+
+* centralized storage, decentralized ownership: storing data in a central location like Azure Blob Storage, with each department managing their data in their own bucket, strikes a balance between control and accessibility
+* cloud IAM for access control: replacing legacy authentication mechanisms with cloud IAM ensures standardized and auditable access across the organization
+* semantic layer: a semantic layer, implemented through tools like Looker or Collibra, ensures consistent terminology and KPI calculations, further breaking down analytics silos. We need to encourage data analytics teams to build a semantic layer (so that vocabularies can be standardized and KPIs can be computed once and reused everywhere else) and apply governance through it
+
+### Step 4: Make Decisions in Context Faster
+
+The value of business decisions decreases with latency and distance, so a key goal for modernising a data platform should be to enable contextual decision-making capabilities such as geographic information systems (GIS), streaming, and machine learning (ML) without making copies of the data
+
+* batch to stream: data volumes grow exponentially, transitioning from batch to stream processing reduces latency, spreads computational load, and improves timeliness
+* contextual info: automating decision-making enhances user experience, particularly for occasional users. Embedding real-time, location-based visualizations into applications is a best practice
+* cost management: streaming can be costly, but combining batch and stream processing using tools like Apache Flink or managed services can reduce complexity and cost. Micro-batching is a viable alternative when near-real-time insights suffice
+
+### Step 5: Leapfrog with Packaged AI Solutions
+
+* use AI for predictive analysis
+* understanding and generating unstructured data
+* more personalised user experiences
+* use already available packaged solutions
+
+### Step 6: Operationalize AI-Driven Workflows
+
+The final stage of a data platform strategy is to automate workflows for solving business problems autonomously. This involves determining the desired level of automation and fostering a data-driven culture to achieve the organization’s goals
+
+![image](https://github.com/user-attachments/assets/184fff27-5279-4edf-aacf-db9273254666)
+
+* identifying the right balance of automation and assistance: organizations must experiment to find the right mix of automation and human involvement, which varies across industries
+* building a data culture: a data culture requires more than analytics platforms—it involves encouraging data-driven experiments, enhancing data literacy, rewarding innovation, and training employees on data tools like dashboards and analytics
+* populating a data science team
+
+### Step 7: Product Management for Data
+
+We should treat data as a product to maximize its value and impact on the organization. To treat data as a product, apply product management principles when conceiving and building data products. What product management principles? (1) Have a product strategy, (2) be customer-centric, (3) do lightweight product discovery, and (4) focus on finding market fit.
+
+The authors recommend 10 data practices for this:
+
+![image](https://github.com/user-attachments/assets/c9f100ba-c24e-447e-9284-b681a47d0f1d)
+
+1. Understand and maintain a map of data flows  
+   - maintain a high-level model of data flows for discoverability
+   - include multiple levels of granularity, showing data sources, transformations, and usage  
+   - document data attributes (e.g., PII, quality assurances) and production use cases
+
+2. Identify key metrics  
+   - align on key metrics (e.g., business KPIs, SLAs, engagement, satisfaction)  
+   - define targets for these metrics and track performance consistently
+
+3. Agreed criteria, committed roadmap, and visionary backlog  
+   - establish prioritization criteria for transparency  
+   - maintain a committed roadmap based on validated ROI  
+   - use a backlog to capture long-term vision and unscheduled priorities
+
+4. Build for the customers you have  
+   - understand user needs (i.e tools, technical expertise, data formats)  
+   - serve data via APIs, dashboards, warehouses, or feature stores, depending on the audience
+
+5. Don’t shift the burden of change management  
+   - minimize downstream disruption during schema or format changes  
+   - ensure backward compatibility or provide migration support
+   - integrate security and compliance measures proactively
+
+6. Interview customers to discover data needs  
+   - regularly engage with customers to identify problems and validate needs  
+   - avoid building speculative features; prioritize validated use cases
+
+7. Whiteboard and prototype extensively  
+   - collaborate with users during the design phase  
+   - validate ideas using minimum viable prototypes before production  
+   - scope requirements and expected ROI before committing to development
+
+8. Build only what will be used immediately  
+   - prioritize delivering essential features with immediate impact
+   - capture future needs in a backlog and build iteratively
+
+9. Standardize common entities and KPIs  
+   - create enriched, standardized datasets for shared, high-impact use cases  
+   - limit standardization to critical datasets to avoid over-engineering
+
+10. Provide self-service capabilities  
+   - balance standardization with flexibility  
+   - enable teams to customize datasets for domain-specific use cases
+   - adopt principles from data mesh architectures to promote self-sufficiency
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 301b519a363b56aa0a633a1deb6a561ddcbde42a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 27 Dec 2024 21:51:59 +0900
Subject: [PATCH 567/638] Create 2024-12-27-day-361.md

---
 _posts/2024-12-27-day-361.md | 768 +++++++++++++++++++++++++++++++++++
 1 file changed, 768 insertions(+)
 create mode 100644 _posts/2024-12-27-day-361.md

diff --git a/_posts/2024-12-27-day-361.md b/_posts/2024-12-27-day-361.md
new file mode 100644
index 000000000..cc80fc0cd
--- /dev/null
+++ b/_posts/2024-12-27-day-361.md
@@ -0,0 +1,768 @@
+---
+layout: post
+title: (Day 361) More of Architecting Data and Machine Learning Platforms + Some Flink Forward talks
+categories: [mlops,theory,data-eng]
+---
+
+# Hello :) Today is Day 361!
+A quick summary of today:
+* read a few chapters of the Architecting Data and Machine Learning Platforms book
+* watched talks on Apache Paimon from Flink Forward Jakarta '24
+
+## Architecting Data and Machine Learning Platforms - Chapter 3 - Designing for Your Data Team
+
+When designing a data platform, technical aspects like performance, cost, and integration of new approaches are important. However, addressing company culture, employee skills, and willingness to adapt is crucial for success. Implementing a data platform often requires employees to learn new skills, change workflows, and potentially transition into new roles.
+
+We should tailor our strategies to existing talent:
+
+* focus on existing strengths
+* consider workforce domain knowledge and evaluate the feasibility of upskilling
+* explore tools like generative AI or no-code platforms to empower a broader workforce
+
+However, public cloud tech and modern tools have blurred traditional distinctions between roles (data analysts, engineers, scientists) as analysts can now handle tasks previously requiring engineers, while blended ETL-ELT approaches enable flexibility. This convergence of skills leads to more efficient data processing.
+
+Organizations can generally be classified as:
+
+* data analysis-driven: focus on leveraging existing analytical skills
+* data engineering-driven: build robust engineering capabilities
+* data science-driven: prioritize advanced algorithm development
+
+### Data Analysis–Driven Organization
+
+DAs play a central role in decision making. It is important to note that whether an organization is analyst driven is not a black-and-white issue but rather a spectrum of overlapping characteristics
+
+**The vision**
+
+The vision for modern cloud data platforms is to empower analysts to perform advanced analytics using familiar interfaces like spreadsheets, SQL, and BI tools. Traditional ETL processes are being replaced by ELT, where data is ingested into the cloud and transformations occur within the cloud data warehouse (DWH). This approach simplifies data pipelines, reduces dependency on data engineering teams, and enables analysts to leverage SQL skills for faster, more direct data insights.
+
+To scale cloud data platform adoption, analysts should have access to user-friendly tools (like dbt) and interfaces (Excel, Tableau) for analytics and visualization. Modern BI tools, such as Power BI, Tableau, or Looker, are recommended for handling large datasets efficiently, overcoming the limitations of traditional spreadsheets. This setup accelerates data analysis and enhances the timeliness and usability of insights.
+
+**The personas**
+
+![image](https://github.com/user-attachments/assets/dbb5765b-38c0-4af7-99b3-1fe48b27446a)
+
+Data analysts:
+
+* focus on fulfilling business information needs by designing, maintaining, and transforming data
+* tasks include creating table layouts, reorganizing data, and generating reports with insights
+* skill expansion is crucial, emphasizing deep business domain knowledge and technical proficiency in analyzing large datasets using tools like SQL and BI platforms
+
+Business analysts:
+
+* domain experts using data for actionable insights
+* cloud-based data warehouses (DWHs) and serverless technologies enable analysts to focus on value-driven analysis rather than technical tasks
+* while proficient in no-code/low-code ML models, they lack skills for complex workflows like ranking or recommendations, necessitating data science teams for advanced tasks
+
+Data Engineers:
+
+* manage the downstream data pipeline, including loading, integration, governance, and data quality
+* promote ELT processes over ETL, leveraging SQL for post-loading transformations, improving efficiency and accessibility
+* handle complex tasks like batch jobs, mainframe data integration, and real-time analytics using tools like Pub/Sub, Kafka, or Kinesis
+* set up templates for generic tasks (i.e. DQ checks) that analysts can reconfigure
+
+**The technological framework**
+
+There are three fundamental principles underlying the high-level reference architecture for an analysis-driven organization:
+
+* SQL as a standard
+* from EDW/data lake to a structured data lake
+* schema-on-read first approach
+
+Combining these principles, we can define a high-level architecture that supports a few key data analysis patterns:
+
+* the 'traditional' BI workloads, such as creating a dashboard or report
+* an ad hoc analytics interface that allows for the management of data pipelines through SQL (ELT)
+* enabling data science use cases with ML techniques
+* real-time streaming of data into the DWH and processing of real-time events
+
+![image](https://github.com/user-attachments/assets/04b2864c-a658-4283-9cd7-e1f0070edf36)
+
+### Data Engineering–Driven Organization
+
+An engineering-driven organization is focused on data integration. 
+
+**The vision**
+
+_When your data transformation needs are complex, you need data engineers to play a central role in the company’s data strategy to ensure that you can build reliable systems cost-effectively. Data engineers are at the crossroads between data owners and data consumers._
+
+Data engineers have the following responsibilities:
+
+* transporting data and enriching data while building integrations between analytical and operational systems (as in the real-time use cases)
+* parsing and transforming messy data coming from business units and external suppliers into meaningful and clean data, with documented metadata
+* applying DataOps — that is, functional knowledge of the business plus software engineering methodologies applied to the data lifecycle; this includes monitoring and maintenance of data feeds
+* deploying ML models and other data science artifacts analyzing or consuming data
+
+Building complex data engineering pipelines is expensive but enables increased capabilities:
+
+* enriched data
+* better ML training data
+* using unstructured data
+* productionising
+* real-time analytics
+
+**The personas**
+
+* knowledge: data warehousing and data lakes, db systems, ETL tools, programming languages, data structures and algorithms, automation and scripting, container technology, orchestration platforms
+* responsibilities: data pipeline development, data integration, DQ assurance, automation, collaboration with data consumers, scalability and performance optimization, monitoring and troubleshooting, deployment and maintenance of data systems, support for machine learning models, data security and compliance
+
+**The technological framework**
+
+The technological framework emphasizes that data engineers should focus on data preparation (cleaning, transformation, enrichment) and not on maintaining infrastructure. The ingestion layer uses scalable solutions like Kinesis, Event Hubs, or Pub/Sub for real-time data and S3, Azure Data Lake, or Cloud Storage for batch data. These solutions automatically scale based on workload demands.
+
+Example reference architecture on AWS:
+
+![image](https://github.com/user-attachments/assets/18d94011-43bf-4619-9d28-9578bc042f5b)
+
+On GCP:
+
+![image](https://github.com/user-attachments/assets/728800be-bd4f-4297-a51b-1373db16606d)
+
+The reference architecture's main benefits include the use of serverless, NoOps technologies that reduce maintenance burdens and allow for flexibility and scalability, such as handling traffic spikes. It also supports custom code development for complex parsing and low-latency business logic, with an emphasis on reusing code through libraries and templates. Additionally, it enables the transformation of existing batch pipelines into streaming pipelines.
+
+### Data Science–Driven Organization
+
+These organizations leverage data science and machine learning to gain a competitive advantage. They rely on automated decision-making and strive for high business impact from their data initiatives.
+
+_For example, a bank that is data analysis driven would have data analysts assess each commercial loan opportunity, build an investment case, and have an executive sign off on it. A data science–driven fintech, on the other hand, would build a loan approval system that makes decisions on the majority of loans using some sort of automated algorithm._
+
+**The vision**
+
+Principles to follow:
+
+* adaptability
+* standardization
+* accountability
+* business impact
+* implementation
+
+_A science-driven organization should be built on a technical platform that is highly adaptable in terms of technological openness._
+
+**The personas**
+
+* DEs
+* MLEs
+* DSs
+* DAs
+
+**The technological framework**
+
+The authors strongly recommend standardizing on the ML pipelines infrastructure of your public cloud provider rather than cobbling together one-off training and deployment solutions.
+
+![image](https://github.com/user-attachments/assets/300c0af5-03ad-47b6-89d3-2ba033b9cb9b)
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 4 -  A Migration Framework
+
+_Unless you are at a startup, it is rare that you will build a data platform from scratch. Instead, you will stand up a new data platform by migrating things into it from legacy systems_
+
+### Modernise data workflows
+
+Before starting to make a migration plan, we ought to have a comprehensive vision of why we are doing it and what we are migrating toward
+
+**Holistic view**
+
+Data modernization transformation should be considered holistically. There are 3 main pillars:
+
+* what are the business outcomes of the workflows we aim to modernise
+* identify and maybe help educate the stakeholders
+* tech stack
+
+It's important to not treat the migration as a pure IT or data science project.
+
+**Modernise workflows**
+
+Modernizing workflows involves focusing on end-to-end tasks, not just upgrading tools. Instead of like-for-like replacements, rethink workflows from first principles to achieve transformational change. Key principles include:
+
+* automation
+* streaming by default
+* automatic scaling
+* built-in optimization
+* automated ML
+
+**Transform the workflow itself**
+
+Focus on transforming workflows by questioning whether they need precomputation by data engineers. Prebuilt data pipelines are optimizations, not necessities. Instead, consider enabling self-serve, ad hoc workflows using tools like Looker to move calculations into a declarative semantic layer.
+
+This approach empowers business teams to define and manage metrics, ensuring consistent KPIs and reducing dependency on data engineering resources while fostering a library of reusable measures.
+
+### A 4-step migration framework
+
+![image](https://github.com/user-attachments/assets/7048ad7e-fea5-4a54-85c3-0b9a42e50313)
+
+* Prepare and discover
+
+All stakeholders should conduct a preliminary analysis to identify the list of workloads that need to be migrated and current pain points (e.g., inability to scale, process engine cannot be updated, unwanted dependencies, etc.).
+
+* Assess and plan
+
+Assess the information collected in the previous stage, define the key measures of success, and plan the migration of each component.
+
+There will be different groups depending on their business value and effort to migrate.
+
+![image](https://github.com/user-attachments/assets/e2425102-e8d3-4126-bb17-1ff48b433bd7)
+
+* Execute
+
+For each identified use case, decide whether to decommission it, migrate it entirely (data, schema, downstream and upstream applications), or offload it (by migrating downstream applications to a different source). Afterward, test and validate any migration done.
+
+* Optimize
+
+Once the process has begun, it can be expanded and improved through continuous iterations. A first modernization step could focus only on core capabilities.
+
+### Estimating the overall cost of the solution
+
+It is important to remember that it is not just a matter of the cost of technology — there are always people and process costs that have to be taken into account.
+
+**Audit of the existing infrastructure**
+
+Can be done:
+
+* manually by the internal IT/infrastructure team
+* automatically by the internal IT/infrastructure team
+* leveraging a 3rd-party
+
+**Request for information/proposal and quotation**
+
+Consulting companies do these cost calculations for a living so we can consult with them as well. We can send some documents to potential vendors/partners and once we have all the information we can pick the best path forward. Docs:
+
+* request for information (questionnaire used to collect detailed info about vendors'/possible partners' solutions and services)
+* request for proposal (questionnaire used to collect detailed info about how vendors/partners will leverage their products and services to solve a specific organization problem)
+* request for quotation (questionnaire used to collect detailed info about pricing of different vendors/possible partners based on specific requirements)
+
+**PoC/MVP**
+
+* proof of concept: builds a small, testable portion of the solution to assess feasibility, usability, and scalability. It focuses on potential redesign needs without covering all features
+* minimum viable product: develops a fully functional, narrowly scoped product deployable in production, enabling feedback from real users for improvement and cost estimation
+* hybrid: starts with a broad but shallow PoC, transitioning to an MVP to refine and progress toward the complete solution
+
+### Setting up security and data governance
+
+**Framework**
+
+There are three risk factors that security and governance seek to address:
+
+* unauthorised data access
+* regulatory compliance
+* visibility
+
+Given these risk factors a data governance framework should include:
+
+* data lineage
+* data classification
+* data catalog
+* data quality management
+* access management
+* auditing
+* data protection
+
+**Artifacts**
+
+To provide the above capabilities to the organization, a central data governance team needs to maintain the following artifacts:
+
+* enterprise dictionary
+* data classes
+* policy book
+* use case policies
+* data catalog
+
+**Governance over the life of the data**
+
+Data governance involves bringing together people, processes, and technology over the life of the data.
+
+The data lifecycle consists of the following stages:
+
+1. data creation
+2. data processing
+3. data storage
+4. data catalog
+5. data archive
+6. data destruction
+
+When it comes to data, we can have different 'hats':
+
+* legal: ensures that data usage conforms to contractual requirements and government/industry regulations
+* data steward: the owner of the data, who sets the policy for a specific item of data
+* data governor: sets policies for data classes and identifies which class a particular item of data belongs to
+* privacy tsar: ensures that use cases do not leak personally identifiable information
+* data user: typically a data analyst or data scientist who is using the data to make business decisions
+
+### Schema, Pipeline, and Data migration
+
+**Schema migration**
+
+When migrating a legacy application to a new system, it is best to first transfer the existing schema as-is, connecting upstream and downstream processes. After ensuring functionality in the new environment, we need to make schema changes in a second phase to minimize downtime risks. Using the facade pattern, we can create views that mask underlying table complexities, presenting a simplified schema to downstream processes while leveraging target system features. If this is not feasible, data must be transformed and converted before ingestion, handled by dedicated transformation pipelines.
+
+**Pipeline migration**
+
+There are two strategies:
+
+* we are offloading the worklod: we retain the upstream data pipes feeding our source system, and put incremental copy of the data into the target system. Finally, we update our downstream processes to read from the target system - then we continue the offload with the next workload until we reach the end. Once completed, we can start fully migrating the data pipeline
+* we are fully migrating the workload: we migrate everything in the new system, and then deprecate the corresponding legacy tables
+
+General data pipeline patterns:
+
+* ETL
+* ELT
+* EL
+* CDC
+
+**Data migration**
+
+* data transfer requires plnanning. We need to identify and involve technical owners, approvers, the migration (delivery) team
+
+Before proceeding we need to answer:
+
+* what are the datasets we need to move
+* where does the underlying data sit within the organisation
+* what are the datasets we are allowed to move
+* is there any specific regulatory requirement we have to respect
+* where is the data going to land
+* what is the destination region
+* do we need to perform any transformation before the transfer
+* what are the data access policies we want to apply
+* is it a one-off transfer, or do we need to move data regularly
+* what are the resources available for the data transfer
+* what is the allocated budget
+* do you have adequate bandwidth to accomplish the transfer, and for an adequate period
+* do you need to leverage offline solutions
+* what is the time needed to accomplish the entire data migration
+
+**Regional capacity and network to the cloud**
+
+* public internet connection
+* partner interconnect
+* direct interconnect
+
+**Transfer option considerations**
+
+* cost
+* time
+* offline vs. online transfer
+* available tools
+
+**Migration stages**
+
+![image](https://github.com/user-attachments/assets/d71c9592-e725-4ef7-b9b0-e9727dcbef87)
+
+1. upstream processes feeding current legacy data solutions are modified to feed the target env
+2. downstream processes reading from the legacy env are modified to read from the target env
+3. historical data is migrated in bult into the target env; at this point, the upstream processes are migrated to also write to the target env
+4. downstream processes are now connected to the target env
+5. data can be kept in sync between the legacy and target envs, leveraging CDC pipelines until the old env is fully dismissed
+6. downstream processes become fully operational, leveraging the target env
+
+Checks to do to ensure minimum bottlenecks or data transfer issues:
+
+* perform a functional test
+* perform a performance test
+* perform a data integrity check
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 5 -  Architecting a data lake
+
+### Data lake and the Cloud - a perfect marriage
+
+**Challenges with on-premises data lakes**
+
+* TCO
+* scalability
+* governance
+* agility
+* resource-intensive data and analytics processing
+
+**Benefits of cloud data lakes**
+
+* no need to store all data in an expensive HDFS cluster
+* hadoop clusters can be created on demand in a short amount of time
+* hyperscalers usually offer capabilities to leverage less expensive VMs
+* managed VMs
+* separation between storage and compute which gives orgs better flexibility in handling data governance challenges
+
+### Design and implementation
+
+**Batch and Stream**
+
+Whether batch or stream, there are four storage areas in a data lake:
+
+* raw/landing/bronze (where raw data is collected and ingested directly from source systems)
+* staging/dev/silver (where more advanced users process the data to prepare it for final users)
+* production/gold (where the final data used by prod systems is stored)
+* sensitive (optional)
+
+Lambda architecture:
+
+![image](https://github.com/user-attachments/assets/27c97d67-f2d2-457b-86f9-20647b24b610)
+
+Kappa architecture:
+
+![image](https://github.com/user-attachments/assets/89035b1f-5ac8-4a53-a7c7-a34fee8cdbb8)
+
+**Data catalog**
+
+![image](https://github.com/user-attachments/assets/7f0513bd-b099-4ac3-b05d-7b65e4a29e72)
+
+A centralized repository of metadata describing organizational datasets, making scattered data across databases, data warehouses, filesystems, and blob storage more discoverable and accessible. It supports various users like data scientists, engineers, and business analysts by enabling efficient data discovery and rationalization.
+
+* connects to data lakes and other platforms, ensuring a comprehensive metadata view without duplicating data
+* helps identify and managed duplicated, unused, or similar data, enabling cost savings by focusing on relevant data
+* facilitates data syncing, transformations, and updates in the lake while maintaining clarity on dataset levels
+* uses json/yaml files to formalise agreements on data schema, ingestion freq, ownership, and access levels
+
+**Cloud data lake reference architecture**
+
+**AWS**
+
+![image](https://github.com/user-attachments/assets/6ffda068-4083-4c5a-95a8-94e7d4a6938b)
+
+* data sources, which are S3 and NoSQL dbs
+* storage zones on top of S3 buckets
+* data catalog, data governance, security and process engine orchestrated by AWS lake formation
+* analytical services such as Athena, RedShift, and EMR to provide access to the data
+
+**Azure**
+
+![image](https://github.com/user-attachments/assets/6f9d7855-7e06-4304-bc37-d9a93e8e8622)
+
+**GCP**
+
+![image](https://github.com/user-attachments/assets/ea477b5b-6153-4049-b1f5-0cff601f9d61)
+
+_When choosing a cloud vendor, you should also consider the datasets that are natively available through their offerings. In the marketing or advertising space, for example, it can be very impactful to implement your business solutions using these datasets. Some examples of datasets include Open Data on AWS, the Google Cloud Public Datasets, and the Azure Open Datasets. If you advertise on Google or sell on Amazon, the ready-built integrations between the different divisions of these companies and their respective cloud platforms can be particularly helpful._
+
+### Integrating the data lake: the real superpower
+
+_The superpower of a data lake is its ability to connect the data with an unlimited number of process engines to activate potentially any use case you have in mind._
+
+* there are many data sources that can be accessed through APIs
+
+**The evolution of data lakes with Apache Iceberg, Apache Hudi, and Delta Lake**
+
+* ACID compliance, giving users the certainty that the info they are querying is consistent
+* overcoming the inherent limitations of HDFS in terms of file size - with this approach, even a small file can work perfectly
+* logging of every single change made on data, guaranteeing a complete audit if ever cenessary and enabling time travel queries
+* no difference in handling batch and streaming ingestions and elaboration
+* full compatibility with the Spark engine
+* storage based on Parquet that can achieve a high level of compression
+* stream processing enablement
+* ability to treat the object store as a db
+* applying governance at the row and column level
+* partition evolution
+* schema evolution
+
+**Interactive analytics with notebooks**
+
+Jupyter general notebook architecture and Spark-based kernel:
+
+![image](https://github.com/user-attachments/assets/dc9252df-3538-4d76-ac9f-5eb113d4c962)
+
+### Democratising data processing and reporting
+
+**Build trust in the data**
+
+Establishing trust in data is essential for organizations, particularly when transitioning from IT-managed to democratized data access. The example of a retail consultancy highlights the importance of transparency in data extraction, transformation, and reporting. When IT teams lack in-depth knowledge, they may struggle to assure decision-makers of data reliability. Empowering end users with tools to explore, audit, and manage data fosters autonomy and trust.
+
+* identify the key stakeholders who have the right information on a timely basis
+* defend the data from any kind of exfiltration, both internal and external, with a focus on personnel
+* cooperate with other people inside or outside the company to unlock the value of data
+
+**Data ingestion is still an IT matter**
+
+
+Summary: Data Ingestion and IT's Role
+
+Data ingestion is a critical step in maintaining an efficient data lake and preventing it from becoming a data swamp filled with unused, stale, and inconsistent data. Poor planning during ingestion can lead to wasted storage, errors, and inefficiencies. To avoid this, organizations should follow some best practices:
+
+* file formats (JSON, Avro, Parquet)
+* file sizes (in hadoop - larger is better; for smaller it is a good idea to use a streaming engine to consolidate the data into a few large files)
+* data compression to save space
+* directory structure (tailor the structure to the use case)
+
+---
+
+## [Watching projects from Zach Wilson's Analytics Eng boot camp](https://youtu.be/XD2b8V_RfFc)
+
+**1st project** 
+
+Big Bag Data - build an automated pipe that scrapes, cleans, and transforms data from sold handbags - then build a website that presents this data in a meaningful way
+
+<img width="1118" alt="image" src="https://github.com/user-attachments/assets/ec71a49b-001b-407e-bbc7-b69c0d5bc025" />
+
+**2nd project**
+
+F1 Insights: Real-Time Replay & Historical Analytics - [github](https://github.com/sonGokuKhk98/Formula_1)
+
+![image](https://github.com/user-attachments/assets/0776a119-f8b8-4762-9eea-a643b4a688f1)
+
+**3rd project**
+
+Global historical climatology network daily
+
+Data model:
+
+<img width="601" alt="image" src="https://github.com/user-attachments/assets/a87a627b-ea1b-4354-8d9c-c67f5429917e" />
+
+**4th project**
+
+Real-time sports betting analytics platform - [github](https://github.com/sanchitvj/sports_betting_analytics_engine)
+
+Real-time pipe: 
+
+<img width="561" alt="image" src="https://github.com/user-attachments/assets/bca40383-5d9f-4f50-ae13-f1a091e1d1e0" />
+
+Batch pipe:
+
+<img width="643" alt="image" src="https://github.com/user-attachments/assets/7597dfc8-097d-4ab9-abbc-a8cacbbd7651" />
+
+**5th project**
+
+Chess.com analytics project
+
+**6th project**
+
+Meme coin tracker - understand who are profitable traders, find what factors make a coin promote to better markets
+
+<img width="859" alt="image" src="https://github.com/user-attachments/assets/42c286d8-ca29-48d0-87ce-abdc96c13e17" />
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 6 -  Innovating with an EDW
+
+### A modern data platform
+
+**Organisational goals**
+
+* no silos
+* democratisation
+* discoverability
+* stewardship
+* single source
+* security and compliance
+* ease of use
+* data science
+* agility
+
+**Tech challenges**
+
+* size and scale
+* complex data and use cases
+* integration
+* real time
+
+**Tech trends and tools**
+
+* separation of compute and storage
+* multitenancy
+* separation of authentication and authorisation
+* analytics hubs
+* multicloud semantic layer
+* consistent admin interface
+* cross-cloud control pane
+* multicloud platforms
+* converging of data lakes and DWHs
+* built-in ML
+* streaming ingest
+* streaming analytics
+* CDC
+* embedded analytics
+
+### Hub-and-Spoke architecture
+
+![image](https://github.com/user-attachments/assets/d48c0467-e0b4-4288-868d-82d59483c179)
+
+This architecture is an ideal design for modern cloud data platforms centered around a DWH. The DWH acts as the central hub, collecting and managing all enterprise data, while the spokes—custom applications, dashboards, ML models, and more—interact with the DWH via standard interfaces like APIs. This structure enables seamless data access without duplication and offers scalability, flexibility, and resilience.
+
+* well-suited for startups, organizations seeking transformation, and enterprises with minimal legacy constraints
+* efficient data ingestion is achieved through mechanisms like scheduled exports for SaaS data, real-time CDC tools for operational databases, and dynamic queries for federated datasets
+* tech advancements like the separation of compute and storage, stateless ML APIs, and unified batch/stream processing have made this architecture a powerful alternative to older paradigms that relied on data marts, ETL, and sharded storage
+
+**Data ingest**
+
+3 ingest mechanisms:
+
+* prebuilt connectors
+
+![image](https://github.com/user-attachments/assets/4cb4a551-53d8-411d-a58a-69df1baafa0e)
+
+Third-party vendors such as Fivetran can automatically handle landing data from a wide range of sources into a cloud DWH such as BigQuery, Redshift, or Snowflake
+
+* real-time data capture
+
+![image](https://github.com/user-attachments/assets/b281557e-a76c-411d-953d-819bf0443a13)
+
+* federated querying
+
+**BI**
+
+![image](https://github.com/user-attachments/assets/57345243-d9cf-4206-b448-1d61f622ad91)
+
+* make sure that the BI tool pushes all queries to the enterprise DWH and doesn’t do them on OLAP cubes (extracts of the database/DWH)
+* visualisations
+* offer embedded analytics
+* use a semantic layer
+
+**Transformations**
+
+A better option is to define transformations within the DWH using SQL to create views or tables, making the results available to all use cases. Another method, reverse ETL, uses event-driven serverless functions to transform and push data into backend systems.
+
+For handling large datasets, ELT (Extract, Load, Transform) can be used instead of traditional ETL
+
+Alternatively, scheduled queries periodically extract and aggregate data, which reduces query costs but can result in outdated data.
+
+Materialized views offer a balanced approach by storing precomputed results, which are automatically updated in modern DWHs, improving query performance.
+
+**Organisational structure**
+
+_In many organizations, there are many more business analysts than engineers. Often, this ratio is 100:1. A hub-and-spoke architecture is ideal for organizations that wish to build a data and ML platform that primarily serves business analysts. Because the hub-and-spoke architecture assumes that business analysts are capable of writing ad hoc SQL queries and building dashboards, some training may be necessary._
+
+![image](https://github.com/user-attachments/assets/7cac60a6-e250-4517-abcf-395dca656a9b)
+
+The central data engineering team is responsible for the filled shapes, while the business unit is responsible for the unfilled shapes
+
+### DWH to enable data scientists
+
+**Query interface**
+
+Before automating decision making, DSs need to do EDA and lots of experimentation. Thankfully tools like jupyter notebook allow to execute both python and sql in their cells (two common languages for DS)
+
+**Storage API**
+
+Using the storage API that ML frameworks support to read data efficiently and in parallel out of the DWH (i.e. Spark and TensorFlow offer such API)
+
+QI and Storage APIs are the two methods DSs use to access data in the DWH for analysis, but nowadays there is a trend of DWH having the capability of implementing, training and using an ML alg directly in them. 
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 7 - Converging to a Lakehouse
+
+### The need for a unique architecture
+
+Data lakes and DWHs emerged to meet the needs of different users. An organization with both types of users is faced with an unappealing choice.
+
+**User personas**
+
+Traditional DWH users are BI analysts closer to the business, focusing on deriving insights from data; they tend to be proficient in SQL
+
+Data lake users include DAs, DEs and DSs. They not only transform the data to make it more accessible by the business but also experiment with it and use it for ML. 
+
+**Antipattern: Disconnected systems**
+
+Different IT departments or teams will manage the DWH and the data lake - resulting in spending resources on operational aspects rather than focusing on business insight and not being able to allocate resources to focus on the key business drivers.
+
+Maintaining 2 separate systems can cause DQ and consistency problems.
+
+**Antipattern: Duplicated data**
+
+Why not simply connect the two systems with data being synced by the platform team? We will end up with something like:
+
+![image](https://github.com/user-attachments/assets/8b8bc6d4-9234-4c43-b438-dec3b495998c)
+
+Drawbacks of such architecture:
+
+* proliferation of the data (data exists in various forms and maybe even outside the boundaries of the platform, which may cause security and compliance risks, data freshness issues, or data versioning issues)
+* slowdown in time to market (orgs may spend up to 2 weeks implementing a minor change to a report for management because htye must first request that DEs modify the ETLs in order to access the data necessary to complete the task)
+* limitations in data size
+* infrastructure and operational cost
+
+### Converged Architecture
+
+**Two forms**
+
+The common storage can take one of two forms:
+
+* data is stored in an open source format (parquet, avro, etc) on the cloud and we can use Spark to process it (using Python, SQL, Scala, R); we can also use open source formats like Delta Lake and Apache Iceberg, or
+* the common storage can be a highly optimised DWH format where we can natively leverage SQL
+
+We need to choose based on user skills and do a proper evaluation based on some criteria. 
+
+**Lakehouse on cloud storage**
+
+* cloud storage allows compute resources (like Spark clusters) to be scaled independently from storage, and compute can move to the data, rather than the other way around. This is a major advantage over on-premise systems
+* instead of separate engines for data lakes and DWHs, a single analytical engine accesses all data, streamlining analysis and eliminating silos
+* cloud storage's high throughput and the ability to bring compute to data result in faster query speeds
+* data democratization: enables self-service queries for data engineers, data scientists, and even business user
+* centralized data governance and security, ensuring consistent policies
+* supports both schema-on-read (data lake) and schema-on-write (DWH) approaches, and easily integrates with streaming data using ACID transactions with technologies like Apache Iceberg, Apache Hudi, and Delta Lake
+* by using a single storage layer, this architecture reduces the need to download or copy data for offline processing
+
+Migration strategy:
+
+* iterative approach
+* gain org buy-in by showing a quick solution
+* phased decommissioning of the old system
+* start with new workloads
+
+**SQL-first lakehouse**
+
+This enables users to execute analytics and BI, but also carry out orchestration and ML.
+
+* SQL's wide familiarity makes data access and analysis easier for business users, not just technical staff
+* SQL handles most transformations, while Spark is used for complex tasks and ML
+* business users can train and deploy ML models with SQL, empowering data-driven decisions
+* provides a single, integrated platform for data processing, analytics, and ML
+
+Migration:
+
+* iterative approach
+* direct DWH ingestion
+* elevate DWH SQL engine
+* insource ML worklouds into the DWH env
+* Emphasize SQL-based data transformation skills, potentially requiring a new role (analytics engineers) or skillset within your organization
+
+**The benefits of convergence**
+
+* quicker time to market
+* reduced risk
+* predictive analytics
+* data sharing
+* ACID transactions
+* multimodal data support
+* unified env
+* schema and governance
+* streaming analytics
+
+---
+
+## [Apache Flink 2.0 talk at Flink Forward Jakarta '24](https://youtu.be/go8ZAtZhwvY)
+
+**First thing** that caught my attention was a new ForSt DB (for streaming db) that will be used in Flink 2.0
+
+<img width="842" alt="image" src="https://github.com/user-attachments/assets/08e730e8-510c-4557-8e4d-77325ac19279" />
+
+**Next, stream-batch unification**
+
+Learned about [Apache Paimon](https://paimon.apache.org/docs/master/) - a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark for both streaming and batch operations.
+
+<img width="1074" alt="image" src="https://github.com/user-attachments/assets/b2d6f777-2e05-46c4-bf61-df28ac25070a" />
+
+(above pic is from another Paimon talk (links below))
+
+<img width="848" alt="image" src="https://github.com/user-attachments/assets/3a1cc7e2-1822-42fa-ba4d-6dc1dd6990be" />
+
+<img width="794" alt="image" src="https://github.com/user-attachments/assets/636f2b01-fe5b-49ef-ba71-ab13fb3bb52a" />
+
+They introduce this materialized table:
+
+<img width="829" alt="image" src="https://github.com/user-attachments/assets/135372be-3438-4d72-ae9a-294d0fe8ed19" />
+
+<img width="911" alt="image" src="https://github.com/user-attachments/assets/e5d33ae2-e5c4-4590-9757-16ecff0604f7" />
+
+**ML models in Flink SQL**
+
+<img width="766" alt="image" src="https://github.com/user-attachments/assets/61e93ad1-9494-470c-b50f-af03f6d908bb" />
+
+---
+
+## Other talks from Flink Forward Jakarta
+
+* [Apache Flink 101](https://youtu.be/1bjvkdhX4Vk)
+* [Flink + Paimon: Build Streaming Pipeline on Lakehouse](https://youtu.be/DEm6uZvqQeA)
+* [Paimon 1.0 Unified Lake Format for Data + AI](https://youtu.be/3HrM6gQfWbE) (similar to the above)
+* [Embrace Unified Stream and Batch Processing with Flink SQL](https://youtu.be/MvGnm8qSGGg)
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 32b8cccb7a1db99551bdaf9b0be6d4e038d6f874 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 28 Dec 2024 22:22:46 +0900
Subject: [PATCH 568/638] Create 2024-12-28-day-362.md

---
 _posts/2024-12-28-day-362.md | 636 +++++++++++++++++++++++++++++++++++
 1 file changed, 636 insertions(+)
 create mode 100644 _posts/2024-12-28-day-362.md

diff --git a/_posts/2024-12-28-day-362.md b/_posts/2024-12-28-day-362.md
new file mode 100644
index 000000000..b77927d43
--- /dev/null
+++ b/_posts/2024-12-28-day-362.md
@@ -0,0 +1,636 @@
+---
+layout: post
+title: (Day 362) Learning about Apache Pinot + Data Mesh
+categories: [theory,data-eng]
+---
+
+# Hello :) Today is Day 362!
+A quick summary of today:
+* Apache Pinot 101
+* more of Architecting Data and Machine Learning Platforms
+* starting the Data Mesh book
+
+---
+
+## [Apache Pinot 101](https://www.youtube.com/playlist?list=PLihIrF0tCXdfN6y-twj9KtWaXM1GH4RSe)
+
+[Zach Wilson](https://www.youtube.com/watch?v=uaf0NW4MEY4) did a livestream this morning and mentioned that Pinot is a good OLAP database for real-time data and a concept that is interesting. So I decided to look for a video ~ I found not just a video but a short playlist of videos by Tim Berglung (the nice instructor from Confluent's Kafka/FLink videos) - [here](https://www.youtube.com/playlist?list=PLihIrF0tCXdfN6y-twj9KtWaXM1GH4RSe)
+
+### What is Apache Pinot?
+
+_Apache Pinot is a real-time distributed OLAP database, designed to serve OLAP workloads on streaming data with extreme low latency and high concurrency._
+
+The essence of real-time analytics:
+
+* latency: as low as 10ms
+* concurrency: as many as 100,000 queries per second
+* freshness: seconds from event time till queryable in Pinot
+
+**Data model**
+
+* Pinot uses the usual tabular data model
+* table creation and schema definition expressed in JSON
+* queries with SQL
+
+### Architecture: Tables and Segments
+
+**Tables**
+
+* the basic unit of data storage
+* composed of rows and columns
+* expected to scale to arbitrarily large row counts
+* defined using a schema and tableConfig JSON file
+* three variaties: offline(contains batch data), real-time and hybrid
+* every column is either a metric, dimension or date/time
+
+**Segments**
+
+* tables are split into units of storage called segments
+* similar to shards or partitions, but transparent to users
+* for offline tables, segments are created outside of Pinot and pushed into the cluster using a REST API
+* for real-time tables, segments are created automatically from events sourced by the event streaming system (like Kafka)
+* standard utilities support batch ingest from standard file types (avro, json, csv)
+* APIs are available to create segments from Spark, Flink, and Hadoop
+
+**Segment structure**
+
+* Pinot is a columnar db
+* all of a segment's values for a single column are stored contiguously
+* dimension columns are typically dictionary-encoded
+* indexes are stored as a part of the segment
+* segments are immutable once written
+* segments have a configurable retention period
+
+Segment structure example:
+
+<img width="687" alt="image" src="https://github.com/user-attachments/assets/576ac723-b1ce-4094-b29d-94f705bdccb5" />
+
+Zooming in on ip:
+
+<img width="569" alt="image" src="https://github.com/user-attachments/assets/0925f92e-7203-4519-8244-e5f3a080f072" />
+
+zooming again
+
+<img width="689" alt="image" src="https://github.com/user-attachments/assets/a7a9ae0b-9c74-44df-94e3-2693a504be29" />
+
+It is many many many individual values of the IPs.
+
+### Architecture: Servers and Brokers
+
+**Servers**
+
+* the computers that host segments and do most query processing
+* they are generally segregated into real-time and offline servers
+* this is an edministrative convention only; servers do not know they are one kind or another
+* real-time servers host real-time tables exclusively, and offline servers host offline tables exclusively - by convention
+
+<img width="707" alt="image" src="https://github.com/user-attachments/assets/bd44fa0e-3ff6-4ad1-bbed-92a7674c94ad" />
+
+Since segments are immutable it's easy to store replication is easier since we don't need to think about which replicas are out-of-date, we just need to make sure we have the copies. 
+
+**Brokers**
+
+* accept queries from clients and forward them to the appropriate servers
+* have access to metadata that helps them determine which segments are implicated in a given query, and which servers host those segments
+* for single-stage queries, they collect results returned from the servers and consolidate them to return the client
+* for multi-stage queries, they issue a plan to several tiers of servers, which eventually return a single result set to the issuing broker
+
+**Deep store**
+
+* the deep store is the permanent storage system for segment files
+* stores a compressed version of segment data without indexes
+* can be located on a POSIX-compatible file system, HDFS, or a cloud blob store
+* does not host backups of Zookeeper metadata
+
+<img width="422" alt="image" src="https://github.com/user-attachments/assets/7ef983a3-0e1b-4f44-a640-ef597b4ce4df" />
+
+### Architecture: Cluster Management
+
+**Controller**
+
+* the Controller hosts the Apache Helix controller
+* responsible for cluster management (e.g node failure detection and recovery, replication management, segment assignment)
+* maintains global metadata (table schemas and configs)
+* maintains the mapping of segments to servers
+* expose admin APIs for vieweing and changing configs
+* the endpoint for offline segment uploads
+* manage real-time ingestion and segment creation
+* controllers can be configured for redundancy
+
+**Zookeeper**
+
+* distributed, fault-tolerant, strongly consistent state store
+* provides persistence for the Controller to store cluster metadata
+
+**Multitenancy**
+
+* Pinot supports multitenance natively
+* every broker and server can be assigned a tenant tag
+* a table can be assigned a server tenant tag and a broker tenant tag
+* this allows us to ensure that certain tables are not hosted on the same hardware as others
+
+<img width="596" alt="image" src="https://github.com/user-attachments/assets/5d547a0a-a280-49a1-8e6b-8019d20bc562" />
+
+### Architecture: Minions
+
+* refers to a cluster component that runs tasks on a standby basis
+* minions offload computationally intensive work from other cluster components, like servers
+* often involved in segment creation, segment purge, and segment merging
+* we can create our own minion tasks by implementing the `PinotTaskExecutor` and `MinionEventObserver` interfaces
+
+<img width="607" alt="image" src="https://github.com/user-attachments/assets/cf174424-e51e-456f-997c-7a8aeb517833" />
+
+Zooming in ~
+
+<img width="533" alt="image" src="https://github.com/user-attachments/assets/b2f949b1-aecb-4c3d-a33e-f42958910be1" />
+
+### Indexes
+
+Once data is ingested into Pinot, we need to read it and reading it is about how the data is indexed - we have different index options in Pinot
+
+<img width="524" alt="image" src="https://github.com/user-attachments/assets/bc9575c2-79c0-473f-91ad-0acb324a626d" />
+
+### Batch (offline) ingestion
+
+* start by creating a schema (name, dimensiton fields, metric fields, datetime), and a table config (pics below)
+
+<img width="562" alt="image" src="https://github.com/user-attachments/assets/30545ef6-35da-45aa-8930-7a30628f495b" />
+
+<img width="561" alt="image" src="https://github.com/user-attachments/assets/3eceb90a-84d3-4905-83be-62c6d407dd43" />
+
+We can then ingest the data. 
+
+When we run the ingest, the controller kicks off a task and says this minion worker - go do this ingestion task with this csv file and config. This worker creates a segment in its storage, it pushes that segment to the controller which decides on which server to store it and then it is saved on a server. 
+
+<img width="623" alt="image" src="https://github.com/user-attachments/assets/0622058d-a5fb-4421-b9cd-2c83d9b1e52d" />
+
+### Stream (real-time) ingestion
+
+Usually we read from a Kafka topic and a Pinot table is a consumer. Since it is real-time we might not have enough records to create a segment. Data is first gathered in a consuming segment and after some threshold the records in that consuming segment are flushed into a segment. As soon as a record is written to the consuming segment it's available for querying. 
+
+<img width="545" alt="image" src="https://github.com/user-attachments/assets/1c4b32b7-b359-4b42-a7f5-5657260f8b44" />
+
+Realtime table config:
+
+<img width="460" alt="image" src="https://github.com/user-attachments/assets/66400a09-73c7-4c21-96b0-04d8edb11439" />
+
+<img width="568" alt="image" src="https://github.com/user-attachments/assets/0d7e92b7-8851-411c-a4f6-3f55628acf1c" />
+
+When we create a table, we tell it to the Controller, it then tells a server it has the duty to connect to this Kafka cluster and starts consuming
+
+<img width="663" alt="image" src="https://github.com/user-attachments/assets/3de6e688-e87e-42a7-b698-cab0b791291e" />
+
+When the server is writing those segment files it is asking the Controller where they should go. 
+
+### Multi-stage query engine
+
+* Pinot's architecture supports low-latency queries on batch and streaming data at high concurrency
+* it is optimised for queries that filter and aggregate
+* query processing occurs on servers
+* last-mile reducing happens on Brokers - for small datasets
+* Brokers cannot procecss large join datasets - that's where the multi-stage query engine comes in
+
+Simpler query:
+
+<img width="449" alt="image" src="https://github.com/user-attachments/assets/09f3b146-7667-40f7-855c-7250f1de0563" />
+
+More complex query:
+
+<img width="717" alt="image" src="https://github.com/user-attachments/assets/ab99d4e8-260f-47b7-9be0-910af0ff43fb" />
+
+<img width="725" alt="image" src="https://github.com/user-attachments/assets/a914c42a-f41c-40d5-ad6f-b5d6f0c6adff" />
+
+<img width="749" alt="image" src="https://github.com/user-attachments/assets/9c0cf1c6-0ea0-4335-ae0b-d65db58d9f43" />
+
+### Upsert
+
+* offline segments are immutable
+* real-time segments can be appended-to while being consumed, but once written to the segment store, they are also immutable
+* when ingesting streams of events, this is not a problem
+* when ingesting streams of mutable entities (i.e. orders which change status frequently), lack of upsert capability is a blocker
+* upserts apply only to real-time tables
+* implemented through a metadata map that servers maintain per table
+
+Put simply, if we have consumed A:1, B:2, C:3, and then we get B:4 - we insert it into the segment and we just update the metadata map
+
+<img width="465" alt="image" src="https://github.com/user-attachments/assets/14c5ea93-a89b-4e6b-b6b5-5614fb97934f" />
+
+**Delete support**
+
+* upserts originally supported re-writing a given primary key value
+* by itself, that is like dding UPDATE to INSERT in a transactional db
+* DELETE is required for ingesting CDC feeds and other use cases
+* each server maintains a metadata map
+* historically, the metadata map points to a segment/docID/timestamp tuple, which holds the the most recent UPDATE
+* delete adds the equivalent of a tombstone
+
+We see that B gets deleted:
+
+<img width="469" alt="image" src="https://github.com/user-attachments/assets/a842653c-2d2c-4ca1-aec6-d4e78c7099e0" />
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 8 - Architectures for Streaming
+
+### The value of streaming
+
+![image](https://github.com/user-attachments/assets/c9c8dd89-f8cc-4fb1-86b6-44292a85e28d)
+
+The value of a decision typically drops with time; stream processing allows an organization to make decisions in near real time
+
+**Industry use cases**
+
+* in healthcare for real-time patient monitoring
+* in finance to detect and prevent fraud
+* in retail for personalised marketing
+* in media and entertainment to generate personalised content and ads
+
+**Streaming use cases**
+
+* streaming ingest
+* real-time dashboards
+* stream analytics
+* continuous intelligence
+
+### Streaming ingest
+
+Could be done in 2 ways:
+
+* we could agg events and write only the aggregates to a persistent store (called streaming ETL)
+* we could ingest the data directly into a lake or DWH (streaming ELT)
+
+**Streaming ETL**
+
+![image](https://github.com/user-attachments/assets/c23583ab-c48d-4a77-bd4f-9822e4bfb097)
+
+Two options for streaming ETL of logs data: micro-batching (shown with dashed lines) or streaming pipeline (shown with solid lines)
+
+**Streaming ELT**
+
+![image](https://github.com/user-attachments/assets/813a330b-2a7c-43f0-b443-2d480a8064c4)
+
+Streaming ELT can be done in micro-batches (dashed lines) or as events happen (solid lines)
+
+**Streaming insert**
+
+Streaming ELT in a cloud native application can take advantage of client libraries to directly insert the data
+
+![image](https://github.com/user-attachments/assets/3065f903-eeb1-49fa-851a-680c5de16491)
+
+**Streaming from edge devices (IoT)**
+
+For streaming data from IoT devices on the edge, make use of IoT-specific functionality
+
+![image](https://github.com/user-attachments/assets/13de9fe0-8239-44a0-ad25-1295b5e6fa94)
+
+### Real-time dashboards
+
+**Live querying**
+
+Dashboards periodically query the DWH using SQL
+
+![image](https://github.com/user-attachments/assets/d9409609-52ac-4fa7-8876-676e650938c0)
+
+**Materialise some views**
+
+It is better to not have complex SQL queries in our dashboards. Instead of complex queries we can create views that retrieve the required data and have the dashboard tool query the view.
+
+![image](https://github.com/user-attachments/assets/4a079d8d-1f8a-489a-bea2-ca4475e97a7a)
+
+### Stream analytics
+
+Useful for:
+
+* time-series analytics: track assets, predict impact of events, and do predictive maintenance
+* clickstream analysis: to make real-time offers, create dynamic websites, and optimize a customer journey
+* anomaly detection: predict equipment failure, prevent fraud, monitor system health
+
+**Time-series analytics**
+
+Architecture for time-series analytics:
+
+![image](https://github.com/user-attachments/assets/f0d10aff-08a9-49d8-9b7f-8ccecfa48e2d)
+
+**Clickstream analytics**
+
+We can use a backfill pipeline to enrich clickstream data and improve identity stitching, privacy redaction, and bot detection
+
+![image](https://github.com/user-attachments/assets/eaae02de-e578-475a-b08a-118a81372b17)
+
+**Anomaly detection**
+
+Anomaly detection involves two streaming pipelines: one to compute clusters within a sliding window and a second to compare incoming events against the clusters
+
+![image](https://github.com/user-attachments/assets/d3816248-78f3-415d-891e-2989360771f2)
+
+**Resilient streaming**  
+
+Handling errors in streaming pipelines requires resilience to ensure uninterrupted operation:  
+
+- store unprocessable events in a dead-letter queue for periodic inspection and logic fixes, preventing data loss
+
+- to update a pipeline:  
+  - in-place update: achieve exactly-once semantics by reusing persistent stores
+  - pipeline draining: ensure at-least-once processing by completing in-flight data before transitioning
+
+### Continuous intelligence through ML
+
+**Training model on streaming data**
+
+* windowed training
+
+![image](https://github.com/user-attachments/assets/2c3a91eb-4266-440a-a2f4-c5d7e4d55055)
+
+We need a streaming pipe to create a dataset consisting of data within the time window, and an automated training pipe that is parametrized in terms of where it obtains the train data.
+
+* scheduled training
+
+We can do this in cases where a trained model will be valid for a moderately long time period. 
+
+![image](https://github.com/user-attachments/assets/5609da4a-f6cc-4ada-9118-68d5d6caf739)
+
+* continuous evaluation and retraining
+
+![image](https://github.com/user-attachments/assets/1abcdc88-a051-4213-9c09-42721d962cac)
+
+We don't want to use a model that has drifted in performance. To determine that the model has drifted in performance, we will need to employ continuous evaluation. For instance, if we have a model to predict whether a user will buy an item, you could verify a few days later if the user has bought the item.
+
+**Streaming ML inference**
+
+![image](https://github.com/user-attachments/assets/789fed55-97fd-4c7c-b951-a3eb383c5c64)
+
+Streaming ML inference involves invoking trained ML models for predictions as events occur. While embedding the model in the streaming pipeline is efficient, it doesn't scale well for large models or diverse applications. For non-Python environments or hardware without GPUs, models are often deployed as microservices accessible via HTTP requests. Efficiency improves when processing small batches of events simultaneously, leveraging matrix operations in modern ML frameworks.
+
+**Automated actions**
+
+![image](https://github.com/user-attachments/assets/6bdafd69-f4ac-4f21-945b-3501161932ce)
+
+---
+
+## Architecting Data and Machine Learning Platforms - Chapter 9 - Extending a data platform using Hybrid and Edge
+
+### Why multicloud?
+
+While a single cloud provider offers simplicity, cost efficiency, and streamlined learning curves, the reality for many organizations is a multicloud environment. 
+
+Multicloud might be inevitable:
+
+* M&As may lead to inherited cloud infrastructure and mixing clouds
+* seeking best of multiple clouds
+* mitigating vendor lock-in and establishing exit strategies
+* address regulatory requirements and data residency concerns
+
+### Multicloud Architectural Patterns
+
+**Single Pane of Glass**
+
+Utilizing either BI tools or distributed process engines to connect, query, and blend data from various sources, providing a unified view for users
+
+<img width="562" alt="image" src="https://github.com/user-attachments/assets/8e8bb715-79ef-4d63-b03a-b96de375ec3b" />
+
+**Write once, run anywhere**
+
+This can be accomplished through 
+
+* managed open source software
+* multicloud products
+* utilizing multiple runners (e.g., Apache Beam)
+* open source abstraction layers
+* running open source software on Infrastructure as a Service (IaaS)
+
+**Bursting from On Premises to Cloud**
+
+![image](https://github.com/user-attachments/assets/48af234b-ce09-4b77-bf4c-e4e9ec234244)
+
+We can expand on-premises data lakes into the cloud for specific workloads like large batch jobs, testing version upgrades, or experimenting with new technologies. DistCp can be used to transfer data to cloud blob storage, and PaaS clusters are spun up for processing.
+
+**Pass-Through from On Premises to Cloud**
+
+![image](https://github.com/user-attachments/assets/8b92ece5-8eb6-4492-b1a8-7c0a0274a630)
+
+Similar to bursting, but the data that is moved to cloud storage can be processed by both on-prem tools and cloud-based processing engines, providing greater flexibility
+
+**Data Integration Through Streaming**
+
+We can use CDC to capture real-time data changes and enable real-time analytics, database replication, and event-driven architectures.
+
+### Adopting multicloud
+
+**Framework**
+
+Example: Architecture Development Model by The Open Group Architecture Framework (ADM by TOGAF)
+
+![image](https://github.com/user-attachments/assets/180a01cd-1793-4299-a627-4f63835d32cf)
+
+**Time scale**
+
+* some orgs may never fully migrate to a public cloud
+* others might be commited to multi-year transitions
+* there are orgs who are not ready to migrate
+
+It is important to identify the time scale at which multicloud is to be adopted
+
+**Define a target multicloud architecture**
+
+* define your strategy: clearly articulate business goals, drivers, and identify the time scale for multicloud adoption
+* develop the team: build a 'Cloud Center of Excellence' with diverse skillsets to address the complexities of a multicloud environment
+* assess: thoroughly evaluate existing workloads and identify suitable migration paths, considering replatforming or redevelopment
+* design the architecture: define communication, data exchange, and integration mechanisms between components to ensure flexibility, security, performance, and cost efficiency
+* prepare a migration plan: establish clear migration modalities, timelines, effort estimations, milestones, and interdependencies between activities
+* build the landing zone: Create foundational environments on each cloud platform, addressing identity management, network connectivity, security, monitoring, and cost management
+* migrate and transform: execute the migration plan, leveraging new services and potentially rebuilding applications using cloud-native tools
+
+### Why edge computing?
+
+_Edge computing in a nutshell is the architectural pattern that promotes the execution of the data processing right where the data is generated, even though it’s at the periphery of the architecture._
+
+**Benefits**
+
+* reliability: ensures functionality even with intermittent connectivity
+* legal/compliance: facilitates data processing adhering to local regulations
+* security: reduces risk of data exfiltration and strengthens data protection
+
+**Challenges:**
+
+* limited computing and storage: edge devices often have specialized hardware with constraints
+* device management/remote control: updating and monitoring devices can be challenging, especially in remote locations
+* backup and restore: implementing robust backup and restore mechanisms in potentially disconnected environments requires careful consideration
+
+### Edge Computing Architectural Patterns
+
+**Smart devices**
+
+Devices themselves possess sufficient processing power to execute complex logic, such as ML algorithms. Requires significant management effort but offers high flexibility.
+
+![image](https://github.com/user-attachments/assets/a0c15b01-0cdd-482c-b22c-c66d250d2165)
+
+**Smart gateways**
+
+'Dumb' devices/sensors connect to a centralized gateway that performs data processing and analysis.
+
+![image](https://github.com/user-attachments/assets/cfab1118-41fa-4028-87b3-e84d3098b0f2)
+
+**ML Activation on the Edge**
+
+Enabling edge devices to execute ML models for real-time decision-making involves data collection, analysis, model development and training (typically in the cloud), deployment to the edge, and feedback data collection for model improvement.
+
+![image](https://github.com/user-attachments/assets/c07f071a-117d-41fc-a499-108cffb1e4a3)
+
+---
+
+## [Data Mesh](https://learning.oreilly.com/library/view/data-mesh/9781492092384/)
+
+The time has come ~ finally 😆 to read the Data Mesh book
+
+## Prologue: Imagine Data Mesh
+
+A fictional company called Daff is presented. It's a global music and audio streaming company in 2022. Three years prior, they faced significant challenges in harnessing data for business value due to a centralised, monolithic data architecture and functionally siloed teams. This structure resulted in bottlenecks, slow data delivery, and limited data-driven experimentation.
+
+Daff recognized the need for a new approach and embraced data mesh. This entailed a significant organizational and cultural shift, alongside the development of a self-serve data platform.
+
+### Key Features
+
+* decentralized domain ownership: data ownership shifted to cross-functional domain teams, promoting data accountability and alignment with business objectives
+* data as a product: data is treated as a product with defined service-level objectives (SLOs), documentation, and discoverability, ensuring data quality and usability
+* self-serve data platform: a platform that provides tools and services for building, deploying, and consuming data products, enabling autonomy and agility in data sharing and experimentation
+* federated computational governance: global data governance policies are established through collaboration between domain representatives and enforced via the platform
+
+### Benefits of data mesh for Daff:
+
+Daff started with data mesh small, proved its benefits and continued growing. 
+
+* enhanced data culture: data mesh fostered a culture of data curiosity and experimentation across domains
+* accelerated data delivery: the self-serve platform and domain ownership enabled rapid data product creation and sharing
+* improved DQ and trust: data product standards and ownership led to higher data quality and trust among data users
+* scalability and agility: the data mesh architecture enabled the addition of new data products and connections, accommodating the company's growth and evolving business needs
+
+### Data Mesh Transformation Journey 
+
+Daff undertook a multi-year transformation journey, incrementally implementing data mesh principles, building platform capabilities, and delivering data-driven use cases. It was not just with a flip of a switch. They explored, and are expanding the mesh to different domains. Data mesh is a whole new way of thinking about data so it's not just doing the tech, it's also about changing people's views on data.
+
+The book was written when _data mesh [was] arguably still in the innovator and early adopter phase of an innovation adoption curve_ and this was 2022 which is just 2 years ago ~ but I have seen people talking about data mesh throughout this year so maybe it is picking up, or even still a good framework to aim for. 
+
+---
+
+## Data Mesh - Chapter 1 - Data Mesh in a Nutshell
+
+_Data mesh is a decentralized sociotechnical approach to share, access, and manage analytical data in complex and large-scale environments—within or across organizations._
+
+_Data mesh is a new approach in sourcing, managing, and accessing data for analytical use cases at scale._
+
+### The outcomes
+
+To get value from data at scale in complex and large-scale organizations, data mesh sets to achieve these outcomes:
+
+* respond gracefully to change: a business' essential complexity, volatility and uncertainty
+* sustain agility in the face of growth
+* increase the ratio of value from data to investment
+
+### The shifts
+
+Data mesh dimensions of change
+
+![image](https://github.com/user-attachments/assets/9e775e24-9fc1-4f1f-9741-49968ec7cb9e)
+
+Is data mesh an architecture? a list of principles? an operating model? 
+
+The author has decided to classify it as a _sociotechnical paradigm_ - an approach that recognizes the interactions between people and the technical architecture and solutions in complex organizations
+
+### The principles
+
+In the simplest form data mesh can be described through four interacting principles
+
+**Principle of domain ownership**
+
+_Decentralize the ownership of analytical data to business domains closest to the data—either the source of the data or its main consumers. Decompose the (analytical) data logically and based on the business domain it represents, and manage the life cycle of domain-oriented data independently._
+
+Benefits:
+
+* the ability to scale out data sharing aligned with the axes of organizational growth
+* optimization for continuous change
+* enabling agility
+* increasing data business truthfulness
+* increasing solutions' resiliency
+
+**Principle of data as a product**
+
+_With this principle in place, domain-oriented data is shared as a product directly with data users—data analysts, data scientists, and so on._
+
+Data as a product adheres to a set of usability characteristics:
+
+* discoverable
+* addressable
+* understandable
+* trustworthy and truthful
+* natively accessible
+* interoperable and composable
+* valueable on its own
+* secure
+
+Motivations:
+
+* remove the possibility of creating domain-oriented data silos
+* create a data-driven innovation culture
+* create resilience to change with built-time and run-time isolation between data products and explicitly defined data sharing contracts
+* get higher value from data by sharing and using data across organizational boundaries
+
+**Principle of the self-serve data platform**
+
+_The platform services manage the full life cycle of individual data products. They manage a reliable mesh of interconnected data products. They provide mesh-level experiences such as surfacing the emergent knowledge graph and lineage across the mesh. The platform streamlines the experience of data users to discover, access, and use data products. It streamlines the experience of data providers to build, deploy, and maintain data products._
+
+Motivations:
+
+* reduce total cost of decentralized data ownership
+* abstract data management complexity and reduce the cognitive load of domain teams in managing the end-to-end life cycle of their data products
+* mobilize a larger population of devs to embark on data product development and reduce the need for specialization
+* automate governance policies to create security and compliance standards for all data products
+
+**Principle of federated computational governance**
+
+_This principle creates a data governance operating model based on a federated decision-making and accountability structure, with a team composed of domain representatives, data platform, and subject matter experts—legal, compliance, security, etc. The operating model creates an incentive and accountability structure that balances the autonomy and agility of domains, with the global interoperability of the mesh. The governance execution model heavily relies on codifying and automating the policies at a fine-grained level, for every data product, via the platform services._
+
+Motivation:
+
+* the ability to get higher-order value from aggregation and correlation of independent yet interoperable data products
+* countering the undesirable consequences of domain-oriented decentralizations
+* making it feasible to build in cross-cutting governance requirements such as security, privacy, legal compliance
+* reducing the overhead of manual sync between domains and the governance function
+
+### Interplay of the principles
+
+![image](https://github.com/user-attachments/assets/c32c4688-33d2-46d2-a411-882bcb666ac4)
+
+### Data mesh model at a glance
+
+![image](https://github.com/user-attachments/assets/8b1dc11d-f316-4662-86ed-0345df7ccb83)
+
+### The data
+
+_Data mesh focuses on analytical data. It recognizes the blurry delineation of the two modes of data, introduces a new model of tight integration of the two, and yet respects the clear differences between them._
+
+**Operational data**
+
+* supports running the business in real time through OLTP systems
+* captured, stored, and processed by applications, microservices, or systems of record
+* represents the current state of the business, designed for transactional integrity and frequent read/write access
+* private to apps/microservices but can be shared via APIs
+* used directly for business transactions and decision-making
+
+**Analytical Data**
+
+* provides a historical, integrated, and aggregated view of business data through OLAP systems
+* optimized for insights, comparisons, trends, and training ML models
+* includes intensive read access but minimal writes, with a focus on history and future predictions
+* fuels AI, analytics, and user experience optimization
+
+Chapters 2, 3, 4, and 5 are each about one of the 4 principles of data mesh. For them I decided to casually listen to the [audio book](https://learning.oreilly.com/videos/data-mesh/9781663721198/) since it's available on O'Reilly while also following the respective chapter's text. Tomorrow I will continue from chapter 6.  I think this book is very in depth and understandably so. Data mesh is a new way of working, designing, and thinking about data and these initial chapters (probably the later ones too) are something to reference back to. 
+
+---
+
+I started thinking about what to write on my last post 😆 I want to share some of the things on LinkedIn/X/Reddit to potentially help other beginners with the resources I found helpful. I need to write something more in-depth here on the blog, but something concise and eye-catching on social media 😆
+
+This is not related to the content on this post but something that I have been thinking about is putting model params in a yaml file and loading them from there so that we can version those and keep track. Last time I saw this was a few days ago from Maria Vechtomova's linkedin learning course. (I remembered to share it now when I was writing today's post 😆)
+
+That is all for today!
+
+See you tomorrow :)

From 746e96c8ae6f5b3bf67e8005df113b3bed4002f6 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 29 Dec 2024 22:46:27 +0900
Subject: [PATCH 569/638] Create 2024-12-29-day-363.md

---
 _posts/2024-12-29-day-363.md | 567 +++++++++++++++++++++++++++++++++++
 1 file changed, 567 insertions(+)
 create mode 100644 _posts/2024-12-29-day-363.md

diff --git a/_posts/2024-12-29-day-363.md b/_posts/2024-12-29-day-363.md
new file mode 100644
index 000000000..c50ce094d
--- /dev/null
+++ b/_posts/2024-12-29-day-363.md
@@ -0,0 +1,567 @@
+---
+layout: post
+title: (Day 363) Data mesh + K8s for MLOps
+categories: [theory,data-eng,mlops]
+---
+
+# Hello :) Today is Day 363!
+A quick summary of today:
+* read more of the Data Mesh book
+* covered Apache Pinot 201
+* started reading the book Real-World ML Systems on Kubernetes
+
+---
+
+## Data Mesh - Chapter 6 - The Inflection Point
+
+![image](https://github.com/user-attachments/assets/a2b587fe-dbe1-42a6-9ccb-35d77ea2c554)
+
+### Great Expectations of Data
+
+Companies have similar goals in mind:
+
+* provide the best customer experience based on data and personalization
+* reduce operational costs and time through data-driven optimizations
+* empower employees to make better decisions with trend analysis and BI
+
+_All of these scenarios require data—a high volume of diverse, up-to-date, and truthful data that can, in turn, fuel the underlying analytics and machine learning models._
+
+Before, companies' data aspirations were related to make good BI reports, but now it is beyond that - using ML, smart assistants, personalization, streamlining operations. These new expectations require a new approach to data management - an approach that can seamlessly fulfill the diversity of uses for data. 
+
+### The Great Divide of Data
+
+Many of the technical complexities companies face stem from the division of data between operational and analytical. 
+
+![image](https://github.com/user-attachments/assets/c283a29e-4bc1-4765-a647-f695c3b83e02)
+
+The two-plane data topology creates a fragile integration architecture, where operational data feeds analytical data via ETL pipelines lacking explicit contracts. This results in brittle pipelines prone to failures from upstream changes. Over time, these pipelines grow complex, managing bidirectional data flow. These challenges, along with reliance on centralized data warehouses or lakes, drive the need for improved solutions.
+
+![image](https://github.com/user-attachments/assets/8db9e964-fc97-46f8-8115-08e4e02bcdb6)
+
+### Scale: Encounter of a new kind
+
+The data industry has evolved to handle large-scale challenges in volume, velocity, and variety, with parallel processing, stream processing, and specialized storage systems addressing these needs. 
+
+However, the new challenge lies in managing data from diverse sources and locations. Unlocking insights requires connecting data from multiple sources rather than centralizing it. For example, intelligent healthcare and banking rely on integrating diverse data types beyond organizational control. This shift necessitates a decentralized approach to data management, focusing on connecting data wherever it resides.
+
+### Beyond order
+
+Organizations need to adapt to complexity, uncertainty, and volatility. Modern businesses experience continuous change reflected in their data, showing the need for real-time, high-quality, and trustworthy insights for agile decision-making. Future data management systems must:
+
+* enable rapid response to change as a fundamental feature, not an optimization
+* move away from rigid schemas toward flexible, dynamic models that embrace complexity
+* facilitate autonomous, peer-to-peer team collaborations
+* support multi-platform data access and management across cloud and on-premise environments
+dscape.
+
+### Aproaching the Plateau of Return
+
+The gap between investments in data and AI are highlighted. According to NewVantage Partners' annual reports, while most companies invest heavily in data and AI (99%, with 62% spending over $50M), only a minority report meaningful progress: 24.4% have a data culture, 24% are data-driven, and 41.2% compete using data. 
+
+Challenges include:
+
+* legacy systems
+* cultural resistance
+* competing priorities
+* skill gaps
+* governance issues
+* data usability friction
+
+Future data management strategies must address these barriers to achieve better returns on investment.
+
+## [llmops-python-package](https://github.com/callmesora/llmops-python-package)
+
+Apparently this package for LLMOps was announced yesterday (or this week). It includes tools for model registry, experiment tracking, and real-time inference, with a focus on automation, CI/CD, code quality, model management and observability. It's nice seeing packages like this. 
+
+## Data Mesh - Chapter 7 - Navigating the Inflection Point
+
+Data mesh is all about:
+
+* embracing change: data mesh assumes constant evolution and diversity in data and its uses as the natural state, enabling organizations to respond gracefully to change in a complex business environment
+* agility at scale: data mesh aims to sustain agility in the face of growth by removing organizational bottlenecks and reducing the need for coordination and synchronization
+* maximizing data value: data mesh strives to increase the value derived from data relative to investment by simplifying the data landscape and applying product thinking
+
+![image](https://github.com/user-attachments/assets/922837c7-9cf8-4ddc-898b-b7ae60fd8570)
+
+Key Ideas and Facts:
+
+### Responding Gracefully to Change
+
+* complexity as the default: data mesh acknowledges the inherent complexity of modern businesses and embraces the continuous evolution of data landscapes
+* alignment is key: the principle of domain data ownership aligns business, technology, and analytical data by assigning responsibility to those closest to the data. This echoes the decomposition seen in microservices architecture
+
+![image](https://github.com/user-attachments/assets/4dc5714f-b08c-4772-ab31-d77f63e5c6b3)
+
+* bridging the gap: data mesh closes the gap between operational and analytical data planes by integrating data products tightly with operational systems, enabling near real-time insights and faster feedback loops
+
+![image](https://github.com/user-attachments/assets/af19e8af-7a8d-4ee7-ade5-7840b4f739ad)
+
+* flexibility in data models: data mesh removes the need for a centralized canonical data model, allowing domains to evolve their models independently without hindering downstream data users. Well-defined data contracts ensure compatibility and smooth transitions
+
+### Sustaining Agility in the Face of Growth
+
+* remove centralised and monolithic bottlenecks: data mesh replaces centralized data lakes and warehouses with a peer-to-peer data sharing model, removing architectural bottlenecks and reducing reliance on intermediary data teams
+
+![image](https://github.com/user-attachments/assets/e3b0e638-1897-4df0-a912-7c8f199fbc07)
+
+* reduce coordination of data pipelines: data mesh shifts from technical partitioning of data management to a domain-oriented approach, reducing coordination overhead associated with data pipelines
+
+![image](https://github.com/user-attachments/assets/8a1d539c-705a-4f2d-b870-0f640e10431c)
+
+* reduce coordination of data governance: federated computational governance empowers domain data product owners with automated policy enforcement, reducing friction in data access and sharing
+
+![image](https://github.com/user-attachments/assets/389356f0-bd86-4dbe-92c1-52c1329b712d)
+
+### Increasing the Ratio of Value from Data to Investment
+
+* abstract technical complexity with a data platform: a data-product-developer-centric platform abstracts technical complexity, enabling generalist developers to manage data products, reducing reliance on specialized data engineers
+* open interfaces: standardized interfaces across data products foster interoperability and a collaborative ecosystem of technologies, lowering integration costs and encouraging innovation
+* embed product thinking everywhere: applying product thinking to both data and the data platform prioritizes user happiness, focusing on delivering value and a seamless user experience
+* go beyond the boundaries: while data mesh focuses on democratizing data access, the author acknowledges the need for continuous delivery of robust analytical and ML solutions to fully realize the value of data
+
+---
+
+## Data Mesh - Chapter 8 - Before the Inflection Point
+
+### Evolution of Analytical data architectures
+
+**First generation: The Warehouse**
+
+![image](https://github.com/user-attachments/assets/81b0333b-5288-4a06-9590-2deadaf7708e)
+
+**Second generation: The Data lake**
+
+![image](https://github.com/user-attachments/assets/0287be6b-c7f0-4875-8aeb-935be91c72a5)
+
+**Third generation: Multicloud**
+
+* support streaming for near real-time data avalability with architectures like Kappa
+* attempt to unify batch and stream
+* fully embrace cloud-based managed services and use modern cloud-native implementations with isolated compute and storage
+* converge the warehouse and lake into 1 technology
+
+### Characteristics of Analytical data architectures
+
+![image](https://github.com/user-attachments/assets/6563997f-6263-4f6d-8cdb-0f174e3e2a1e)
+
+There has been an explosion in the number of solutions in big data in AI
+
+But there are fundamental assumptions that have remained the same:
+
+* data must be centralised to be useful - managed by a centralised organisation, with an intention to have an enterprise-wide taxonomy
+* data management architecture, technology, and organisation are monolithic
+* the enabling technologies drive the paradigm - architecture and organisation
+
+**Monolithic structure**
+
+A monolithic architecture, technology stack, and organizational structure lead to bottlenecks, slow data delivery, and difficulty in adapting to evolving data needs - and one of the core assumptions of data mesh challenges this
+
+The 30,000-foot view of the monolithic data platform:
+
+![image](https://github.com/user-attachments/assets/72b308ef-2154-4095-acfa-ff0695cc915e)
+
+Siloed hyper-specialized data team:
+
+![image](https://github.com/user-attachments/assets/3fc721fc-524c-43c8-a6ea-d86fc557f6dc)
+
+**Centralised data ownership**
+
+This approach, while aiming to combat siloed data, creates a disconnect between data originators and consumers, impacting data quality and hindering domain-specific data understanding.
+
+Centralized data ownership with no clear domain boundaries:
+
+![image](https://github.com/user-attachments/assets/dad4d6cc-c397-4f0b-a754-ef50f0ce0e5b)
+
+**Technology oriented**
+
+Modern data architectures prioritize technical functions over business needs, creating silos and requiring heavy synchronization. Teams organized by activities (like ingestion or serving) struggle to deliver outcomes efficiently which can slow down innovation. Shifting to domain-oriented architectures embeds functions within business domains and enables faster and independent changes and better scalability.
+
+---
+
+## [Apache Pinot 201](https://www.youtube.com/playlist?list=PLihIrF0tCXdckH2BSA1D8l-QPGfVXEuFV)
+
+### What is a Pinot cluster? 
+
+At it's core, it's a collection of servers or processes that work together to handle ingestion, storage and query processes. A typical cluster consists of:
+
+<img width="676" alt="image" src="https://github.com/user-attachments/assets/c8c1f342-7b3b-40b5-8227-f61d316f16cc" />
+
+* Controller - manages cluster metadata
+* Broker - handles query routing
+* Server - stores and processes data in segments
+* Minions - perform different tasks
+
+I used [this github repo from StarTree](https://github.com/startreedata/learn/) to setup a local Pinot cluster and follow the exercises. 
+
+<img width="558" alt="image" src="https://github.com/user-attachments/assets/45bea4f5-4b77-4e41-810b-49cf37aa20cb" />
+
+One thing that is not ideal is that for every exercise there is a different docker compose file and each next one adds something. I will use the docker compose in the last exercise hoping it's sufficient and I won't need to run docker-compose for each exercise.
+
+This is what the Pinot UI looks like when I ran the last docker-compose:
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/89a8c297-3ffd-4a05-9f99-44ec15480718" />
+
+We can see servers, brokers, servers, minions, tenants, tables and minion task manager. 
+
+### Tenants
+
+A tenant is a logical component defined as a group of server/broker nodes with the same tag. Tenants can be defined for broker and/or server
+
+<img width="406" alt="image" src="https://github.com/user-attachments/assets/1d9c0102-22c5-48e3-b777-8a1c49ee3142" />
+
+**Broker tenants**
+
+Allow for separation of data and compute at the broker level. A sample use case might be a cluster that hosts data for multiple clients, and might need to segregate traffic as well as data.
+
+**Server tenants**
+
+Allow for separation of data and compute at a server level.
+
+A typical use case might be segregating data by department within a company of critical and non-critical tasks
+
+### Data Flow in Pinot
+
+* in the case of real time tables, data is written to a segment during the commit phase
+* the server responsible for the segments communicates the segment changes to the controller
+* the controller writes the segment to the deep store
+* tables can bypass the controller and load data directly into the deep store
+* in case of batch, the data is written directly to the deep store when configured
+
+We can insert tables through the CLI, the UI, or API. 
+
+**Exercise**
+
+Here the exercise was to add a table. When I open the main pinot url and go to /help, I see (like in FastAPI):
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/7a18fcf5-4b63-4037-b480-c20591f1c456" />
+
+This is the exercise:
+
+```
+- Navigate to the Swagger APIs located at [http://localhost:9000/help](http://localhost:9000/help)
+- Find the "Schema" section in the APIs.  
+- Find the POST - create schema, and click the button "Try It Out"
+- Paste the following code and hit Enter: {json schema for a table}
+```
+
+Added it:
+
+<img width="640" alt="image" src="https://github.com/user-attachments/assets/30662b91-dda1-46f0-b38f-1cce6f11312f" />
+
+I can also see it in the UI:
+
+<img width="1240" alt="image" src="https://github.com/user-attachments/assets/077e5ec7-ce38-402f-9728-1aefd779b782" />
+
+<img width="1246" alt="image" src="https://github.com/user-attachments/assets/bf6f629a-5e99-4490-be77-80f4c3a50a1d" />
+
+### Ingesting data
+
+* batch
+* streaming
+* hybrid
+
+**Exercise**
+
+I ran the docker compose for this specific exercise, and there already was a table:
+
+<img width="743" alt="image" src="https://github.com/user-attachments/assets/16561ad2-3bed-45c4-a28c-cdf918574eb7" />
+
+And I loaded data through the CLI:
+
+```
+curl -X POST -F file=movies.csv  -H "Content-Type: multipart/form-data"  "http://localhost:9000/ingestFromFile?tableNameWithType=movies_OFFLINE&batchConfigMapStr=%7B%22inputFormat%22:%22csv%22,%22recordReader.prop.delimiter%22:%22,%22%7D"
+
+Response:
+
+{"status":"Successfully ingested file into table: movies_OFFLINE as segment: movies_1735447958741_1735447958741_1735447958617"}%  
+```
+
+### How to query data
+
+**The single query engine**
+
+<img width="742" alt="image" src="https://github.com/user-attachments/assets/b85b9fb1-3cf2-4011-ad6c-fb17e4024e33" />
+
+**The multi stage query engine**
+
+<img width="668" alt="image" src="https://github.com/user-attachments/assets/d9188810-bdec-4c8c-9d0e-0bef428ddee9" />
+
+### Indexes exercise
+
+* create table and load data
+
+```bash
+docker exec -it pinot-controller sh
+/opt/pinot/bin/pinot-admin.sh AddTable -schemaFile /scripts/gitHub_events_schema.json -tableConfigFile /scripts/gitHub_events_offline_table_config.json -exec
+```
+
+* next, populate some data in the tables
+
+```bash
+cd data
+wget https://data.gharchive.org/2021-07-21-9.json.gz
+wget https://data.gharchive.org/2021-07-21-10.json.gz
+wget https://data.gharchive.org/2021-07-21-11.json.gz
+wget https://data.gharchive.org/2021-07-21-12.json.gz
+gunzip *.gz
+/opt/pinot/bin/pinot-admin.sh LaunchDataIngestionJob -jobSpecFile /scripts/job-spec.yaml
+```
+
+Next, I can create an index just by adding this:
+
+```json
+"bloomFilterColumns": [
+      "id",
+      "commit_author_names",
+      "label_ids"
+    ],
+```
+
+a bloom filter to the column 'commit_author_names' in the tableIndexConfig section
+
+(I wish 1 docker compose worked for all exercises. It does not as there are different config files and extra assets for each exercise. :/ )
+
+### Upserts
+
+* can be full or partial
+* utilise comparison columns to determine the latest iteration of data
+* snapshots help create resiliency in metadata map to stay on top of any lost in memory metadata related to upserts in case of server outage
+* enable preload helps faster recovery of upsert metadata map by using snapshots
+* StarTree index is not supported when upserts are on
+* to use upserts, a primary key must be defined in the schema config, as used to partition data
+* in the table config, define a section `upsertConfig` with the appropriate mode (full or partial; partial can use overwrite, increment, append, union, max, min strategies)
+
+(I feel like the exercises are a bit incomplete maybe rushed. Some parts are copy-pasted from previous exericses)
+
+### Transformations during ingestion
+
+We can use groovy functions.
+
+* to use Groovy functions, we need to modify the controller config to enable it, set `disabled = false`  
+  - `controller.disable.ingestion.groovy=false`  
+
+**Groovy Function Syntax:**  
+
+`Groovy({groovy script}, argument1, argument2...argumentN)`  
+
+**Example:**  
+
+```json
+"ingestionConfig": {
+  "transformConfigs": [{
+    "columnName": "fullName",
+    "transformFunction": "Groovy({firstName+' '+lastName}, firstName, lastName)"
+  }]
+}
+```  
+
+### Minions in Pinot
+
+* a minion is a standby component that leverages a task framework to offload computationally intensive tasks from other components
+* some typical minion tasks are: batch ingestion, segment creation, segment purge, segment merge
+
+<img width="663" alt="image" src="https://github.com/user-attachments/assets/75d2b0ec-446d-45a4-aace-7d2c993693a5" />
+
+### Use cases that work for Pinot
+
+**Real-time User Activity Monitoring**
+
+* monitoring user interactions, analyzing engagement, and detecting anomalies in real-time on large-scale platforms like social media
+* Pinot leverages Kafka integration to ingest massive data streams and deliver instant analytics
+
+**Fraud Detection**
+
+* rapid detection of fraudulent transactions in finance and e-commerce
+* Pinot can be used for real-time analysis of transaction patterns to flag suspicious activities as they occur
+
+**Real-time Personalization**
+
+* delivering personalized experiences across e-commerce, streaming services, and online gaming to boost user engagement
+* Pinot enables real-time recommendation engines that tailor content based on individual preferences and behaviours
+
+**Operational Analytics**
+
+* real-time monitoring of infrastructure, applications, and services to maintain efficiency and uptime
+* Pinot processes logs, metrics, and traces in real-time, providing actionable insights for smooth operations
+
+---
+
+## Data Mesh - Chapter 9 - The Logical Architecture
+
+### Domain ownership extends domains with analytical data sharing interfaces
+
+Domain ownership of data results in a domain-centric organization of analytical data - this means that a domain’s interfaces must extend to sharing its analytical data
+
+![image](https://github.com/user-attachments/assets/1055009e-1434-422f-a32b-80eb8b23dac0)
+
+### Data as a product introduces a new architecture quantum, aka data quantum
+
+A data quantum is the core building block of a data mesh architecture, encapsulating all components needed to ensure usability, secure data sharing, and policy enforcement. Each data quantum represents a domain-oriented data product, including the data itself, transformation code, sharing mechanisms, and governing policies. This concept forms the foundation of data mesh by treating data products as self-contained architectural units.
+
+![image](https://github.com/user-attachments/assets/65fc9e3c-c326-45b4-b23b-a09c5faa5120)
+
+### The self-serve data platform drives a multiplane platform architecture
+
+The self-serve platform provides a variety of services that enable different types of data mesh users to perform their roles effectively: data producers (such as data product developers and owners), data consumers (like data analysts and scientists), and data governance team members. The platform is structured into three planes, which consist of coordinated services organized according to the needs and experiences of data mesh users.
+
+![image](https://github.com/user-attachments/assets/b1d38c72-d732-4375-a81d-178234d8ede2)
+
+### Federated computational governance embeds computational policies into each data product
+
+The principle of federated computational governance enhances each data product with a computational container capable of hosting a sidecar process. This sidecar enforces computational policies at the appropriate stages of the data lifecycle—such as building, deploying, accessing, reading, or writing—integrating policy execution seamlessly into the data flow.
+
+A data product sidecar operates within the execution context of a data product, handling domain-independent, cross-cutting concerns like policy enforcement. It can also be extended to support standardized features like discovery. Notably, the sidecar's implementation remains consistent across all data products.
+
+![image](https://github.com/user-attachments/assets/2dadd971-9c13-437c-bc20-3afda23109a5)
+
+---
+
+## Data Mesh - Chapter 10 - The Multiplane Data Platform Architecture
+
+![image](https://github.com/user-attachments/assets/3f4d8afc-6c16-4dc7-963c-1e3076fd5323)
+
+### Design a Platform Driven by User Journeys
+
+_The ultimate purpose of the platform is to serve the cross-functional domain teams so they can deliver or consume data products. Hence, the best way to begin the design of the platform is to understand the main journeys of your platform users and evaluate how you can make it easy for them to complete their journeys successfully using the platform._
+
+High-level personas:
+
+* data product developer: from generalist developers with general programming skills to specialist data engineers who are well-versed in the existing analytical data processing technologies
+* data product consumers: they need access and use data to do their job (i.e. DSs, DAs, data product devs, app devs)
+* data product owners: deliver and promote domain-specific data products, ensuring adoption, value delivery, policy compliance, and interoperability using the platform
+* data governance members: operate within a federated structure, collectively ensuring secure and optimal mesh operations, with roles including security, legal experts, and data product owners  
+* data platform product owner: develops platform services as a product, prioritizing user needs to deliver an optimal experience for all user roles  
+* data platform developer: builds and operates the data platform, contributing to its design while using its utility and product experience services  
+
+### Data Product Developer Journey
+
+High-level example of the data product development journey:
+
+![image](https://github.com/user-attachments/assets/0cff3fd3-ee32-4f57-935e-e8de60f90e50)
+
+High-level example of the data product development journey using the platform:
+
+![image](https://github.com/user-attachments/assets/65d0fc60-c8b7-43c7-8b03-bce55a07f3fd)
+
+**Incept, Explore, Bootstrap, and Source**
+
+Data products begin with identifying real-world analytical use cases to demonstrate their value. Consumer-aligned data products are created through direct collaboration with their users. Developers explore potential data sources, which may include upstream data products, external systems, or organizational systems, assessing their suitability based on guarantees, documentation, and profiling. Once sources are identified, the developer bootstraps the data product using platform-provisioned infrastructure to connect to and experiment with source data. This exploratory phase includes rapid discovery, source access, quick scaffolding, and infrastructure setup.
+
+**Build, Test, Deploy, and Run**
+
+![image](https://github.com/user-attachments/assets/b4fa16f0-cab9-4c9c-a1b3-745bfd735982)
+
+Data product developers have an end-to-end responsibility of building, testing, deploying, and operating their data products. This stage is a continuous and iterative series of activities that data product developers perform to deliver all the necessary components of a successful data product.
+
+**Maintain, Evolve, and Retire**
+
+* maintaining and evolving a data product involves continuous updates to transformation logic, data models, access methods, and policies while ensuring uninterrupted data processing and sharing. The platform simplifies maintenance by handling complexities like schema versioning and resource management, allowing developers to focus on product-specific changes.  
+
+* evolution can range from minor updates, like bug fixes, to significant changes, such as migrating storage vendors. Monitoring operational health, including performance, reliability, and costs, is crucial, both for individual data products and the mesh as a whole, with insights and alarms for potential bottlenecks.  
+
+* data products may retire if migrated to new versions or if their data is no longer needed. The platform ensures graceful retirement, allowing downstream users to transition. Dormant products serve old data and enforce policies, while fully retired products cease to exist.
+
+**Example ML model development journey**
+
+![image](https://github.com/user-attachments/assets/d1e85c6d-d7cc-40e6-bd77-b779b41d0b11)
+
+### Data Product Consumer Journey
+
+**Incept, Explore, Bootstrap, Source**
+
+* inception: developers hypothesize intelligent actions or decisions based on existing data, such as creating a playlist loved by listeners
+* exploration: they search and evaluate data products for bias and fairness using apis and data sampling
+
+**Build, Test, Deploy, Run**
+
+* build: developers train models using data transformed into features (e.g playlist features), treating these transformations as data products
+* test: they track and version data using timestamps for model reproducibility
+* deploy: models are packaged to run as part of a data product and executed using specialized infrastructure like gpus or tpus
+
+**Maintain, Evolve, and Retire**
+
+* developers monitor model performance by evaluating user behavior metrics (playlist replay rates and listening duration)
+* metrics are collected using operational monitoring tools integrated with user applications
+
+---
+
+## Real-World ML Systems on Kubernetes
+
+Saw this intriguing book on Manning's MEAP and decided to get it with one of my free credits from when I had a sub. I want to learn more about K8s' application in MLOps and also the authors seem interesting - Re Alvarez Parmar - a Principal Specialist Solutions Architect at AWS, and Elamaran (Ela) Shanmugam - a Sr. Specialist Solutions Architect with AWS as well. 
+
+This is the TOC at the time of writing this post:
+
+<img width="528" alt="image" src="https://github.com/user-attachments/assets/6cfbc912-f2ef-4865-990c-c6a50c140113" />
+
+---
+
+## Real-World ML Systems on Kubernetes - Chapter 1 - Accelerating machine learning innovation using MLOps and Kubernetes
+
+The complexity of taking ML models to production has led to the rise of MLOps, a blend of ML and DevOps, focused on creating production-grade ML systems. MLOps integrates software engineering, data management, and operational principles to streamline the development and scaling of machine learning models. Kubernetes, an open-source platform for managing containerized applications, plays a vital role in MLOps by enabling scalability, flexibility, and alignment with software development best practices.
+
+This book explores designing scalable MLOps platforms using Kubernetes, offering insights into improving developer productivity, managing systems at scale, and ensuring reliability. Kubernetes' ability to run on diverse infrastructures and its widespread adoption makes it an ideal foundation for modern MLOps solutions.
+
+### The need for speed
+
+Developing machine learning applications is a highly iterative process - EDA, features, hyperparams - getting a good model might require lots of experimenting. To make this faster we could use some parallelisation to conduct experiments concurrently and reduce the time to learn what works and doesn't. 
+
+MLOps is the application of DevOps principles to machine learning engineering, and automation starts as soon as developers merge their code changes into a central repository. Changes in code trigger automated builds and tests to ensure that the code meets standards and is free of any known bugs. And since there is a lot of iterativeness in model development, adopting DevOps practices is a good idea for MLOps engineers. 
+
+### Developing ML systems
+
+Here the CRISP-DM process is mentioned (which was also referenced in the CRISP-ML(Q) paper), and then the CRISP-ML(Q) itself is mentioned as well. 
+
+CRISP-DM consists of 6 major phases of data mining:
+
+1. business understanding
+2. data understanding
+3. data preparation
+4. model training
+5. evaluation
+6. deployment
+
+And CRISP-ML(Q) adds a 7th - monitoring and maintenance, to address the unique challenges in ML. 
+
+Typical ML lifecycle:
+
+![image](https://github.com/user-attachments/assets/496cc537-8b9d-4b31-87bf-b7efd5b9c053)
+
+And it is a circle - an ongoing process.
+
+### MLOps platforms
+
+An MLOps platform streamlines the development of machine learning systems by providing end-to-end capabilities that support collaboration among data engineers and machine learning engineers. It automates hardware provisioning, software configuration, and workflows for tasks like data processing, model training, deployment, and monitoring, enabling efficient and scalable solutions.
+
+K8s-based MLOps platforms enhance efficiency by automating repetitive tasks and offering services tailored to different development roles. Some features include single sign-on, unified logging and monitoring, resource management, version control, collaboration tools, and integration with machine learning frameworks, creating a seamless environment for building and deploying machine learning systems.
+
+And building an ML system requires collaboration between different roles
+
+![image](https://github.com/user-attachments/assets/327287d3-875d-4a66-a4d4-78b1f3aa2a7a)
+
+### Benefits of building MLOps on K8s
+
+* the platform will be cloud-native
+* K8s has a vibrant community
+* it enables the creation of immutable infrastructure, enhancing a system’s consistency, predictability, and security
+* it abstracts the underlying infrastructure
+* provides a highly scalable infrastructure for deploying and managing machine learning models
+* supports scalable distributed training to speed up model development
+* provides the scalable infrastructure for distributed data ingestion data analytics, providing near-native integration with tools like Airflow, Spark, and others
+* integrates seamlessly with CI/CD pipelines
+* has extensive support observability through Prometheus, Grafana, and 3rd-party tools
+* offers capabilities to isolate workloads, making it ideal for multitenant scenarios
+* it supports policies and the enforcement of security standards in clusters
+
+### When to choose K8s as your ML platform
+
+* your organization is already familiar with it
+* you prefer to build your own systems instead of buying solutions
+* you must remain cloud agnostic at all costs
+
+---
+
+Today I wrote about ~ 8 pages on google docs for my final post. I am not sure if this will be the final version as I keep changing things. I don't want to be philosophical but I also want to share my experience and mindset throughout this journey ~ we'll see ~
+
+That is all for today!
+
+See you tomorrow :)

From ec240d442a3e87f43c97504702e3a0e3a873c596 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 30 Dec 2024 22:41:50 +0900
Subject: [PATCH 570/638] Create 2024-12-30-day-364.md

---
 _posts/2024-12-30-day-364.md | 555 +++++++++++++++++++++++++++++++++++
 1 file changed, 555 insertions(+)
 create mode 100644 _posts/2024-12-30-day-364.md

diff --git a/_posts/2024-12-30-day-364.md b/_posts/2024-12-30-day-364.md
new file mode 100644
index 000000000..c0e311a8e
--- /dev/null
+++ b/_posts/2024-12-30-day-364.md
@@ -0,0 +1,555 @@
+---
+layout: post
+title: (Day 364) Learning about LangSmith and reviewing K8s 
+categories: [theory,mlops,nlp,applying-knowledge]
+---
+
+# Hello :) Today is Day 364!
+A quick summary of today:
+* read Real-World ML Systems on Kubernetes - Chapter 2
+* read Software Engineering for Data Scientists - Chapter 1
+* read a white paper by Google on MLOps
+* streamed for the last time
+
+---
+
+## Real-World ML Systems on Kubernetes - Chapter 2 - Fundamentals of Kubernetes
+
+_At its core, Kubernetes is a highly extensible orchestration system for containerized workloads._
+
+K8s is cloud-agnostic and it's one of the reason for its wide adoptation by businesses.
+
+### K8s architecture
+
+Consists of 2 major components:
+
+* control plane which comprises the internal components that K8s needs to function
+* data plane where apps run
+
+![image](https://github.com/user-attachments/assets/7b18680b-bbe1-42e6-951a-fcd575ba49b7)
+
+**The control plane**
+
+This is where the brains of a K8s cluster live and comprises a highly available set of servers running K8s' system processes and a database
+
+* on the cloud, the control plane is managed by the cloud provider (maintaining, scaling, upgrading, monitoring, troubleshooting)
+
+Components:
+
+* api server: the `kube-api-server` is the central control point of the K8s cluster; it exposes the K8s API which is used by both the internal and external components to interact with the cluster
+* controller manager: runs a collection of controllers that manage the state of the cluster - it ensures the current state matches the desired state
+* scheduler: decides where to run a particular workload
+* cloud controller manager: manages cloud specific resources like VMs, networking and storage
+* ETCD: a cluster's state is persisted in  highly-available etcd key-value store and the API server interacts with it to read/write this state
+
+**The data plane**
+
+Consists of the infrastructure that executes the workloads we deploy. Technically, we can run workloads on the same hosts that run a cluster's control plane, but this is an uncommon practice available only in self-managed clusters.
+
+Key components:
+
+* nodes: VMs that run containers
+* CNI: provides networking for containers
+* CoreDNS: DNS server in a cluster
+
+More in-depth:
+
+* nodes run 3 processes: container runtime - responsible for creating, running, stopping containers (i.e. containerd, docker, CRI-O); the kubelet - acts as the agent between the K8s control plan and the container runtime; Kube-proxy - implements K8s network abstraction by maintaining network rules
+* autoscaling nodes: a typical data plane is amde up of a group of nodes that provide resources to run workloads; over time nodes might run out of resources and more nodes need to be added, and this autoscaling is usually done using K8s Cluster Autoscaler
+* Container Network Interface (CNI): provides network connectivity to containers that run inside a K8s cluster
+* CoreDNS: handles name resolution and provides service discovery within a cluster
+
+**Interacting with a K8s cluster**
+
+The primary way is through the `kubectl` CLI. 
+
+![image](https://github.com/user-attachments/assets/df8cee9a-df12-4aaa-8f3d-cdbc50c3d452)
+
+Common commands:
+
+| Command                           | Operation                                      |
+|-----------------------------------|------------------------------------------------|
+| `kubectl get`                     | List all resources                            |
+| `kubectl describe`                | Describe a resource                           |
+| `kubectl delete`                  | Delete a resource                             |
+| `kubectl edit`                    | Edit a resource                               |
+| `kubectl logs`                    | View container logs                           |
+| `kubectl patch`                   | Update a resource                             |
+| `kubectl apply -f filename.yaml`  | Create or update a resource                   |
+| `kubectl config view`             | View kubectl configuration                    |
+| `kubectl events`                  | Show events                                   |
+| `kubectl exec`                    | Execute a command inside a running container  |
+| `kubectl port-forward`            | Forward a port on the local machine to a Pod  |
+| `kubectl run --image=<container_image>` | Create a Pod from an image                 |
+
+### K8s Objects
+
+Objects are persistent entities that represent the state of the K8s cluster. When an object is created, the K8s system will constantly work to ensure that object exists, and its desired state is maintained. 
+
+**Containers**
+
+Popularised by Docker, containers are bundled packages that contain all code, dependencies and configs needed to run an app
+
+**Pods**
+
+A pod contains one or more containers that run on the same node. Typically, there is one container running the main app and sidecar containers that run processes that provide helper functions
+
+![image](https://github.com/user-attachments/assets/6276bc24-c41e-46be-834e-35563cc9de84)
+
+All containers within a Pod run on the same node. They also share the Linux network namespace, which enables them to intercommunicate using localhost
+
+_Containers in a Pod always run on the same node. They can intercommunicate using the loopback interface. Every Pod gets an IP address where it is accessible. Pods intercommunicate with each other using Pod’s IP address (or DNS name)_
+
+![image](https://github.com/user-attachments/assets/08fc1fbf-e6f8-49dd-9f15-61232485f8d3)
+
+A pod has an IP address, and other pods in the cluster can connect to other pods using that IP address. The pod's private IP address is inaccessible from outside the cluster and we cannot connect to the pod directly. However, we can use port forwarding to test an app running inside a cluster. 
+
+Port forwarding tunnels traffic through the Kubernetes API server and Kubelet. This tunnel is temporary and exists only for the duration of the kubectl port-forward command.
+
+![image](https://github.com/user-attachments/assets/d585237b-fdaf-4245-890f-c2e88c45eed4)
+
+* resource allocation: within a pod we can define the min and max resources a container gets and the scheduler considers resource requirements when allocating a pod to a node; if no nodes meet the min resource requirements, the pod will not be scheduled onto any node and will remain in a `pending` state until a suitable node is available for scheduling
+* probes: K8s can detect failire in an app container and restart or stop sending it more work; probes are used to determine the health and readiness of containers in a pod and are defined per container
+* init containers: all containers in a pod are started simultaneously; init containers are special containers that run before the main app container starts and their job is to handle any initialisation or config tasks that need to happen before the main app can start (i.e. if a website needs to wait for a db to start so it can connect to it)
+* sidecar containers: a sidecar container can be created by setting an init container’s `restartPolicy` to Always. This means the container starts before the main application, ensuring setup tasks are completed, and then continues running for the Pod's entire lifetime, acting as a sidecar. This combines the benefits of init containers’ startup order with sidecar containers’ persistence
+
+**Deployments**
+
+Apps are deployed in a K8s cluster by creating Deployments which contain declarative configuration that instructs K8s how we'd like to manage and update a workload. 
+
+A deployment is made up of one or more pods, giving us the ability to manage a group of identical pods as single resource. 
+
+Behind the scenes, when a Deployment is created, K8s creates a Replica Set which represents a group of pods. This Replica Set maintains the number of desired replicas specified in the config. 
+
+K8s Deployments are perfect for horizontal scaling as we can increase/decrease the number or replicas depending on traffic and optimize resource utilization. 
+
+Horizontal Pod Autoscaler (HPA) helps with automatically scaling the number of Pods in a Deployment. It has configurable scaling policies that define the min and max number of Pods as well as the rate at which Pods are added/removed. 
+
+To update a deployment in Kubernetes, create a new container image for the application, push it to a container registry, and update the deployment's image field. Kubernetes performs a rolling update by default, creating new Pods with the updated image and terminating old ones gradually to ensure availability. Alternatively, the 'Recreate' strategy can be used, where all old Pods are terminated before new ones are created, causing temporary downtime.
+
+We can also easily rollback deployments if needed
+
+**Services**
+
+As Deployments scale up and down, Pods get created and destroyed, how can apps in the cluster connect to pods that may have different IPs? The Services acts as a stable network endpoint that our apps can use to find and commuicate with each other. A service gives us a stable IP address and DNS name that doesn't change as we scale and update Pods.
+
+There are different types of K8s Services:
+
+* ClusterIP: exposes the service on a cluster-internal IP, allowing other apps running in the cluster to access it
+* NodePort: exposes the service on each node's IP at a static port, making it accessible from outside the cluster
+* LoadBalancer: provisions a cloud-provided load balanced and assigns it a fixed, external IP to access the service from outside the cluster
+* ExternalName: maps the service to an external DNS name, allowing internal apps to access external services
+
+_A Kubernetes Service of type LoadBalancer is designed to expose an application to the public internet by provisioning a cloud load balancer. The LoadBalancer Service will have a public IP address or DNS name that external users can connect to. The load balancer then forwards traffic to the appropriate Kubernetes Service and pods._
+
+![image](https://github.com/user-attachments/assets/ed7e8948-788b-4ae8-86d3-e1889b7b9fcc)
+
+In the cloud, we'll mostly deal with ClusterIP (internal) and LoadBalancer (external). 
+
+A service uses labels and selectors to identify the group of Pods to rout to traffic to.
+
+* labels are key-value pairs attached to K8s objects like pods, deployments services, nodes, others; labels are applied to K8s objects to provide identifying metadata
+* selectors are used to target a set of objects based on their labels;
+
+**Namespaces**
+
+_With namespaces, each team can deploy their objects, like apps and services, into their own designated area of the Kubernetes cluster. They can manage and modify their own stuff without interfering with what the other teams are doing._
+
+A pod can connect to any service in the same namespace using just the name of the services. If a Pod wants to call a service in another namespace, it must add the target's namespace to the DNS name. 
+
+**Ingress**
+
+A LoadBalancer Service provides an external entry point for a single K8s Service, giving each service its own public IP and DNS name. While simple, managing multiple LoadBalancers for numerous services can become expensive and complex.
+
+An Ingress, on the other hand, acts as a centralized entry point for multiple services, routing traffic based on URL paths or hostnames 
+
+![image](https://github.com/user-attachments/assets/db2d1868-5037-4496-913a-ab89775f79f7)
+
+**Volumes**
+
+Volumes can be used to attach persistent storage to a container, allowing the app running inside the container to read and write data to a designated location.
+
+For ML workloads, object storage systems (like Amazon S3, GCS) are preferred for their scalability, durability, and compatibility with ML frameworks like TensorFlow and Spark. Applications access data via embedded code or mounted storage systems.
+
+K8s also simplifies traditional storage integration through the Container Storage Interface (CSI). CSI allows storage vendors to develop drivers for seamless K8s compatibility, supporting various storage types (block, file, object). This decoupled approach lets storage providers innovate independently while Kubernetes users access modern storage features efficiently.
+
+We can add storage to workloads through:
+
+* static provisioning: administrators manually allocate Persistent Volumes (PVs) beforehand. Persistent Volume Claims (PVCs) are then created by users to request and attach storage to Pods
+* dynamic provisioning: K8s automatically provisions storage volumes based on PVC requests and a defined StorageClass, simplifying management for dynamic workloads
+
+Persistent Volume (PV) are the actual storage resource. Access modes include:
+* ReadWriteOnce: single Pod access
+* ReadOnlyMany: multiple Pods with read-only access
+* ReadWriteMany: multiple Pods with read-write access
+
+Persistent Volume Claim (PVC) specify storage requirements (size, access mode) and is used by Pods to bind to PVs.
+
+**ConfigMaps**
+
+Stores app configuration as key-value pairs, enabling us to externalize settings like db connection strings or logging levels. It decouples configuration from application code, allowing the configuration to be provided to containers as environment variables or files
+
+**Secrets**
+
+Similar to ConfigMaps but for sensitive information.
+
+### Jobs
+
+The key difference between a Job and a regular Deployment is that a Job will run a task to completion and then stop, rather than continuously running the application.
+
+There are 3 types of jobs:
+
+* non-parallel jobs: simplest type, designed to run a single task to completion
+* multiple parallel jobs: for tasks that can be parallelised, like processing a large batch of data
+* parallel jobs with fixed completions: similar to multiple parallel jobs but with a specified number of successful completions required before the job is considered done
+
+We can schedule jobs in a cluster using CronJobs which are similar to traditional cron jobs.
+
+**StatefulSets**
+
+Manage stateful applications that require stable identities, persistent storage, and ordered deployment.
+
+* stable network identities: each Pod gets a predictable DNS name (e.g., web-0, web-1) that persists across rescheduling
+* persistent storage: Pods are bound to specific Persistent Volumes (PVs) using volumeClaimTemplates. Data persists even if a Pod is replaced
+* ordered Deployment and scaling: Pods are created, deleted, and scaled sequentially to ensure consistent behavior
+* headless Service integration: a headless Service (ClusterIP: None) provides direct DNS entries for each Pod, supporting stateful communication patterns
+
+**DaemonSets**
+
+This is for when we need a system service that we want to run on every single node in our cluster (like to collect logs or metrics) - we don't need multiple replicas, we need exactly 1 instance on each node. A DaemonSet ensures one copy of a Pod is running on every (or a selection of) nodes in a cluster. 
+
+### Advanced scheduling techniques
+
+In addition to Deployments, where we let K8s pick the node on which a workload should run, and a Daemon service which runs on every node, there are times when we might need finer control over which node runs a particular Pod.
+
+* node labels: we can apply labels to K8s nodes, and then use those labels to specify which nodes should run a particular Pod
+* affinity and anti-affinity: we can use affinity to schedule a Pod based node labels or other Pods running on the node
+* taints and tolerations – taints, like labels, are key-value pairs that we can add to nodes. When a node is tainted, K8s requires that a toleration is added to any Pod that should be scheduled on the tainted node
+
+This can be used in ML when we need only workloads tht need a GPU get placed on nodes with GPUs.
+
+### GitOps
+
+A practice where infrastructure is treated like source code, using Git as the single source of truth for infrastructure and runtime configurations. Changes to workloads or infrastructure are made via pull requests, and GitOps tools, like Flux or ArgoCD, automatically apply these changes to the cluster when detected. Unlike traditional infrastructure-as-code (IaC), which requires manual intervention to apply changes, GitOps automates this process. The infrastructure's actual state is continuously aligned with the desired state defined in the Git repository, managing Kubernetes cluster configurations like software deployments.
+
+![image](https://github.com/user-attachments/assets/b3e6adba-cd91-4437-aad7-19bdac5e917f)
+
+---
+
+## [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)
+
+This was another book on my ML-related books, and as the author says in the preface: _learning to write better code will let you do more data science_
+
+---
+
+## Software Engineering for Data Scientists - Chapter 1 - What is good code?
+
+### Why good code matters
+
+Good code is crucial, especially when it integrates with larger systems, like ML models in production or tools for other data scientists. As projects grow, the importance of good code increases. 
+
+'Code as craft' - much like a carpenter takes pride in building a well-made wooden cabinet. Good code is efficient, elegant, provides satisfaction and is easier to maintain.
+
+However, quick, poorly written code—often referred to as technical debt—can create problems down the line. It may involve missing documentation, bad variable names, or disorganized structure, all of which make the code harder to maintain. While technical debt is sometimes necessary due to deadlines or budget constraints, it can lead to more time spent fixing bugs later.
+
+### Adapting to changing requirements
+
+When writing code for ML, and other projects, change is inevitable. And good code can be easility adapted to work well with such changes. Starting to write good code from the beginning is important - setting up these best practices and following them and the benefits will come as a project grows. 
+
+The book divides clean code principles into: simplicity, modularity, readability, performance, and robustness.
+
+### Simplicity
+
+Simplicity is key in coding, as complex systems are harder to understand and modify. As projects grow, keeping all details in your head becomes impossible, and complexity can lead to unexpected issues, such as errors caused by missing steps in the workflow. 
+
+Accidental complexity can be reduced by keeping the code simple, avoiding repetition, and making it modular.
+
+Example principles are:
+
+* DRY
+* avoiding verbose code
+
+### Modularity
+
+Modular code:
+
+* makes the code easier to read
+* it's easier to locate where a problem comes from
+* it's easier to reuse code in the next project
+
+It's important to break down big tasks into smaller modules. It is not something that can be done right from the get go, but it's important to keep in mind and that our code will change as a project evolves.
+
+For instance, for a task like:
+
+![image](https://github.com/user-attachments/assets/59c73993-249f-4b27-855a-892048a01905)
+
+We can start with a skeleton for each function
+
+```python
+def load_data(csv_file):
+    pass
+
+def clean_data(input_data, max_length):
+    pass
+
+def plot_data(clean_data, x_axis_limit, line_width):
+    pass
+```
+
+### Readability
+
+`…​code is read much more often than it is written…​` - PEP8
+
+Code is for humans to read and for machines to execute. 
+
+We can follow best practices like PEP8, naming conventions, removing unnecessary prints, write docstrings
+
+### Performance
+
+Good code needs to be performant and this can be measured in running time and memory usage. This is especially important for product code.
+
+### Robustness
+
+This means code should be reproducible, and respond gracefully if system inputs change unexpectedly.
+
+Can be achieved through:
+
+* properly handling errors
+* logging
+* writing good tests
+
+I guess this would be a fine book to read. It's not long either.
+
+---
+
+## [Practitioners guide to MLOps - White paper by Google](https://services.google.com/fh/files/misc/practitioners_guide_to_mlops_whitepaper.pdf)
+
+The MLOps lifecycle addresses the challenges organizations face in deploying and managing machine learning (ML) systems effectively. Despite AI/ML being central to digital transformation, most organizations struggle to move beyond pilot projects, with many failing to deploy models into production or maintain them effectively once deployed.
+
+**Challenges**
+
+- manual and non-reproducible workflows  
+- inefficient handoffs between data scientists and IT teams  
+- lack of mid- to senior-level talent  
+- poor change-management processes and governance models  
+- challenges in deployment, scaling, and versioning  
+
+**The role of MLE**
+
+MLE integrates software engineering principles with the unique complexities of ML:
+
+- data preparation and maintenance  
+- monitoring and tracking model performance  
+- experimentation with data, algorithms, and hyperparameters  
+- continuous retraining on fresh data  
+- addressing data inconsistencies between training and serving environments  
+- ensuring model fairness and security  
+
+**MLOps defined**
+
+MLOps unifies ML development (ML) with system operations (Ops), automating and standardizing critical steps in building, deploying, and managing ML systems. It mirrors the role of DevOps in app development but is tailored to ML-specific problems, such as changing data patterns, adversarial risks, and model drift.
+
+**Benefits of MLOps**
+
+- shorter development cycles and faster time-to-market  
+- improved collaboration across teams  
+- enhanced reliability, scalability, and security  
+- streamlined governance and operational processes  
+- higher return on ML investments  
+
+In essence, MLOps provides the framework and tools necessary to ensure ML systems are deployed reliably, monitored effectively, and scaled efficiently, aligning them with evolving business goals.
+
+### ML-enabled systems
+
+<img width="828" alt="image" src="https://github.com/user-attachments/assets/2b1eb09f-340b-4c18-91c1-0c42cd27e843" />
+
+DE is essential for supporting operational, analytics, and ML tasks by ingesting, integrating, curating, and refining data. Effective DE processes are crucial for the success note only for BI and analytics but also ML. ML models are built and deployed using curated data, often provided by the DE team, and integrated into various application systems like BI tools and process control systems. Integrating models requires ensuring their effectiveness within applications and monitoring performance. Additionally, tracking relevant business KPIs helps assess the model's impact and guide adjustments.
+
+### The MLOps lifecycle
+
+<img width="513" alt="image" src="https://github.com/user-attachments/assets/2d3a3647-1850-4c0a-a7da-fc671734aab4" />
+
+* ML development consists of experimenting and developing a robust and reproducible model training procedure
+* training operationalisation is about automating the process of packaging, testing, and deploying repeatable and reliable training pipelines
+* continuous training is repeatedly executing the training pipes in response to new data or to code changes, or on a schedule
+* model deployment is about packaging, testing, and deploying a model to a serving env for online experimentation and production serving
+* prediction serving is about serving the model that is deployed in prod for inference
+* continuous monitoring is about monitoring the effectiveness and efficiency of a deployed model
+* data and model management is a central, cross-cutting function for governing ML artifacts to support auditability, traceability, and compliance; this can also promote shareability, reusability, and discoverability of ML assets
+
+### And end-to-end workflow
+
+<img width="747" alt="image" src="https://github.com/user-attachments/assets/58d088ec-9036-448e-93d0-608333dcdb10" />
+
+This is not a waterfall workflow that has to sequentially pass through all the processes. The processes can be skipped, or the flow can repeat a given phase or a subsequence of the processes.
+
+1. the core activity during the ML dev phase is experimentation. As DSs and ML researchers prototype model architectures and training routines, they create labeled datasets, and they use features and other reusable ML artifacts that are governed through the data and model management process. The primary output of this process is a formalized training procedure, which includes data preprocessing, model architecture, and model training settings
+2. if the ML system requires continuous training, the training procedure is operationalized as a training pipe. This requires a CI/CD routing to build, test, and deploy the pipe to the target execution env
+3. the continuous training pipe is executed repeatedly based on retraining triggers, and it produces a model as output. The model is retrained as new data becomes available, or if model performance decay is detached. Other training artifacts and metadata that are produced by a training pipeline are also tracked. If the pipeline produces a successful model candidate, that candidate is then tracked by the model management process as a registered model
+4. the registered model is annotated, reviewd, and approved for release and is then deployed to a prod env. This process might be relatively opaque if you are using a no-code solution, or it can invovle building a custom CI/CD pipeline for progressive delivery
+5. the deployed model serves predictions using the deployment pattern that you have specified: online, batch, or streaming preds. In addition to serving predictions, the serving runtime can generate model explanations and capture serving logs to be used by the continuous monitoring process
+6. the continuous monitoring process monitors the model for predictive effectiveness and service. The primary concern of effectiveness performance monitoring is detecting model decay - for example, data and concept drift. The model deployment can also be monitored for efficiency metrics like latency, thoughput, hardward resource utilisation and exeuction errors
+
+### MLOps platform capabilities
+
+To implement MLOps effectively, orgs should establish core technical capabilities, which can be provided by a single ML platform or a combination of vendor tools and custom services. 
+
+Key MLOps platform capabilities include foundational infrastructure (reliable, scalable, and secure compute resources), configuration management, and CI/CD tools. Core MLOps capabilities include experimentation, data processing, model training, evaluation, serving, online experimentation, monitoring, pipelines, and model registries. Additionally, cross-cutting capabilities such as an ML metadata repository and an ML dataset/feature repository are necessary for integration and interaction across workflows.
+
+<img width="485" alt="image" src="https://github.com/user-attachments/assets/c8a179c5-54ca-42c3-9230-9c1e8494df47" />
+
+**Experimentation**
+
+* provide notebook envs that are integrated with version control
+* track experiments, include info about the data, hyperparams, and eval metrics for reproducability and comparison
+* analyse and visualise data and models
+* support exploring datasets, finding experiments, and reviewing implementations
+* integrate with other data services and ML services in your platform
+
+**Data processing**
+
+* support interactive execution for quick experimentation and for long-running jobs in prod
+* provide data connectors to a wide range of data sources and services, as well as data encoders and decoders for various data structures and formats
+* provide both rich and efficient data transformations and ML feature engineering for structured and unstructured data
+* support scalable batch and stream data processing for ML training and serving workloads
+
+**Model training**
+
+* support common ML frameworks and support custom runtime envs
+* support large-scale distributed training with different strategies for multiple GPUs and multiple workers
+* enable on-demand use of ML accelerators
+* allow efficient hparam tuning and target optimisation at scale
+* ideally, provide built-in AutoML functionality
+
+**Model evaluation**
+
+*  perform batch scoring of your models on eval datasets at scale
+*  compute pre-defined or custom eval metrics for your model on different slices of the data
+*  track trained-model predictive performance across different continuous-training executions
+*  visualise and compare performances of different models
+*  provide tools for what-if analysis and for identifying bias and fairness issues
+*  enable model behaviour interpretation using various explanable AI techniques
+
+**Model serving**
+
+* provide support for low-latency, near-real-time (online) preds and high-throughput batch (offline) prediction
+* provide built-in support for common ML serving frameworks for for custom runtime envs
+* enable composite prediction routines
+* allow efficient use of ML inference accelerators with autoscaling
+* support model explainability
+* support logging of prediction serving requests and responses for analysis
+
+**Online experimentation**
+
+* support canary and shadow deployments
+* support traffic splitting and A/B testing
+* support multi-armed bandit tests
+
+**Model monitoring**
+
+* measure model efficiency metrics
+* detect data skews, data concept shifts and drifts
+* integrate monitoring
+
+**ML pipelines**
+
+* trigger pipes on demand, schedule, in response
+* enable local interactive execution for debugging during development
+* integrate with ML metadata tracking
+* provide a set of built-in components for common ML tasks
+* run on different envs
+* optionally, provide GUI-based tools for designing and building pipes
+
+**Model registry**
+
+* register, organize, track, and version trained and deployed models
+* store model metadata and runtime dependencies for deployability
+* maintain model documentation and reporting
+* govern the model launching process
+
+**Dataset and feature repos**
+
+* enable shareability, discoverability, reusability, and versioning of data assets
+* allow real-time ingestion and low-latency serving for event streaming and online prediction workloads
+* allow high-throughput batch ingestion and serving for ETL process and model training
+* enable feature versioning for point-in-time queries
+* support various data modalities
+
+**ML metadata and artifact tracking**
+
+* provide traceability and lineage tracking of ML artifacts
+* share and track experimentation and pipe param configs
+* store, access, investigate, visualise, download and archive ML artifacts
+* integrate with all other MLOps capabilities
+
+### Deep dive of MLOps processes
+
+**ML development**
+
+<img width="616" alt="image" src="https://github.com/user-attachments/assets/b44821dd-93f6-4fd7-b897-f9fa395f4b15" />
+
+**Training operationalization**
+
+<img width="630" alt="image" src="https://github.com/user-attachments/assets/fdfeee03-7c8b-448f-a2a2-4562756a9cb6" />
+
+**Continuous training**
+
+<img width="636" alt="image" src="https://github.com/user-attachments/assets/9ceacb04-78b0-457c-8556-5e42daa1d093" />
+
+**Model deployment**
+
+<img width="614" alt="image" src="https://github.com/user-attachments/assets/c5df90a5-3820-49de-b5a8-38d3e0f6c959" />
+
+A more complex CI/CD system for model deployment:
+
+<img width="652" alt="image" src="https://github.com/user-attachments/assets/00d0cdce-d4f2-4c2a-86da-e33930d661fb" />
+
+**Prediction serving**
+
+<img width="426" alt="image" src="https://github.com/user-attachments/assets/9ea467e7-5d1d-4518-b406-afc54d243ae2" />
+
+**Continuous monitoring**
+
+<img width="575" alt="image" src="https://github.com/user-attachments/assets/7776fec1-42ab-4757-a424-a938b7660bcf" />
+
+**Data and model management**
+
+<img width="524" alt="image" src="https://github.com/user-attachments/assets/ae6e7977-010f-4354-bd6e-17f07fa949ae" />
+
+ML metadata tracking:
+
+<img width="515" alt="image" src="https://github.com/user-attachments/assets/94b29696-119f-4e5c-8e5d-c03f9654593a" />
+
+**Model governance**
+
+<img width="648" alt="image" src="https://github.com/user-attachments/assets/1a1cc614-d2e0-414d-931a-b111b49656d8" />
+
+### Putting it all together
+
+_Delivering business value through ML is not only about building the best ML model for the use case at hand. Delivering this value is also about building an integrated ML system that operates continuously to adapt to changes in the dynamics of the business environment. Such an ML system involves collecting, processing, and managing ML datasets and features; training, and evaluating models at scale; serving the model for predictions; monitoring the model performance in production; and tracking model metadata and artifacts._
+
+End-to-end MLOps workflow:
+
+<img width="544" alt="image" src="https://github.com/user-attachments/assets/c1688b3f-f73b-481a-9448-bfd0c55b09fc" />
+
+This white paper seems condensed, yet it is in-depth and detailed. Definitely something to refer back to when working on a project, preparing for an interview, or just thinking about MLOps. 
+
+---
+
+## [Streamed - Intro to LangSmit](https://youtu.be/GQog75OQ9ao)
+
+Today I saw that LangChain announced a new free course - [Introduction to LangSmith](https://academy.langchain.com/courses/intro-to-langsmith)
+
+Tldr - it's amazing, and the LangSmith platform is great for anyone doing LLM observability. It's important to note that I am just a single learning dev, and I saw that it's paid for companies ~ so maybe MLflow's free LLM observability features can suffice.
+
+They also showed this [Chat LangChain](https://chat.langchain.com/) webapp kind of like the Ask Astro for Airflow.
+
+<img width="1110" alt="image" src="https://github.com/user-attachments/assets/c6e8fe4b-972b-44bc-9a3d-48aa928778d1" />
+
+I also saw that this stream was my 100th video on my youtube channel. Nice ending
+
+---
+
+That is all for today!
+
+See you tomorrow :)

From 1b5da87e33cfbcb4394f139aef6383c48eb21c07 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 31 Dec 2024 17:04:17 +0900
Subject: [PATCH 571/638] Create 2024-12-31-day-365.md

---
 _posts/2024-12-31-day-365.md | 431 +++++++++++++++++++++++++++++++++++
 1 file changed, 431 insertions(+)
 create mode 100644 _posts/2024-12-31-day-365.md

diff --git a/_posts/2024-12-31-day-365.md b/_posts/2024-12-31-day-365.md
new file mode 100644
index 000000000..d02430cfb
--- /dev/null
+++ b/_posts/2024-12-31-day-365.md
@@ -0,0 +1,431 @@
+---
+layout: post
+title: (Day 365) Learning about feature stores
+categories: [theory,mlops]
+---
+`
+# Hello :) Today is Day 365!
+A quick summary of today:
+* watching feature store videos by Hopsworks academy
+* using the Hopsworks python API and the UI
+
+---
+
+## [Learning about Feature Stores w/ Hopsworks Academy](https://www.youtube.com/playlist?list=PLgN6fhzkSui_YzFsY6E1f_U86OATEYLC6)
+
+### Batch & Real-Time Machine Learning Systems; A better framework with Hopsworks Feature Store
+
+<img width="642" alt="image" src="https://github.com/user-attachments/assets/f040d770-841f-4a3d-a76a-ceb702d4b993" />
+
+They mention that an ML model trained on static data doesn't qualify as an ML system - it's a one-off ML experiment and we can generate one-off preds with that model on that static data. For an *ML system* we need to have new data coming in and the model will generate more value when it keeps making predictions on more and more data. 
+
+This is their model for building ML systems - they call it 'The Feature, Training, and Inference pipeline' model.
+
+<img width="649" alt="image" src="https://github.com/user-attachments/assets/a5e3fade-ca6a-4cd0-b67b-b7819b17e56d" />
+
+An ML pipeline consists of these 3 pipelines.
+
+The 3 pipes can be developed separately and is good for system modularity so that even different roles/teams develop/maintain each pipeline (if needed).
+
+<img width="425" alt="image" src="https://github.com/user-attachments/assets/29768efe-ada3-41a2-a72a-b22cccc00c71" />
+
+Pipelines also need to be orchestrated, and given the modularity we can run them on different platforms depending on what we have/use case
+
+<img width="424" alt="image" src="https://github.com/user-attachments/assets/77ac73dd-603b-40ce-b3bb-c4ca160c75c0" />
+
+We also need to make sure we test our pipelines. We need to test each part of our pipelines:
+
+<img width="318" alt="image" src="https://github.com/user-attachments/assets/1167ab61-17f3-46aa-a25b-5acc692509e0" />
+
+And then we can do A/B testing, etc. 
+
+When we have a new model, we might have a new set of features that were used to train that model so we need to connect model versions to feature versions. And Hopsworks helps with this.
+
+<img width="329" alt="image" src="https://github.com/user-attachments/assets/ab10bc8c-8d9b-485c-896c-c887cd35d60d" />
+
+Hopsworks stitches all these pipes together, and each of their outputs can be stored in Hopsworks as well
+
+<img width="430" alt="image" src="https://github.com/user-attachments/assets/8e0eea7d-e9e8-45d5-be51-b148ed406569" />
+
+### Feature Pipelines in Production with Hopsworks: Code, Deployment & Monitoring
+
+Pipelines in prod:
+
+* managing the codebase - like github
+* deployment
+
+We can schedule and run jobs from the Hopsworks UI:
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/66ca368c-ee54-46d9-8c95-971fce4b9208" />
+
+We can also see more detailed info about each run:
+
+<img width="750" alt="image" src="https://github.com/user-attachments/assets/0dcac79d-5c85-4575-8b0e-fee2de84c04f" />
+
+* monitoring - we can set alerts
+
+### What is a feature store for ML ?
+
+<img width="1300" alt="image" src="https://github.com/user-attachments/assets/bf8e51dd-93ef-4e9d-812d-b1e502ca07ed" />
+
+They help:
+
+* maintain models
+* train models
+* improve model
+
+Features are the data for AI, and as such can be regarded as the fuel for AI. 
+
+<img width="705" alt="image" src="https://github.com/user-attachments/assets/83b48935-9ea5-4cf5-ba2d-7f55b7caec91" />
+
+Features are used throughout all the stages of an ML lifecycle and managing them can be a challenge as they need to be properly organised, stored and readily available for models. This is where the feature store comes in. It manages, stores, and provides access to both historical and real-time features. 
+
+Who does it help?
+
+Mostly DEs, DSs, MLEs
+
+<img width="632" alt="image" src="https://github.com/user-attachments/assets/be2277e0-7c1b-4989-94cb-a4a0130a1423" />
+
+The feature store helps make more models and productionise them, faster. It's value comes in its ability to bridge the gap between different stages of the model lifecycle and the many stakeholders involved. 
+
+### Deploying Managed Hopswork Via the UI or Terraform
+
+This video was just showing how to set up Hopsworks either through the UI - connecting with AWS/GCP/Azure, or using Hopswork's pre-set Terraform platform. 
+
+### Real-Time Inference: How to retrieve data from Hopsworks
+
+The Hopsworks API provides us with very low-latency options for retrieving features for real-time inference cases like fraud detection. 
+
+Features can be viewed in the Feature View:
+
+<img width="794" alt="image" src="https://github.com/user-attachments/assets/3f37bb96-e1c8-49eb-9a1f-25bff8f56e3e" />
+
+Once we click on fraud, we can see:
+
+<img width="866" alt="image" src="https://github.com/user-attachments/assets/67ed22cb-fc4e-4b4e-b883-3bb1635e4dbf" />
+
+More detailed info. We can even see the `label_encoder` tag which means that that column has had a transformation applied to it (i.e. scikit-leanr's le)
+
+And we can interact with this using Hopsworks' Python API.
+
+I will just give a list of the Hopsworks feature discussed as I will do a hopsworks tutorial I found after this ~
+
+* we can create train/test splits alongside extra info like stats and see them through the UI
+* creating a [feature view](https://www.hopsworks.ai/dictionary/feature-view): a curated selection of features (and optionally labels) from one or more feature groups within a feature store and serves as a logical representation tailored for specific machine learning models
+
+We can also see lineage:
+
+<img width="819" alt="image" src="https://github.com/user-attachments/assets/aa5680cd-fa7c-4fca-9c04-b3fad742a1b4" />
+
+* [external feature groups](https://docs.hopsworks.ai/3.2/concepts/fs/feature_group/external_fg/): allow users to register features stored in external systems (like Snowflake, BigQuery, S3) within Hopsworks, providing a unified platform for data scientists to access all features
+* streaming feature pipelines: we can use different streaming platforms (i.e. Spark Streaming, Beam, Flink)
+* [feature groups](https://www.hopsworks.ai/dictionary/feature-groups): a structured table of features with an online store for fast, real-time access to the latest feature values and an offline store for historical data used in training and batch inference
+* [versioning](https://docs.hopsworks.ai/feature-store-api/2.5.9/generated/versioning/): there is schema (metadata for feature views, feature groups - the features included, tags, other metadata) and data versioning (actual data versioning)
+
+---
+
+## Hopsworks Tutorial - Part 01: Feature Pipeline
+
+I created a profile on hopsworks.ai and there was a tutorial button with:
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/597a31b2-2958-4503-a527-ecafaf4c1541" />
+
+So I cliced on the 1st one and it took me to [this colab notebook](https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/churn/1_churn_feature_pipeline.ipynb).
+
+Loaded some data, and logged in Hopsworks with an API key ~ 
+
+<img width="692" alt="image" src="https://github.com/user-attachments/assets/05b61e17-e4a8-49ab-93ad-36bd846db01e" />
+
+* create a feature group
+
+```python
+customer_info_fg = fs.get_or_create_feature_group(
+    name="customer_info",
+    version=1,
+    description="Customer info for churn prediction.",
+    primary_key=['customerID'],
+    event_time="datetime",
+)
+```
+
+* insert data into the feature group
+
+```python
+customer_info_fg.insert(customer_info_df)
+```
+
+I inserted data for all 3 datasets:
+
+<img width="1511" alt="image" src="https://github.com/user-attachments/assets/41160b30-b4d7-4695-a221-87f4659d81ad" />
+
+Next is Part 2
+
+---
+
+## [Hopsworks Feature Store - Part 02: Training Pipeline](https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/churn/2_churn_training_pipeline.ipynb)
+
+![image](https://github.com/user-attachments/assets/9b42a0f4-d2c7-4cf0-a041-629e7ce049c0)
+
+**Load and select features from feature groups**
+
+```python
+# Retrieve feature groups
+customer_info_fg = fs.get_feature_group(
+    name="customer_info",
+    version=1,
+)
+
+demography_fg = fs.get_feature_group(
+    name="customer_demography_info",
+    version=1,
+)
+
+subscriptions_fg = fs.get_feature_group(
+    name="customer_subscription_info",
+    version=1,
+)
+
+selected_features = customer_info_fg.select_except(["customerid", "datetime"]) \
+    .join(demography_fg.select_except(["customerid"])) \
+    .join(subscriptions_fg.select_except(["datetime"]))
+```
+
+**Transformation**
+
+```python
+# Load transformation functions from the feature store
+min_max_scaler = fs.get_transformation_function(name="min_max_scaler")
+label_encoder = fs.get_transformation_function(name="label_encoder")
+
+# Define lists of numerical and categorical features
+numerical_features = ["tenure", "monthlycharges", "totalcharges"]
+categorical_features = [
+    "multiplelines", "internetservice", "onlinesecurity", "onlinebackup",
+    "deviceprotection", "techsupport", "streamingmovies", "streamingtv",
+    "phoneservice", "paperlessbilling", "contract", "paymentmethod", "gender", 
+    "dependents", "partner",
+]
+
+# Map features to their corresponding transformation functions
+transformation_functions = []
+
+# For numerical features, use the min_max_scaler transformation
+for feature in numerical_features:
+    transformation_functions.append(min_max_scaler(feature))
+
+# For categorical features, use the label_encoder transformation
+for feature in categorical_features:
+    transformation_functions.append(label_encoder(feature))
+```
+
+**Create a feature view**
+
+_The Feature Views allows schema in form of a query with filters, define a model target feature/label and additional transformation functions._
+
+```python
+feature_view = fs.get_or_create_feature_view(
+        name = 'churn_feature_view',
+        version = 1,
+        labels=["churn"],
+        transformation_functions=transformation_functions,
+        query=selected_features,
+)
+```
+
+I can see it in the UI
+
+<img width="672" alt="image" src="https://github.com/user-attachments/assets/9feb50c4-6860-420f-9fc2-b5eba0b8dc98" />
+
+<img width="1067" alt="image" src="https://github.com/user-attachments/assets/6d814467-99ba-4dc8-97ee-a2ccda92f33a" />
+
+
+**Split data, train a model, check evam metrics**
+
+Then we can get the Hopsworks model registry:
+
+```python
+mr = project.get_model_registry()
+```
+
+**Model schema**
+
+The model needs to be set up with a Model Schema, which describes the inputs and outputs for a model.
+
+A Model Schema can be automatically generated from training examples, as shown below.
+
+```python
+from hsml.schema import Schema
+from hsml.model_schema import ModelSchema
+
+# Create input schema using X_train
+input_schema = Schema(X_train)
+
+# Create output schema using y_train
+output_schema = Schema(y_train)
+
+# Create a ModelSchema object specifying the input and output schemas
+model_schema = ModelSchema(
+    input_schema=input_schema, 
+    output_schema=output_schema,
+)
+
+# Convert the model schema to a dictionary
+model_schema.to_dict()
+```
+
+Here is the dict model schema:
+
+```
+{'input_schema': {'columnar_schema': [{'name': 'seniorcitizen',
+    'type': 'int64'},
+   {'name': 'label_encoder_contract_', 'type': 'int64'},
+   {'name': 'label_encoder_dependents_', 'type': 'int64'},
+   {'name': 'label_encoder_deviceprotection_', 'type': 'int64'},
+   {'name': 'label_encoder_gender_', 'type': 'int64'},
+   {'name': 'label_encoder_internetservice_', 'type': 'int64'},
+   {'name': 'label_encoder_multiplelines_', 'type': 'int64'},
+   {'name': 'label_encoder_onlinebackup_', 'type': 'int64'},
+   {'name': 'label_encoder_onlinesecurity_', 'type': 'int64'},
+   {'name': 'label_encoder_paperlessbilling_', 'type': 'int64'},
+   {'name': 'label_encoder_partner_', 'type': 'int64'},
+   {'name': 'label_encoder_paymentmethod_', 'type': 'int64'},
+   {'name': 'label_encoder_phoneservice_', 'type': 'int64'},
+   {'name': 'label_encoder_streamingmovies_', 'type': 'int64'},
+   {'name': 'label_encoder_streamingtv_', 'type': 'int64'},
+   {'name': 'label_encoder_techsupport_', 'type': 'int64'},
+   {'name': 'min_max_scaler_monthlycharges_', 'type': 'float64'},
+   {'name': 'min_max_scaler_tenure_', 'type': 'float64'},
+   {'name': 'min_max_scaler_totalcharges_', 'type': 'float64'}]},
+ 'output_schema': {'columnar_schema': [{'name': 'churn', 'type': 'int64'}]}}
+```
+
+Then we can save articacts and the model:
+
+```python
+model_dir = "churn_model"
+
+if not os.path.isdir(model_dir):
+    os.mkdir(model_dir)
+
+model.save_model(model_dir + "/model.json")
+figure_cm.figure.savefig(model_dir + '/confusion_matrix.png')
+
+# Create a model in the model registry
+model = mr.python.create_model(
+    name="churnmodel",
+    description="Churn Model",
+    input_example=X_train.sample(),
+    model_schema=model_schema,
+)
+
+model.save(model_dir)
+```
+
+And I can see it in the UI:
+
+<img width="792" alt="image" src="https://github.com/user-attachments/assets/17f051e2-74d4-477e-bfbc-be1a5c5dc4e3" />
+
+And inside I can see all other info:
+
+<img width="616" alt="image" src="https://github.com/user-attachments/assets/1f391859-a044-4f7c-b721-b7cd05b7a883" />
+
+Next is Part 3
+
+---
+
+## [Fetch and test the model](https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/churn/3_churn_batch_inference.ipynb#scrollTo=38d83203)
+
+```python
+# Retrieve the model from the model registry
+retrieved_model = mr.get_model(
+    name="churnmodel",
+    version=1,
+)
+
+# Download the saved model files to a local directory
+saved_model_dir = retrieved_model.download()
+# Initialize the model
+model = XGBClassifier()
+
+# Load the model from a saved JSON file
+model.load_model(saved_model_dir + "/model.json")
+```
+
+And then we can use it as usual for predictions. 
+
+We can get batch data from our feature view:
+
+```python
+# Initialize batch scoring
+feature_view.init_batch_scoring(1)
+
+# Get the batch data
+batch_data = feature_view.get_batch_data()
+
+# Display the first three rows of the batch_data
+batch_data.head(3)
+```
+
+We can also load any transformers needed for features (i.e. min-max scaler)
+
+```python
+df_all = batch_data.copy()
+fv_transformation_functions = feature_view._batch_scoring_server.model_dependent_transformation_functions
+
+for transformation_function in fv_transformation_functions:
+    udf = transformation_function.hopsworks_udf
+    if udf.function_name == "min_max_scaler":
+        transformed_features = udf.transformation_features[0]
+        transformed_feature_name = udf.output_column_names[0]
+        stats = udf.transformation_statistics
+        df_all[transformed_features] = df_all[transformed_feature_name].map(lambda x: x*(stats.feature.max-stats.feature.min)+stats.feature.min)
+        
+    
+    if udf.function_name == "label_encoder":
+        transformed_features = udf.transformation_features[0]
+        transformed_feature_name = udf.output_column_names[0]
+        stats = udf.transformation_statistics
+        unique_data = sorted([value for value in stats.feature.unique_values])
+        index_to_value = {index: value for index, value in enumerate(unique_data)}
+        df_all[transformed_features] = df_all[transformed_feature_name].map(lambda x: index_to_value[x])
+```
+
+This comes at the end of the notebook, but we can also enable extra stats, correlations and graphs for feature groups:
+
+```python
+customer_info_fg = fs.get_feature_group("customer_info", version = 1)
+customer_info_fg.statistics_config = {
+    "enabled": True,
+    "histograms": True,
+    "correlations": True,
+}
+
+customer_info_fg.update_statistics_config()
+customer_info_fg.compute_statistics()
+```
+
+In the UI we can see:
+
+<img width="1451" alt="image" src="https://github.com/user-attachments/assets/e63d6944-a2e3-4214-a43d-bcad731404c6" />
+
+<img width="1459" alt="image" src="https://github.com/user-attachments/assets/d6e762e3-fcbf-4681-94f9-f7fbbd52bb0a" />
+
+<img width="1435" alt="image" src="https://github.com/user-attachments/assets/aef406b5-d050-423c-a8f7-519e93b01e7d" />
+
+On the [Hopsworks repo](https://github.com/logicalclocks/hopsworks-tutorials) they have a few more notebook tutorials which is great. 
+
+I saw that Zach Wilson posted two new lectures + labs but I forgot my tablet so I will cover them by myself tomorrow as well.
+
+I will do them on my own as I think this is the last study material I will post on the blog 😢 
+
+---
+
+Today's post is a bit shorter and earlier as I am going out to meet the New Year :) 
+
+Tomorrow I will post about my journey, what I learned, thoughts, list of books & courses, summary ~
+
+Happy New Year 🥳
+
+That is all for today!
+
+And for the last time ~ See you tomorrow :)

From 6b405ccc73357df1100886da8fa3129ef79ff2f5 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Tue, 31 Dec 2024 17:17:52 +0900
Subject: [PATCH 572/638] Update default.html

---
 _layouts/default.html | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 76af44206..c741dc5a8 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,10 +47,8 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
-          <h1>🔥Learn + Blog EVERY DAY for 366 days!💻</h1>
-          <h1>Math, Stats, Data, AI!</h1>
-          <h2>Ends Jan 2nd, 2025</h2>
-          <hr>
+          <h1>🔥2024 Learn + Blog EVERY DAY for 366 days!💻</h1>
+          <h1>Math, Stats, AI, Data Eng, MLOps!</h1>
           {{ content }}
         </div>
       </div>

From 112dadb861f2efa3a103bf2b48a83698b9e185a3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 1 Jan 2025 18:57:04 +0900
Subject: [PATCH 573/638] Create 2025-01-01-day-366.md

---
 _posts/2025-01-01-day-366.md | 165 +++++++++++++++++++++++++++++++++++
 1 file changed, 165 insertions(+)
 create mode 100644 _posts/2025-01-01-day-366.md

diff --git a/_posts/2025-01-01-day-366.md b/_posts/2025-01-01-day-366.md
new file mode 100644
index 000000000..11ac4737c
--- /dev/null
+++ b/_posts/2025-01-01-day-366.md
@@ -0,0 +1,165 @@
+---
+layout: post
+title: (The Final Day 366) This blog ends, but my learning journey continues
+categories: [the-end,thank-you]
+---
+`
+# Hello :) Today is Day 366! The last day of this blog!
+
+On January 2nd, 2024, I started with zero AI knowledge and self-studied it for 365 days.  
+
+---
+
+## My Background  
+
+I come from accounting and data analysis. Before the above date, my AI-related knowledge was pretty much non-existent - I knew about linear regression, R-squared, the power rule in differential calculus, and working experience with Python and SQL for data manipulation.  
+
+---
+
+## Books and Courses  
+Here is a list of books and courses I completed during these 365 days and my personal favourites: [My Learning List](https://docs.google.com/document/d/1cS6Ou_1YiW72gZ8zbNGfCqjgUlznr4p0YzC2CXZ3Sj4/edit?usp=sharing).  
+I summarised book chapters, talks, videos, shared my handwritten notes from online lectures, talked about projects, tried to stream and made some videos. I tried to make this final post concise while also discussing specific topics that might interest people who found this blog through social media and this is the first post they have seen from my blog. 
+
+---
+
+## Original plan 
+
+Originally, my plan for this learning journey was for it to be 50 days - during my winter break from university. At day 50 I decided to continue until day 100. Days before day 100 I, once again, knew that it was not over, so I continued and after a few days post day 100 I decided to make it a full year of learning and blogging. I never thought about the time constraint - I knew I had plenty of time on my hands as I was just a 2nd semester Master student (out of 4) so I didn’t set a deadline to ‘learn AI’ by some deadline. Will I have learned more if I was doing an AI degree at university, or I had someone as a guide? I cannot know. However, I think I liked my journey as is. Self-studying can be more challenging in the sense that I am more free to look around, but also can get lost in the ocean of materials to study. I believe I managed this ocean pretty well and learned a lot.
+
+---
+
+## Naming the Journey  
+
+When I started I said ‘learn AI’ because the whole blog needed some kind of name. However soon after I realised how ambiguous that name is and that AI contains many many things inside of it. After some time I changed the whole journey to ‘Learn Stats, Math, Data, AI’ as I realised that to understand ‘AI’ I need statistics, and mathematics, and for good AI I need good data. I will revisit these points about stats, math and data below.
+
+---
+
+## One word: Serendipity  
+
+I think a good word to describe my year/learning journey is serendipity. I enjoy difficult topics, I enjoy the process of learning - being a beginner and improving. I started with a random course on ML from caltech by Professor Abu-Mostafa from 2012. I cannot recall how I found it but it seemed like a place to start. I wrote this quote on day 1 of the blog from the 1st lecture from Prof. Abu-Mostafa:
+
+`If you look at linear models, you can think of them as an economic car. They get you where you want to go and they don’t consume a lot of gas. You may not be very proud of them, but they actually do the job. It is remarkable how often they succeed on their own and they are sufficient to get you the learning performance that you want. Try to use them first and see if they achieve what you want.`
+
+It is amazing how relevant that quote is as throughout my journey I saw different incarnations of it again and again. I mentioned serendipity because I was doing something I liked - studying something different, something new to what I have studied so far in my life - and courses, books, resources kept ‘appearing’ randomly. I joined the reddit communities for ML/DL, followed a bunch of people who had AI/ML in their profile descriptions, and suddenly I was flooded with new resources that I can study. So many different resources, how could I know which ones to go for first? I was without guidance, I was diving into this whole journey by myself without any friends or close people who are in the field giving me advice on where to go next. But I was not scared. I knew I had time, and I gave myself time to study, and repeat. The inflow of many resources never stopped, but I was determined to study them - which brings me to my next point - how I wanted to run this blog. 
+
+---
+
+## Philosophy for the Blog  
+
+I wanted to write this blog as if someone randomly opened it and could read the post and just learn something from it. I wanted to write proper (longer) posts about the topic/book/course I did on a day. For this, I imagined I had an annoying version of myself on my shoulder asking me ‘what did you study today?’, ‘are you descriptive enough in today’s post?’, ‘don’t just post that you read something - describe it, tell me about it’, etc. Thanks to this ‘philosophy’ I managed to go over different books and courses that many times talked about similar topics. It’s no surprise there are many books on AI, and many more keep coming out. There are many good and many not as good books - you can find my personal top books in the list. But what I want to share is that - the thing that remains constant in every book, paper or course is the fundamentals which leads me to my next point.
+
+---
+
+## Embracing the Fundamentals  
+
+Ohh the fundamentals - just like many others I see online (not sure about reality), in my 1st days I jumped directly into traditional machine learning and deep learning models and I could definitely make models, make notebooks that do cool predictions, but as soon as I started to ask the question ‘why’, ‘why does this happen’, ‘why does this work like that’ I felt helpless. Thankfully I am not scared of maths. As mentioned, you can run code, you can make a text generation model, but as soon as you start asking why it works, it will all fall apart. Well this led me to actually learn the math behind AI and also revisit my stats knowledge. I never avoided math or stats, I just started from a random point (like a model 😀) and after some time (epochs 😀) I learned where I needed to go and I started heading there. ‘There’ being math and stats for ML - again the books/courses for this can be found in my list. The idea of focusing on the fundamentals became a central point of my learning - try to not focus on specific technologies, rather understand the idea, the need, and the problem being solved. Tech changes, but fundamentals stay the same. I did not ignore tech of course, I did focus on the tech for practical knowledge and I learned a lot about tools for storing, transforming, loading data, model experiment tracking, orchestration, serving, monitoring, testing; and of course in projects I used specific tools, which can be seen on my blog’s Project tab. I intentionally don’t mention specific names again because tech changes, but the need behind tools stays the same. A recurring theme was studying so my knowledge is tool-agnostic. 
+
+---
+
+## Applying Knowledge  
+
+I realised I like to apply knowledge. When I was looking over my blog posts for what books/courses I read I saw time and time again that after I learned about some architecture/model, in the next few days I tried to make a notebook on Kaggle, or make a project that involves it - I like to apply knowledge. Further to applying knowledge, I wanted to learn to explain it simply. 
+
+I will bring in two quotes. 
+
+Einstein: ‘If you can't explain it simply, you don't understand it well enough.’
+
+Aristotle (allegedly): ‘Teaching is the highest form of understanding.’
+
+Maybe it’s something inherent to me, or something I picked up on my 1st job at Lloyds Banking Group - I like to transfer my knowledge onto others. I like to learn in such a way that I ask myself silly questions, run into different errors, edge cases - so that I can teach it to a beginner. It is a bit tricky - thinking about teaching a beginner when I am a beginner myself, but my idea is to question ideas from multiple perspectives and remember what struggles, what questions I asked when learning something - so that after I have learned it I can say it simply to another person. My idea is to help each other and learn together. I respect and value teachers immensely - that is why I look up to people who are experts in their field and also can explain difficult ideas in simple terms for beginners - this (in my opinion) is a skill, and, as Aristotle allegedly said, is the highest form of understanding. One such time when I felt that I understood something was with multicollinearity. Many books/videos say it is bad and just end there. But I started asking a lot of why questions and ended up digging in and finding that eigenvalues/eigenvectors are related and I made a notebook on Kaggle about it - The Full Story behind Multicollinearity. Maybe such a dive is not so valuable per se, but this dive helped me recall and see an actual application of eigenvalues/eigenvectors - math concepts that I had only seen in textbooks up until that point - and it showed me that things go full circle and reinforced my previous belief that if we start to question why and we don’t know the fundamentals - our knowledge ‘falls apart’. As mentioned, it is dangerous trying to teach when still learning, so I joined a lot of online student communities and just observed the type of questions being asked and made sure I can answer them, and not only I tried to answer them (confirming my own knowledge), but also ask myself why is the answer this and not that - in a way questioning myself to make sure I know the answer but also why the answer is the answer. 
+
+---
+
+## My background's effect on my learning journey
+
+I want to write a few sentences of how my background in accounting, banking and data analysis might have impacted my studies and approach to certain topics. In accounting data cannot be wrong. Incorrect data can result in inaccurate numbers in reports, suboptimal financial decision making, incorrect financial statements, deceiving stakeholders, etc. etc. Data has to be correct and that is why maybe I got interested more in data when it comes to AI. Yes there are many architectures but to make a good model we need good data (data quality is crucial). Also, I come from banking, and in both accounting and banking - regulations are king. Documentation (processes, decisions, data, code) is part of both data engineering and MLOps. Not only that but also fairness, explainability - crucial, as we need to explain in front of regulators that our data, code, models, whatever follow certain guidelines. So when I was learning about those things in relation to AI, I made the associations to my previous fields of interest/work. 
+
+---
+
+## Data is King
+
+Another fundamental of AI is data. I think data as a fundamental of AI is not that popular like math and stats, but it is. ‘Garbage in, garbage out’ as the saying goes. That is why I was naturally drawn towards data and data engineering - good data will result in good models, and bad data in bad models. Besides data, I saw many times and learned the idea of anyone can make a model, focus on the business case and the business value - I think this led me to be interested in MLOps - understanding business needs, taking models to production, distributed computing (which is a recurring theme), maintaining them and creating a loop along with data. Before getting into MLOps I was just a notebook guy. Sometimes I wondered what happens after a model is developed in a notebook - well I realised there is a whole different world out there - the world of MLOps - and for the books/courses that taught me - you can find them in my list. Also a skill that’s common not just for these two but in general - communication. I learned you are not a good data engineer if you cannot talk with your stakeholders and understand their needs, their perspectives. When building a model - it’s not so much the model, it’s the business need we are trying to cover and value we are trying to bring, and to understand this value we need to communicate with stakeholders and translate those values into models and architectures to deliver those models and also the data for these models. In DE, there are also repeatable (analytical/design) patterns that happen, data structures that appear again and again - I wanted to focus on these recurring themes as they can be applied to any tool and project. 
+
+---
+
+## Surrounding context
+
+Next, I want to talk about my surrounding context - I am a Master’s student, gave myself plenty of time, studied stats, math, data, AI alongside my university degree in Management Information Systems. I believe I had the privilege of time and the ability to not concentrate on specific tech specifically because of my situation - being a student and having time. While I value learning the fundamentals and the ideas behind tools, I can understand if someone who has a job is trying to learn and they don’t want to waste time going through different books/courses and they want to learn a specific tool so that they can apply to a specific job. As a student, I believe I have the privilege of concentrating on the fundamentals so that when the time comes I will translate my knowledge onto the specific tool that will be used on a job. And to be honest, I learned plenty of the popular tools, and saw the options that different tools (orchestrators) offer and the ideas are very similar, just the implementation is a bit different. 
+
+---
+
+## Free University Lectures  
+
+Next, I’d like to mention a bit about university lectures on math/stats/data/AI. It is amazing how many of the world’s biggest universities share their lectures online - there is *so much* free education out there, people just need to commit to learning it. I will specifically mention Stanford University - they have so many full lecture courses + homeworks for free online - anyone can find them, study the materials and even do the homeworks. I think the youtube algorithm was great to me - once I started watching one free lecture then on my home tab many more lectures and courses appeared. To be honest, that was the case with LinkedIn’s, X’s and reddit’s algorithm - once you are in the sphere the posts and content keeps coming in and that’s how I found many good courses and many good books recommended by known people (professors/professionals) in the industry all the time. Again, check my list for all the books/courses that I covered and my personal top. 
+
+---
+
+## Tools I Used extensively 
+
+Okay ~ Many might be interested in the tools I used. I did not want to write this final post without at all mentioning specific tools I learned/used extensively along the way. I tried to group similar ones but it is not perfect as many tools are used for multiple purposes. Here are the tools I feel confident using at the moment, and also feel confident having a conversation about them and talking about them to beginners.
+
+* Scikit-learn, TensorFlow (stopped using after learning PyTorch), PyTorch, torch-geometric, HuggingFace, Unsloth, Ollama, LangChain
+* dbt, Mage, Prefect, Apache Kafka, Apache Flink, Apache Iceberg, Apache Spark, chispa
+* MLflow, EvidentlyAI, FastAPI, Looker, Grafana
+* Docker (+ some K8s), Terraform, Google Cloud Platform: GCS, BigQuery, Compute Engine
+* Neo4j, PostgreSQL, sqlite
+
+---
+
+## Blog Tags and Metrics  
+
+I have tags on each blog. Tags like NLP, theory, GNN - just a base guide for what I did that day. Here is are 2 pie charts about 1)theory vs practice, and 2) learning breakdown by topic
+
+![image](https://github.com/user-attachments/assets/9c468985-73bb-4808-bf5a-e9aaf48c0dd8)
+
+![image](https://github.com/user-attachments/assets/54f80081-eb2c-45be-9e7e-395c20351870)
+
+Here is a more detailed description of the tags:
+
+* Practice: doing exercises, or working on projects/notebooks
+* Theory: consuming theoretical content (books, papers, lectures, videos)
+* Traditional ML: means non deep learning modes like linear models, trees, ensembles of models, clustering models, dimensionality reduction, etc; this tag is accompanied by a theory and/or practice tag
+* Conv NN: anything related to vision models - starting from basic theory, to learning about CNN architectures like LeNet, AlexNet, VGG Net, ResNet, Transformer-based; image generation with GANs, RNN-based architectures; object detection and localization; this tag is accompanied by a theory and/or practice tag
+* NLP: anything related to text - starting from basic theory to learning about word2vec, GloVe, RNN-based, Attention, BERT, Transformer-based; also RAG apps; chat models; LoRA tuning, prompt engineering; this tag is accompanied by a theory and/or practice tag
+* Graph NN: anything related to graph neural networks - starting from basic theory to node2vec, graph conv. networks, graph attention networks, graphormer; using torch-geometric; this tag is accompanied by a theory and/or practice tag
+* Audio & Speech AI - this was mostly related to working on the Glaswegian voice assistant project
+* MLOps: usually accompanied by theory and/or practice tag; starting from theoretical knowledge on the need for MLOps, a model’s life from data engineering, model engineering, evaluation, experiment tracking, orchestration, deployment, monitoring, maintenance - read material and/or did practice on the whole AI lifecycle from birth to development to production, also using cloud tools, containerisation 
+* Math for ML: linear algebra, matrix decomposition, calculus; this tag is accompanied by theory (covering videos/lectures/ppt slides) and/or practice - examples are solving problems, doing the math for neural network’s epoch by hand, or covering backpropagation
+* Data Eng: usually accompanied by theory and/or practice tag; starting from beginner data eng theoretical knowledge (i.e. Joe Reis’ book) and the need for DE to building pipelines, learning about databases, batch, streaming, table formats, pipelines, data quality, data modelling, using cloud tools, containerisation
+
+---
+
+## Projects/cool things I worked on:
+
+* MLOps Architecture for Real-Time Fraud Detection - https://github.com/divakaivan/kb_project 
+* Transaction Stream Data Engineering Pipeline - https://github.com/divakaivan/transaction-stream-data-pipeline
+* Your Personal Finance Voice Assistant - https://github.com/divakaivan/personal_finance_assistant
+* Glaswegian Audio Dataset and ASR model - https://huggingface.co/datasets/divakaivan/glaswegian_audio
+* MLOps Architecture for Insurance Fraud Detection - https://github.com/divakaivan/insurance-fraud-mlops-pipeline
+* Diving into the full story behind multicollinearity - https://www.kaggle.com/code/divakaivan12/the-full-story-behind-multicollinearity
+* Doing a neural network epoch with pen and paper - https://www.kaggle.com/code/divakaivan12/neural-network-epoch-by-hand
+* PDF RAG from scratch - https://github.com/divakaivan/pdf-rag-from-scratch
+* Using video generation models for predicting taxi demand matrices - https://github.com/divakaivan/taxi-demand-video-models-paper
+* Youtube channel with 100 streams + videos + 150 subs: https://www.youtube.com/@ilearnthings123 
+* Became a Notebook Expert on Kaggle: https://www.kaggle.com/divakaivan12
+* Full list on my blog’s Project tab: https://ivanstudyblog.github.io/projects/ 
+
+I wanted to make 1 final big project (or work on improving the real-time transaction fraud detection project) that encapsulates a lot of what I learned but given that I got a free spot in Zach Wilson’s Jan bootcamp, and  got as a present a spot at Maria Vechtomova’s MLOps course I decided to postpone it a little to gain even more knowledge.
+
+---
+
+## Going Forward
+
+Going forward, the studying doesn’t end. Just the blogging concludes. From Jan 6th I have Zach Wilson’s Data Eng bootcamp, and from Jan 27th I have Maria Vechtomova and Basak Eskili’s MLOps courses. For the past few months while studying for uni and this blog, I have also been applying for jobs in the UK but I haven't gotten to any interview stage yet.. Even though I have unlimited right to stay and work in the UK, it is a bit harder as I am applying from abroad but now I can dedicate even more time to applying for jobs, doing pre-recorded interviews, tests, and hopefully I get an invite to an interview where I might be able to even talk about this blog/study journey. Now that I do not have to cover some new source or different every day I can invest more time into re-reading/re-watching content that I wanted to re-watch. I kind of did that some days but I hardly ever talked about it on the blog as I the ‘annoying/evil Ivan’ whom I had on one of my shoulders (as mentioned earlier 😀) kept telling me that writing ‘today I re-read XYZ is not good blog content and that I shouldn’t re-summarise old content’. This might be seen as a bit weird but that’s just how I wanted to run this blog - even if two books, for example, cover the same topic - I will read and summarise the two books rather than read the book and summarise it on day X, and then re-summarise it on some day X+N as I felt that would be kind of cheating 😀but also it gave me the opportunity to see what common patterns appear between books, what is considered important across the board and also be able to say for myself - ohh this book is better than that book because of abc. In addition, this method kept me accountable that I studied on a given day and also a consistent record if *what* I studied and I can use these posts for reference when needed because I covered a *wide* range of topics. What’s more, every post’s history can be seen on github and the commit dates (which is one of the main reasons I moved away from blogger(https://50daysml.blogspot.com/) to a github docs-based blog) so this can be proof that I posted every day. Anyway ~ I can now re-read, re-watch, and study some of my favourite content to prepare for interviews going forward :) 
+
+Also I think it is a good idea to stop now as the blog spans over a full year - 365 days worth of posts talking about different topics, and also when I start Zach Wilson’s and Maria Vechtemova’s DE and MLOps camps, respectively, I cannot share anything about them on public spaces so these posts wouldn’t be up to the standard I tried to follow during those 365 days. 
+
+Thank you.
+
+---
+
+And for the last time:
+
+That is all for today, and this blog series!
+
+P.S. I studied today and it felt weird as I kept thinking how to write/explain that thing in the blog however I should not ~ there will be some 'transition back to normal' period I guess 😄 

From 0d4c9aed5610f3d8a03d4961e554e693d8754096 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 1 Jan 2025 18:59:34 +0900
Subject: [PATCH 574/638] fix 366 formatting

---
 _posts/2025-01-01-day-366.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/_posts/2025-01-01-day-366.md b/_posts/2025-01-01-day-366.md
index 11ac4737c..fb3fc277b 100644
--- a/_posts/2025-01-01-day-366.md
+++ b/_posts/2025-01-01-day-366.md
@@ -4,8 +4,7 @@ title: (The Final Day 366) This blog ends, but my learning journey continues
 categories: [the-end,thank-you]
 ---
 `
-# Hello :) Today is Day 366! The last day of this blog!
-
+## Hello :) Today is Day 366! The last day of this blog!
 On January 2nd, 2024, I started with zero AI knowledge and self-studied it for 365 days.  
 
 ---

From d5f7bad6b2df4a3ef0d6f8cd3befb3e53ade6f47 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 1 Jan 2025 23:40:50 +0900
Subject: [PATCH 575/638] remove search and about temp

---
 _layouts/default.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index c741dc5a8..642021a64 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -38,8 +38,8 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
               <a href="{{ site.baseurl }}/projects">Projects</a>
-              <a href="{{ site.baseurl }}/search">Search</a>
-              <a href="{{ site.baseurl }}/about">About</a>
+<!--               <a href="{{ site.baseurl }}/search">Search</a> -->
+<!--               <a href="{{ site.baseurl }}/about">About</a> -->
             </nav>
           </header>
         </div>

From b1699343d43bd11be89cde9be8fadee1d15dfd61 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 4 Jan 2025 10:59:30 +0900
Subject: [PATCH 576/638] Update about.md

---
 _pages/about.md | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/_pages/about.md b/_pages/about.md
index 19a1b9366..aa7e0f806 100644
--- a/_pages/about.md
+++ b/_pages/about.md
@@ -6,12 +6,20 @@ permalink: /about/
 
 Hello!
 
-On the 2nd of January 2024, I embarked on a, what I thought would be, 50 day journey to learn about Machine Learning. It is way past the 50 day mark, and I keep learning and blogging. 
+On the 2nd of January 2024, I embarked on a, what I thought would be, 50 day journey to learn about Machine Learning. It is way past the 50 day mark, and I keep learning and blogging. On Day 100 I knew it was not over, so I continued. After a few days I decided to go for the full year.
 
-On Day 100 I knew it was not over, so I continued.
+For a full summary, check my [Day 366 post](https://ivanstudyblog.github.io/day-366).
 
-After a few days I decided to go for the full year.
+I spent my time learning about these topics:
 
-My studies have covered a range of topics, including mathematics, statistics, data engineering, classical machine learning, modern deep learning architectures, AI assistants, and MLOps systems. This journey has not only been about mastering the technical aspects but also about adopting the mindset of a Data Engineer, Data Scientist, Machine Learning Engineer, and Data Analyst, while developing the soft skills essential for these roles. Additionally, it has been a process of learning how I learn—and understanding how others learn—which I plan to summarize at the end of this 1-year journey.
+![462581705_2513826982154624_4414551878965446179_n](https://github.com/user-attachments/assets/d4b79fad-5d46-47b3-af59-813493d06d46)
 
-**These days I stream part of my daily studies on [my youtube channel](https://www.youtube.com/@ilearnthings123) - Monday to Thursday**
+I spent my time learning theory and practice:
+
+![464677107_27928007950179000_5488089435055286095_n](https://github.com/user-attachments/assets/2e43d8e5-961a-4baf-a854-7741b797512a)
+
+Here is a word cloud of most common words in my blog:
+
+![wordcloud](https://github.com/user-attachments/assets/6ec47761-425b-417c-aeee-3c7184aaef2e)
+
+You might find me streaming on [my youtube channel](https://www.youtube.com/@ilearnthings123).

From 00eec13b803292625a0032c8734bb04bd4a9f702 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 4 Jan 2025 10:59:48 +0900
Subject: [PATCH 577/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 642021a64..ec2e8934b 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -39,7 +39,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
               <a href="{{ site.baseurl }}/">Home</a>
               <a href="{{ site.baseurl }}/projects">Projects</a>
 <!--               <a href="{{ site.baseurl }}/search">Search</a> -->
-<!--               <a href="{{ site.baseurl }}/about">About</a> -->
+              <a href="{{ site.baseurl }}/about">About</a>
             </nav>
           </header>
         </div>

From 5ece64006cc07bf0cee9a5b1ea4e09118eb1d46f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:10:11 +0900
Subject: [PATCH 578/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index ec2e8934b..47b275338 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -38,7 +38,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
               <a href="{{ site.baseurl }}/projects">Projects</a>
-<!--               <a href="{{ site.baseurl }}/search">Search</a> -->
+<!--               <a href="{{ site.baseurl }}/2025">2025</a> -->
               <a href="{{ site.baseurl }}/about">About</a>
             </nav>
           </header>

From a96a4cfd2ba6285b702836c8deaa44f126db8b9b Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:10:32 +0900
Subject: [PATCH 579/638] Update search.md

---
 _pages/search.md | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index b89ec6a9e..e71f7a3d9 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -1,21 +1,7 @@
 ---
 layout: page
 title: Search
-permalink: /search/
+permalink: /2025/
 ---
 
-<div id="search-container">
-    <input type="text" id="search-input" placeholder="Search through the blog posts...">
-    <ul id="results-container"></ul>
-</div>
-
-<script src="{{ site.baseurl }}/assets/simple-jekyll-search.min.js" type="text/javascript"></script>
-
-<script>
-    SimpleJekyllSearch({
-    searchInput: document.getElementById('search-input'),
-    resultsContainer: document.getElementById('results-container'),
-    searchResultTemplate: '<div style="text-align: left !important;"><a href="{url}"><h1 style="text-align:left !important;">{title}</h1></a><span style="text-align:left !important;">{date}</span></div>',
-    json: '{{ site.baseurl }}/search.json'
-    });
-</script>
\ No newline at end of file
+Test

From 4c4a44a5a45458697013f4903dcfe3d7f7940963 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:12:54 +0900
Subject: [PATCH 580/638] Update default.html

---
 _layouts/default.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 47b275338..9f566b598 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -38,7 +38,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
             <nav>
               <a href="{{ site.baseurl }}/">Home</a>
               <a href="{{ site.baseurl }}/projects">Projects</a>
-<!--               <a href="{{ site.baseurl }}/2025">2025</a> -->
+              <a href="{{ site.baseurl }}/2025">2025</a>
               <a href="{{ site.baseurl }}/about">About</a>
             </nav>
           </header>

From dc297e07e3083549b3c0bbe9964c53c906d70756 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:18:15 +0900
Subject: [PATCH 581/638] Update _config.yml

---
 _config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_config.yml b/_config.yml
index d9087f17a..1dcc46e93 100644
--- a/_config.yml
+++ b/_config.yml
@@ -3,13 +3,13 @@
 #
 
 # Name of your site (displayed in the header)
-name: Study and Blog
+name: 🔥2024 Learn + Blog EVERY DAY for 366 days!💻
 
 # Name of the author
 author: Ivan Ivanov
 
 # Short bio or description (displayed in the header)
-description: Self-study and blog every day
+description: Math, Stats, AI, Data Eng, MLOps!
 
 # URL of your avatar or profile pic (you could use your GitHub profile pic)
 avatar: /avatar.jpeg

From b0e36f9279642f482a2dc5883575e8092d1482cf Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:21:15 +0900
Subject: [PATCH 582/638] Update _config.yml

---
 _config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_config.yml b/_config.yml
index 1dcc46e93..ade93072f 100644
--- a/_config.yml
+++ b/_config.yml
@@ -3,7 +3,7 @@
 #
 
 # Name of your site (displayed in the header)
-name: 🔥2024 Learn + Blog EVERY DAY for 366 days!💻
+name: Learn + Blog every day in 2024
 
 # Name of the author
 author: Ivan Ivanov
@@ -29,7 +29,7 @@ footer-links:
   linkedin: ivansivanov
   pinterest:
   rss: 
-  twitter: hi_iam_ivan
+  twitter: 
   stackoverflow: 
   youtube: channel/UC-nprSI5aP4O7REgzaSUXFg
   googleplus: # anything in your profile username that comes after plus.google.com/

From bb54ae44460aa3cf13c6c8a282ded22563f0c6ee Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:22:25 +0900
Subject: [PATCH 583/638] Update default.html

---
 _layouts/default.html | 2 --
 1 file changed, 2 deletions(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index 9f566b598..dd6545433 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -47,8 +47,6 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
 
       <div class="wrapper-main">
         <div id="main" role="main" class="container">
-          <h1>🔥2024 Learn + Blog EVERY DAY for 366 days!💻</h1>
-          <h1>Math, Stats, AI, Data Eng, MLOps!</h1>
           {{ content }}
         </div>
       </div>

From 8eeda9a1bd5f2ef77c1b5d7d149ca3ccbe7b1c54 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:47:18 +0900
Subject: [PATCH 584/638] Update search.md

---
 _pages/search.md | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index e71f7a3d9..500a2cb84 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -1,7 +1,21 @@
 ---
 layout: page
-title: Search
+title: Casual 2025 learning blog
 permalink: /2025/
 ---
 
-Test
+### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
+
+**Boot camps**
+
+* Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
+* Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
+
+**Personal studies**
+
+* covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
+* found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
+* did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
+* read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
+* doing practice questions on [Data Lemur](https://datalemur.com/)
+* read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)

From ae70979ed249b4aa19638315bc34333986bc7156 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 19:54:03 +0900
Subject: [PATCH 585/638] Update search.md

---
 _pages/search.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index 500a2cb84..56bcb88d6 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -17,5 +17,5 @@ permalink: /2025/
 * found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
 * did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
 * read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
-* doing practice questions on [Data Lemur](https://datalemur.com/)
+* doing daily practice questions on [Data Lemur](https://datalemur.com/)
 * read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)

From e8f6efb01eb1fadb86b6c80ffdd856047ddd6b1b Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 15 Jan 2025 22:26:56 +0900
Subject: [PATCH 586/638] Update search.md

---
 _pages/search.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_pages/search.md b/_pages/search.md
index 56bcb88d6..e817e6924 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -13,6 +13,7 @@ permalink: /2025/
 
 **Personal studies**
 
+* finished all SQL questions in Data Lemur (also ML and Stats) and started doing leetcode's DSA course
 * covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
 * found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
 * did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)

From 37742ff5d1805752da9d725139be920d7b109c4d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 15 Jan 2025 22:29:54 +0900
Subject: [PATCH 587/638] Update search.md

---
 _pages/search.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index e817e6924..e51e61cf9 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -13,10 +13,11 @@ permalink: /2025/
 
 **Personal studies**
 
-* finished all SQL questions in Data Lemur (also ML and Stats) and started doing leetcode's DSA course
+* started doing [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/)
+* finished all SQL questions in Data Lemur (also ML and Stats)
+* re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
 * covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
 * found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
 * did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
 * read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
-* doing daily practice questions on [Data Lemur](https://datalemur.com/)
 * read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)

From 33a6e96e8f3f1967b5ed8425dd2a539a9ecfa0c2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 17 Jan 2025 18:38:48 +0900
Subject: [PATCH 588/638] Update search.md

---
 _pages/search.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index e51e61cf9..5ffe9b23f 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -11,9 +11,10 @@ permalink: /2025/
 * Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
 * Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
 
-**Personal studies**
+**Other studies**
 
-* started doing [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/)
+* (ongoing) reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
+* (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/)
 * finished all SQL questions in Data Lemur (also ML and Stats)
 * re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
 * covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)

From b9adb4be7a205d7ccbe32db0bd6070cfda113944 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 22 Jan 2025 20:10:50 +0900
Subject: [PATCH 589/638] Update search.md

---
 _pages/search.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index 5ffe9b23f..89cb3a321 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -13,8 +13,10 @@ permalink: /2025/
 
 **Other studies**
 
-* (ongoing) reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
 * (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/)
+* (ongoing) reading Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
+* (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
+* reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
 * finished all SQL questions in Data Lemur (also ML and Stats)
 * re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
 * covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)

From 9a18356fe812bbba91f3a98a270e9f2274e1d4d3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 31 Jan 2025 19:56:29 +0900
Subject: [PATCH 590/638] Update search.md

---
 _pages/search.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 89cb3a321..16278681b 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -8,14 +8,14 @@ permalink: /2025/
 
 **Boot camps**
 
-* Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
-* Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
+* (ongoing) Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
+* (ongoing) Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
 
 **Other studies**
 
-* (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/)
-* (ongoing) reading Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
+* (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
+* read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
 * reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
 * finished all SQL questions in Data Lemur (also ML and Stats)
 * re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)

From d75950e8ff35d56781289b4ddd4bc3bb29aa2a5e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 1 Feb 2025 22:22:50 +0900
Subject: [PATCH 591/638] Update search.md

---
 _pages/search.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_pages/search.md b/_pages/search.md
index 16278681b..e41ce5215 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -15,6 +15,8 @@ permalink: /2025/
 
 * (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
+* (ongoing) covering [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
+* read [Refactoring Guru's design patterns](https://refactoring.guru/)
 * read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
 * reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
 * finished all SQL questions in Data Lemur (also ML and Stats)

From 9d3589a7de4c902dd9cce2bd6aeed8b13d9ccc9a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 2 Feb 2025 14:02:48 +0900
Subject: [PATCH 592/638] Update search.md

---
 _pages/search.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_pages/search.md b/_pages/search.md
index e41ce5215..e0af7ea40 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -16,6 +16,7 @@ permalink: /2025/
 * (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
 * (ongoing) covering [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
+* read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin
 * read [Refactoring Guru's design patterns](https://refactoring.guru/)
 * read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
 * reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)

From 587035640ef03869c961c770c4887251ff64b485 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 21 Feb 2025 14:53:50 +0900
Subject: [PATCH 593/638] Update search.md

---
 _pages/search.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index e0af7ea40..0e27da7d4 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -1,4 +1,4 @@
----
+![image](https://github.com/user-attachments/assets/ed4f2479-c8db-4ea7-a7c1-c33478801948)---
 layout: page
 title: Casual 2025 learning blog
 permalink: /2025/
@@ -6,7 +6,7 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-**Boot camps**
+**Bootcamps**
 
 * (ongoing) Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
 * (ongoing) Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
@@ -16,6 +16,9 @@ permalink: /2025/
 * (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
 * (ongoing) covering [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
+* [Finished an example project for a MLOps 101 mini-course I will teach to my university club](https://github.com/divakaivan/mlops-101)
+* [Finished a full-stack voice-to-voice app that lets users interact with their finances through an AI agent](https://github.com/divakaivan/voice2voice-banking-assistant)
+* [Finished my capstone for Zach Wilson's bootcamp - Using voice data to extract insight from team communication](https://github.com/divakaivan/lolesports-voice-analytics)
 * read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin
 * read [Refactoring Guru's design patterns](https://refactoring.guru/)
 * read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin

From 33114154df7aab9144c2e7908966aa804d987e5a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 21 Feb 2025 14:54:02 +0900
Subject: [PATCH 594/638] Update search.md

---
 _pages/search.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index 0e27da7d4..0f82b0564 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -1,4 +1,4 @@
-![image](https://github.com/user-attachments/assets/ed4f2479-c8db-4ea7-a7c1-c33478801948)---
+---
 layout: page
 title: Casual 2025 learning blog
 permalink: /2025/

From 30d93f9fe8524cc356e8f7f8b8069a3b14e5607d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 21 Feb 2025 15:16:12 +0900
Subject: [PATCH 595/638] Update projects.md

---
 _pages/projects.md | 46 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 13 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index d5a65fe0c..0df986a4b 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -5,6 +5,26 @@ title: Projects
 ---
 
 <div id="projects">
+  <div class="project-item">
+    <h3>MLOps 101 Project for a mini-course I teach</h3>
+    <p>
+      <li>After learning a tonne from great online teachers, and projects I decided to transfer my knowledge onto undergraduate students who are curious about the life of a model outside the Jupyter notebook</li>
+      <li>An end-to-end ML system that processes taxi data, stores models in a model registry, exposes them via an API, and deploys this API to Google Cloud, and keeps logs for observability</li>
+    </p>
+    <p><b>Tech: scikit-learn, EvidentlyAI, FastAPI, MLFlow, Docker, Github Actions, Terraform, Google Cloud (GCS, Logging, Compute Engine, Artifact Registry, Kubernetes Engine)</b></p>
+    <a href="https://github.com/divakaivan/mlops-101" target="_blank">View Project</a>
+  </div>
+
+  <div class="project-item">
+    <h3>Esports Voice Data Pipeline (Zach Wilson's DE bootcamp capstone)</h3>
+    <p>
+      <li>Esports team communication data is normally kept private, but for the first time a team is sharing their full voice communication records so with this I am showing a prototype of a pipeline that utilises audio data to extract communication patterns</li>
+      <li>In addition, I developed visualizations that uncover communication patterns and dynamics, providing the underlying team with actionable insights to enhance their gameplay</li>
+    </p>
+    <p><b>Tech: Airflow, dbt, Google BigQuery, Google Cloud Storage, Streamlit, Terraform, Github Actions, Astronomer</b></p>
+    <a href="https://github.com/divakaivan/lolesports-voice-analytics" target="_blank">View Project</a>
+  </div>
+  
   <div class="project-item">
     <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
     <p>
@@ -13,7 +33,19 @@ title: Projects
       <li>High fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models</li>
     </p>
     <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
-    <a href="https://github.com/divakaivan/kb_project" target="_blank">View on GitHub</a>
+    <a href="https://github.com/divakaivan/kb_project" target="_blank">View Project</a>
+  </div>
+
+  <div class="project-item">
+    <h3>Voice-to-Voice Personal Finance Assistant</h3>
+    <p>
+      <li>Talk, Learn and Analyse your spending habits with your Personal Finance Assistant AI Agent. Communicate through speech</li>
+      <li>Frontend + Backend communicating via a websocket</li>
+      <li>Ask follow-up questions (the language model can see the chat history)</li>
+      <li>Detailed observability of live and historical connections to the server via Pydantic Logfire</li>
+    </p>
+    <p><b>Tech: Webhooks, FastAPI, PydanticAI, Logfire, SQLite, OpenAI, Ollama, PostgreSQL, React</b></p>
+    <a href="https://github.com/divakaivan/voice2voice-banking-assistant" target="_blank">View on GitHub</a>
   </div>
 
   <div class="project-item">
@@ -28,18 +60,6 @@ title: Projects
     <p><b>Tech: PostgreSQL DB, Kafka, PySpark, dbt, Grafana</b></p>
     <a href="https://github.com/divakaivan/transaction-stream-data-pipeline" target="_blank">View Project</a>
   </div>
-  
-  <div class="project-item">
-    <h3>Your Personal Finance Voice Assistant</h3>
-    <p>
-      <li>Chat, Talk, Learn and Analyse your spending habits with your Personal Finance Assistant</li>
-      <li>Communicate through text or speech</li>
-      <li>Ask follow-up questions (the language model can see the chat history)</li>
-      <li>On the dev side, see how the RAG is performing by analysing prompts and retrieved information</li>
-    </p>
-    <p><b>Tech: SQLite, OpenAI, HuggingFace, LlamaIndex, Azire Pheonix monitoring, Streamlit</b></p>
-    <a href="https://github.com/divakaivan/personal_finance_assistant" target="_blank">View on GitHub</a>
-  </div>
 
   <div class="project-item">
     <h3>Glaswegian Audio Dataset and ASR model</h3>

From b42fd7b83b2d5a46a71a16f096e40a3966c9ed79 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 28 Feb 2025 10:54:12 +0900
Subject: [PATCH 596/638] Update search.md

---
 _pages/search.md | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 0f82b0564..9bcf9c34e 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -13,20 +13,24 @@ permalink: /2025/
 
 **Other studies**
 
+* (ongoing) covering the study materials for [Certified Kubernetes Applications Developer](https://www.udemy.com/course/certified-kubernetes-application-developer/) (CKAD)
+* (ongoing) reading through [ByteByteGo's ML System Design book](https://bytebytego.com/)
+* (ongoing) covering [Stanford's CS149 - Parallel Computing](https://www.youtube.com/playlist?list=PLoROMvodv4rMp7MTFr4hQsDEcX7Bx6Odp)
 * (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
-* (ongoing) covering [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
-* [Finished an example project for a MLOps 101 mini-course I will teach to my university club](https://github.com/divakaivan/mlops-101)
-* [Finished a full-stack voice-to-voice app that lets users interact with their finances through an AI agent](https://github.com/divakaivan/voice2voice-banking-assistant)
-* [Finished my capstone for Zach Wilson's bootcamp - Using voice data to extract insight from team communication](https://github.com/divakaivan/lolesports-voice-analytics)
-* read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin
-* read [Refactoring Guru's design patterns](https://refactoring.guru/)
-* read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
-* reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
-* finished all SQL questions in Data Lemur (also ML and Stats)
-* re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
-* covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
+* (Feb) read [ML for Tabular data](https://www.manning.com/books/machine-learning-for-tabular-data) by Mark Ryan and Luca Massaron
+* (Feb) covered [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
+* (Feb) [Finished an example project for a MLOps 101 mini-course I will teach to my university club](https://github.com/divakaivan/mlops-101)
+* (Feb) [Finished a full-stack voice-to-voice app that lets users interact with their finances through an AI agent](https://github.com/divakaivan/voice2voice-banking-assistant)
+* (Feb) [Finished my capstone for Zach Wilson's bootcamp - Using voice data to extract insight from team communication](https://github.com/divakaivan/lolesports-voice-analytics)
+* (Jan) read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin
+* (Jan) read [Refactoring Guru's design patterns](https://refactoring.guru/)
+* (Jan) read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
+* (Jan) reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
+* (Jan) finished all SQL questions in Data Lemur (also ML and Stats)
+* (Jan) re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
+* (Jan) covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
 * found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
-* did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
-* read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
-* read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)
+* (Jan) did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
+* (Jan) read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
+* (Jan) read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)

From 510bcf186899050a2788a2a03be76d4d9f63402f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 5 Mar 2025 13:44:20 +0900
Subject: [PATCH 597/638] Update search.md

---
 _pages/search.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 9bcf9c34e..f8dc75f77 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,18 +6,21 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
+*Updated: 5th March 2025*
+
 **Bootcamps**
 
 * (ongoing) Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
-* (ongoing) Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
+* (finished) Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
 
 **Other studies**
 
+* (ongoing) reading Alex Xu's System Design Interview (Vol. 1)
 * (ongoing) covering the study materials for [Certified Kubernetes Applications Developer](https://www.udemy.com/course/certified-kubernetes-application-developer/) (CKAD)
 * (ongoing) reading through [ByteByteGo's ML System Design book](https://bytebytego.com/)
-* (ongoing) covering [Stanford's CS149 - Parallel Computing](https://www.youtube.com/playlist?list=PLoROMvodv4rMp7MTFr4hQsDEcX7Bx6Odp)
 * (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
+* (Mar) covering [Stanford's CS149 - Parallel Computing](https://www.youtube.com/playlist?list=PLoROMvodv4rMp7MTFr4hQsDEcX7Bx6Odp)
 * (Feb) read [ML for Tabular data](https://www.manning.com/books/machine-learning-for-tabular-data) by Mark Ryan and Luca Massaron
 * (Feb) covered [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
 * (Feb) [Finished an example project for a MLOps 101 mini-course I will teach to my university club](https://github.com/divakaivan/mlops-101)

From 7389b991469e19f70b33e25d84809825513a3aa1 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 12 Mar 2025 02:03:26 +0900
Subject: [PATCH 598/638] Create feed.xml

---
 _layouts/feed.xml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 _layouts/feed.xml

diff --git a/_layouts/feed.xml b/_layouts/feed.xml
new file mode 100644
index 000000000..3a75bb7de
--- /dev/null
+++ b/_layouts/feed.xml
@@ -0,0 +1,30 @@
+---
+layout: nil
+permalink: /feed.xml
+---
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>{{ site.title | xml_escape }}</title>
+  <link href="{{ site.url }}/feed.xml" rel="self" type="application/atom+xml"/>
+  <link href="{{ site.url }}" rel="alternate" type="text/html"/>
+  <updated>{{ site.time | date_to_rfc3339 }}</updated>
+  <id>{{ site.url }}/feed.xml</id>
+  <author>
+    <name>{{ site.author }}</name>
+  </author>
+
+  {% for post in paginator.posts %}
+  <entry>
+    <title>{{ post.title | xml_escape }}</title>
+    <link href="{{ site.url }}{{ post.url }}" rel="alternate" type="text/html"/>
+    <id>{{ site.url }}{{ post.url }}</id>
+    <published>{{ post.date | date_to_rfc3339 }}</published>
+    <updated>{{ post.date | date_to_rfc3339 }}</updated>
+    <content type="html">{{ post.content | xml_escape }}</content>
+  </entry>
+  {% endfor %}
+
+  {% if paginator.next_page %}
+  <link rel="next" href="{{ site.url }}/feed{{ paginator.next_page }}.xml"/>
+  {% endif %}
+</feed>

From c50f1054e33e1cdf302677f580663afcb00de2f8 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 12 Mar 2025 02:03:49 +0900
Subject: [PATCH 599/638] Create feed_page.xml

---
 _layouts/feed_page.xml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 _layouts/feed_page.xml

diff --git a/_layouts/feed_page.xml b/_layouts/feed_page.xml
new file mode 100644
index 000000000..0b4645af9
--- /dev/null
+++ b/_layouts/feed_page.xml
@@ -0,0 +1,30 @@
+---
+layout: nil
+permalink: /feed{{ page.number }}.xml
+---
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>{{ site.title | xml_escape }}</title>
+  <link href="{{ site.url }}/feed{{ page.number }}.xml" rel="self" type="application/atom+xml"/>
+  <link href="{{ site.url }}" rel="alternate" type="text/html"/>
+  <updated>{{ site.time | date_to_rfc3339 }}</updated>
+  <id>{{ site.url }}/feed{{ page.number }}.xml</id>
+  <author>
+    <name>{{ site.author }}</name>
+  </author>
+
+  {% for post in paginator.posts %}
+  <entry>
+    <title>{{ post.title | xml_escape }}</title>
+    <link href="{{ site.url }}{{ post.url }}" rel="alternate" type="text/html"/>
+    <id>{{ site.url }}{{ post.url }}</id>
+    <published>{{ post.date | date_to_rfc3339 }}</published>
+    <updated>{{ post.date | date_to_rfc3339 }}</updated>
+    <content type="html">{{ post.content | xml_escape }}</content>
+  </entry>
+  {% endfor %}
+
+  {% if paginator.next_page %}
+  <link rel="next" href="{{ site.url }}/feed{{ paginator.next_page }}.xml"/>
+  {% endif %}
+</feed>

From 55852b458f9f4a58c4ed1d84757d95da127df885 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 12 Mar 2025 02:07:19 +0900
Subject: [PATCH 600/638] Update _config.yml

---
 _config.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/_config.yml b/_config.yml
index ade93072f..ac6ef686d 100644
--- a/_config.yml
+++ b/_config.yml
@@ -99,3 +99,9 @@ exclude:
   - LICENSE
   - README.md
   - CNAME
+defaults:
+  - scope:
+      path: ""
+      type: "posts"
+    values:
+      layout: "feed.xml"

From 242ce193a8abf987fe09b6920473011bd8da1102 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 12 Mar 2025 02:12:50 +0900
Subject: [PATCH 601/638] Delete _layouts/feed_page.xml

---
 _layouts/feed_page.xml | 30 ------------------------------
 1 file changed, 30 deletions(-)
 delete mode 100644 _layouts/feed_page.xml

diff --git a/_layouts/feed_page.xml b/_layouts/feed_page.xml
deleted file mode 100644
index 0b4645af9..000000000
--- a/_layouts/feed_page.xml
+++ /dev/null
@@ -1,30 +0,0 @@
----
-layout: nil
-permalink: /feed{{ page.number }}.xml
----
-<?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns="http://www.w3.org/2005/Atom">
-  <title>{{ site.title | xml_escape }}</title>
-  <link href="{{ site.url }}/feed{{ page.number }}.xml" rel="self" type="application/atom+xml"/>
-  <link href="{{ site.url }}" rel="alternate" type="text/html"/>
-  <updated>{{ site.time | date_to_rfc3339 }}</updated>
-  <id>{{ site.url }}/feed{{ page.number }}.xml</id>
-  <author>
-    <name>{{ site.author }}</name>
-  </author>
-
-  {% for post in paginator.posts %}
-  <entry>
-    <title>{{ post.title | xml_escape }}</title>
-    <link href="{{ site.url }}{{ post.url }}" rel="alternate" type="text/html"/>
-    <id>{{ site.url }}{{ post.url }}</id>
-    <published>{{ post.date | date_to_rfc3339 }}</published>
-    <updated>{{ post.date | date_to_rfc3339 }}</updated>
-    <content type="html">{{ post.content | xml_escape }}</content>
-  </entry>
-  {% endfor %}
-
-  {% if paginator.next_page %}
-  <link rel="next" href="{{ site.url }}/feed{{ paginator.next_page }}.xml"/>
-  {% endif %}
-</feed>

From 823efbbe27deb2886ce8eaa245742223e59309a3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 12 Mar 2025 02:13:15 +0900
Subject: [PATCH 602/638] Delete _layouts/feed.xml

---
 _layouts/feed.xml | 30 ------------------------------
 1 file changed, 30 deletions(-)
 delete mode 100644 _layouts/feed.xml

diff --git a/_layouts/feed.xml b/_layouts/feed.xml
deleted file mode 100644
index 3a75bb7de..000000000
--- a/_layouts/feed.xml
+++ /dev/null
@@ -1,30 +0,0 @@
----
-layout: nil
-permalink: /feed.xml
----
-<?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns="http://www.w3.org/2005/Atom">
-  <title>{{ site.title | xml_escape }}</title>
-  <link href="{{ site.url }}/feed.xml" rel="self" type="application/atom+xml"/>
-  <link href="{{ site.url }}" rel="alternate" type="text/html"/>
-  <updated>{{ site.time | date_to_rfc3339 }}</updated>
-  <id>{{ site.url }}/feed.xml</id>
-  <author>
-    <name>{{ site.author }}</name>
-  </author>
-
-  {% for post in paginator.posts %}
-  <entry>
-    <title>{{ post.title | xml_escape }}</title>
-    <link href="{{ site.url }}{{ post.url }}" rel="alternate" type="text/html"/>
-    <id>{{ site.url }}{{ post.url }}</id>
-    <published>{{ post.date | date_to_rfc3339 }}</published>
-    <updated>{{ post.date | date_to_rfc3339 }}</updated>
-    <content type="html">{{ post.content | xml_escape }}</content>
-  </entry>
-  {% endfor %}
-
-  {% if paginator.next_page %}
-  <link rel="next" href="{{ site.url }}/feed{{ paginator.next_page }}.xml"/>
-  {% endif %}
-</feed>

From 0354afad89c34c599977b565ee70c84254a1793f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 30 Mar 2025 18:30:57 +0900
Subject: [PATCH 603/638] Update search.md

---
 _pages/search.md | 75 ++++++++++++++++++++++++++++--------------------
 1 file changed, 44 insertions(+), 31 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index f8dc75f77..beceafca5 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,34 +6,47 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 5th March 2025*
-
-**Bootcamps**
-
-* (ongoing) Jan 27th - Mar 16th - I am doing Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
-* (finished) Jan 6th - Mar 2nd - I am doing Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp)
-
-**Other studies**
-
-* (ongoing) reading Alex Xu's System Design Interview (Vol. 1)
-* (ongoing) covering the study materials for [Certified Kubernetes Applications Developer](https://www.udemy.com/course/certified-kubernetes-application-developer/) (CKAD)
-* (ongoing) reading through [ByteByteGo's ML System Design book](https://bytebytego.com/)
-* (ongoing) [leetcode's DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
-* (ongoing) covering [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
-* (Mar) covering [Stanford's CS149 - Parallel Computing](https://www.youtube.com/playlist?list=PLoROMvodv4rMp7MTFr4hQsDEcX7Bx6Odp)
-* (Feb) read [ML for Tabular data](https://www.manning.com/books/machine-learning-for-tabular-data) by Mark Ryan and Luca Massaron
-* (Feb) covered [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
-* (Feb) [Finished an example project for a MLOps 101 mini-course I will teach to my university club](https://github.com/divakaivan/mlops-101)
-* (Feb) [Finished a full-stack voice-to-voice app that lets users interact with their finances through an AI agent](https://github.com/divakaivan/voice2voice-banking-assistant)
-* (Feb) [Finished my capstone for Zach Wilson's bootcamp - Using voice data to extract insight from team communication](https://github.com/divakaivan/lolesports-voice-analytics)
-* (Jan) read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin
-* (Jan) read [Refactoring Guru's design patterns](https://refactoring.guru/)
-* (Jan) read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
-* (Jan) reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
-* (Jan) finished all SQL questions in Data Lemur (also ML and Stats)
-* (Jan) re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
-* (Jan) covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
-* found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
-* (Jan) did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
-* (Jan) read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
-* (Jan) read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)
+*Updated: 30th March 2025*
+
+## March
+
+* [project] built an API with Go, which is tested, Dockerized, published on Docker Hub, deployed in a local Kubernetes cluster, and monitored with dashboards for Kubernetes and the API's usage - [github](https://github.com/divakaivan/rssagg)
+* [project] setup and learned about Platform Engineering and automation with Backstage - [video](https://www.youtube.com/watch?v=i4cQzjRNz4M) + [linkedin post](https://www.linkedin.com/posts/activity-7308108817068646400-BLJ5)
+* [book] started reading Kafka: The Definitive Guide, 2nd Edition (will finish it in April)
+* [book] learned about language from linguists' point of view by reading Course in general linguistics by Ferdinand de Saussure (1916) and Syntactic Structures by Noam Chomsky (1957) - [linkedin](https://www.linkedin.com/posts/activity-7310163253425250305-PoYN)
+* [book] read Generative AI on Kubernetes by Roland Huss, Daniele Zonca (+ waiting for new chapters)
+* [book] (Feb - Mar) read Alex Xu's System Design Interview (Vol. 1)
+* [book] (Feb - Mar) read [ByteByteGo's ML System Design book](https://bytebytego.com/)
+* [book-ish] read through FastAPI's full documentation (amazing)
+* [book-ish] read through Kubernetes' documentation related to CKAD (very nice examples)
+* [leetcode] (Jan - ongoing) [DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
+* [course] covered the study materials for [Certified Kubernetes Applications Developer](https://www.udemy.com/course/certified-kubernetes-application-developer/) (CKAD)
+* [course] (Jan-Mar) watched some of [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
+* [course] covered [Stanford's CS149 - Parallel Computing](https://www.youtube.com/playlist?list=PLoROMvodv4rMp7MTFr4hQsDEcX7Bx6Odp)
+* [mini-course] built a [mini-http server from scratch](https://github.com/divakaivan/build-http-scratch) & [learned about nginx](https://github.com/divakaivan/nginx-crash-course)
+* [course] covered Eric Riddoch's [Taking Python to Production: A Professional Onboarding Guide](https://www.udemy.com/course/setting-up-the-linux-terminal-for-software-development/)
+* [bootcamp] will be part of Eric Riddoch's [Cloud Engineering for Python Devs](https://mlops-club.org/) bootcamp
+* [bootcamp] (Jan - Mar) completed Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
+* [bootcamp] (Jan - Mar) Completed Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp) - [Combined Certificate of Superbness](https://www.dataexpert.io/certification/divakaivan/combined-superb)
+
+## February
+
+* [book] read [ML for Tabular data](https://www.manning.com/books/machine-learning-for-tabular-data) by Mark Ryan and Luca Massaron
+* [course] covered [MIT's Intro to Algorithms Spring 2020](https://www.youtube.com/playlist?list=PLUl4u3cNGP63EdVPNLG3ToM6LaEUuStEY)
+* [project] [Finished an example project for a MLOps 101 mini-course I will teach to my university club](https://github.com/divakaivan/mlops-101)
+* [project] [Finished a full-stack voice-to-voice app that lets users interact with their finances through an AI agent](https://github.com/divakaivan/voice2voice-banking-assistant)
+* [project] [Finished my capstone for Zach Wilson's bootcamp - Using voice data to extract insight from team communication](https://github.com/divakaivan/lolesports-voice-analytics)
+
+## January
+
+* [book] read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin
+* [book] read [Refactoring Guru's design patterns](https://refactoring.guru/)
+* [book] read Clean Code: A Handbook of Agile Software Craftsmanship by Robert C. Martin
+* [book] reading Chip Huyen's [ML Interviews book](https://huyenchip.com/ml-interviews-book/)
+* [sql-leetcode]finished all SQL questions in Data Lemur (also ML and Stats)
+* [course] re-watched CS50's lectures on [Data Structures, Algorithms, C](https://cs50.harvard.edu/x/2025/)
+* [course] covered a practical [Kubernetes course](https://www.youtube.com/watch?v=d6WC5n9G_sM&pp=ygUZa3ViZXJuZXRlcyBpbnRybyBic3RzY2h1aw%3D%3D) (really good!)
+* [blog] found podcasts episodes with MarvelousMLOps' [Maria](https://youtu.be/Nnq_2uHBUSo) and [Başak](https://youtu.be/2v8Sops3n1A) and read through a lot of [MarvelousMLops' posts](https://marvelousmlops.substack.com/) 
+* [mini-project] did a conceptual, logical (+physical in Neo4j) graph data model for [EU AI Act data](https://www.linkedin.com/posts/activity-7282017446650159104-gYlO?utm_source=share&utm_medium=member_desktop)
+* [book] read [Machine Learning Engineering with Python - Second Edition](https://learning.oreilly.com/library/view/machine-learning-engineering/9781837631964/)
+* [book] read through [Software Engineering for Data Scientists](https://learning.oreilly.com/library/view/software-engineering-for/9781098136192/)

From 75534b0529ee211a6fee499c1eac625edd0a8c19 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 31 Mar 2025 13:06:58 +0900
Subject: [PATCH 604/638] Update search.md

---
 _pages/search.md | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index beceafca5..11239406a 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -1,12 +1,19 @@
 ---
 layout: page
-title: Casual 2025 learning blog
+title: Monthly 2025 learning blog
 permalink: /2025/
 ---
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 30th March 2025*
+*Updated: 31st March 2025*
+
+## April *(currently)*
+
+* [book] Write an interpreter in Go by [Throsten Ball](https://interpreterbook.com/)
+* [leetcode] [DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode (Jan - ongoing)
+* [bootcamp] will be part of Eric Riddoch's [Cloud Engineering for Python Devs](https://mlops-club.org/) bootcamp (Apr - May)
+
 
 ## March
 
@@ -19,16 +26,15 @@ permalink: /2025/
 * [book] (Feb - Mar) read [ByteByteGo's ML System Design book](https://bytebytego.com/)
 * [book-ish] read through FastAPI's full documentation (amazing)
 * [book-ish] read through Kubernetes' documentation related to CKAD (very nice examples)
-* [leetcode] (Jan - ongoing) [DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode
 * [course] covered the study materials for [Certified Kubernetes Applications Developer](https://www.udemy.com/course/certified-kubernetes-application-developer/) (CKAD)
 * [course] (Jan-Mar) watched some of [Stanford's MLSys seminar series](https://www.youtube.com/playlist?list=PLSrTvUm384I9PV10koj_cqit9OfbJXEkq)
 * [course] covered [Stanford's CS149 - Parallel Computing](https://www.youtube.com/playlist?list=PLoROMvodv4rMp7MTFr4hQsDEcX7Bx6Odp)
 * [mini-course] built a [mini-http server from scratch](https://github.com/divakaivan/build-http-scratch) & [learned about nginx](https://github.com/divakaivan/nginx-crash-course)
 * [course] covered Eric Riddoch's [Taking Python to Production: A Professional Onboarding Guide](https://www.udemy.com/course/setting-up-the-linux-terminal-for-software-development/)
-* [bootcamp] will be part of Eric Riddoch's [Cloud Engineering for Python Devs](https://mlops-club.org/) bootcamp
 * [bootcamp] (Jan - Mar) completed Maria Vechtomova and Başak Eskili's End-to-end MLOps with Databricks boot camp (got as a Christmas present 😆)
 * [bootcamp] (Jan - Mar) Completed Zach Wilson's Data Engineering boot camp (won my spot by being an active member and helper in Zach's free DE boot camp) - [Combined Certificate of Superbness](https://www.dataexpert.io/certification/divakaivan/combined-superb)
 
+
 ## February
 
 * [book] read [ML for Tabular data](https://www.manning.com/books/machine-learning-for-tabular-data) by Mark Ryan and Luca Massaron
@@ -37,6 +43,7 @@ permalink: /2025/
 * [project] [Finished a full-stack voice-to-voice app that lets users interact with their finances through an AI agent](https://github.com/divakaivan/voice2voice-banking-assistant)
 * [project] [Finished my capstone for Zach Wilson's bootcamp - Using voice data to extract insight from team communication](https://github.com/divakaivan/lolesports-voice-analytics)
 
+
 ## January
 
 * [book] read Clean Coder, The: A Code of Conduct for Professional Programmers by Robert C. Martin

From e9651dc7e125ee63154ae74d0971248370838fe0 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 31 Mar 2025 16:57:45 +0900
Subject: [PATCH 605/638] Update search.md

---
 _pages/search.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 11239406a..34a1fa267 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -8,12 +8,13 @@ permalink: /2025/
 
 *Updated: 31st March 2025*
 
-## April *(currently)*
+## April *(ongoing, not final)*
 
-* [book] Write an interpreter in Go by [Throsten Ball](https://interpreterbook.com/)
 * [leetcode] [DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode (Jan - ongoing)
-* [bootcamp] will be part of Eric Riddoch's [Cloud Engineering for Python Devs](https://mlops-club.org/) bootcamp (Apr - May)
-
+* [bootcamp] will be part of Eric Riddoch's [Cloud Engineering for Python Devs](https://mlops-club.org/) bootcamp (~7-8 weeks)
+* continue reviewing/studying system design concepts + [hellointerview](https://www.hellointerview.com/learn/system-design/in-a-hurry/introduction) + [Jordan has no life](https://www.youtube.com/@jordanhasnolife5163)
+* [book] will go through Write an interpreter in Go by [Throsten Ball](https://interpreterbook.com/) (+ compiler in Go)
+* [book] will read REST API design rule book by [Mark Masse](https://www.oreilly.com/library/view/rest-api-design/9781449317904/)
 
 ## March
 

From 9af201d6e28204de058a1cb5cd3c8bc4b52a8b01 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 31 Mar 2025 18:09:43 +0900
Subject: [PATCH 606/638] Update projects.md

---
 _pages/projects.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/_pages/projects.md b/_pages/projects.md
index 0df986a4b..b1daf6a4b 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -15,6 +15,27 @@ title: Projects
     <a href="https://github.com/divakaivan/mlops-101" target="_blank">View Project</a>
   </div>
 
+  <div class="project-item">
+    <h3>RSS Aggregator API</h3>
+    <p>
+      <li>Developed an API that allows users to authenticate, scrape RSS feeds, follow feeds of their choice, and view posts from those feeds</li>
+      <li>The API is fully tested, dockerized, and available on Docker Hub</li>
+      <li>Deployed the API in a local Kubernetes setup with dashboards for monitoring both Kubernetes and the API</li>
+    </p>
+    <p><b>Tech: Go, PostgreSQL, GitHub Actions, Docker, Kubernetes, Prometheus, Grafana</b></p>
+    <a href="https://github.com/divakaivan/rssagg" target="_blank">View Project</a>
+  </div>
+  
+  <div class="project-item">
+    <h3>Platform Engineering with Backstage</h3>
+    <p>
+      <li>Built and deployed a Python API, created CI/CD pipelines with GitHub Actions, Helm, and ArgoCD for streamlined Kubernetes deployments</li>
+      <li>Registered components in Backstage’s software catalog, managed team ownership, published TechDocs, and deployed Backstage in production using Docker & Kubernetes</li>
+    </p>
+    <p><b>Tech: GitHub Actions, Docker, Kubernetes, ArgoCD, Helm, Backstage</b></p>
+    <a href="https://www.youtube.com/watch?v=i4cQzjRNz4M" target="_blank">View Video</a>
+  </div>
+
   <div class="project-item">
     <h3>Esports Voice Data Pipeline (Zach Wilson's DE bootcamp capstone)</h3>
     <p>

From 28f1ce04d06f19b201ebc4c1be85ad6b79bc581a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 1 May 2025 00:35:16 +0900
Subject: [PATCH 607/638] Update search.md

---
 _pages/search.md | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 34a1fa267..0b7628d10 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,15 +6,32 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 31st March 2025*
+*Updated: 1 May 2025*
 
-## April *(ongoing, not final)*
+## May *ongoing/not final*
 
-* [leetcode] [DSA course](https://leetcode.com/explore/interview/card/leetcodes-interview-crash-course-data-structures-and-algorithms/) + blind 75/150 from neetcode (Jan - ongoing)
-* [bootcamp] will be part of Eric Riddoch's [Cloud Engineering for Python Devs](https://mlops-club.org/) bootcamp (~7-8 weeks)
-* continue reviewing/studying system design concepts + [hellointerview](https://www.hellointerview.com/learn/system-design/in-a-hurry/introduction) + [Jordan has no life](https://www.youtube.com/@jordanhasnolife5163)
-* [book] will go through Write an interpreter in Go by [Throsten Ball](https://interpreterbook.com/) (+ compiler in Go)
-* [book] will read REST API design rule book by [Mark Masse](https://www.oreilly.com/library/view/rest-api-design/9781449317904/)
+* preparing for potential interviews
+* continue learning Go and and its application in creating backends/interpreters/databases
+* continue reviewing/studying system design concepts
+* continue with Stanford's AA228V course
+* teaching Python in Stanford's Code in Place 2025
+* learning about FastAPI apps on AWS - [Cloud Engineering for Python Devs](https://mlops-club.org/)
+
+## April 
+
+* [course] Started teaching Python in Stanford's Code in Place 2025
+* [course] Started Stanford's AA228V Validation of Safety Critical Systems I System Modeling [link](https://www.youtube.com/watch?v=hE9iWwMZANE&list=PLoROMvodv4rOq1LMLI8U7djzDb8--xpaC)
+* [course] Learned about memory management in C and built a basic garbage collector on [boot.dev](https://www.boot.dev/courses/learn-memory-management-c)
+* [course] Finished teaching the [MLOps 101](https://github.com/divakaivan/mlops-101) course at my university club
+* [oss] Made my first oss contributions to [skore - the scikit-learn sidekick](https://github.com/probabl-ai/skore)
+* [mini-project] [Develop an ML model API following the Open Inference Protocol](https://github.com/divakaivan/model-api-oip)
+* [book] [The Last Algorithms Course You'll Need](https://theprimeagen.github.io/fem-algos)
+* [book] Write an interpreter in Go by [Throsten Ball](https://interpreterbook.com/) + [this youtube series about a language parser in Go](https://www.youtube.com/watch?v=SWjFg5n23-g)
+* [book] The Alignment Problem by Brian Christian
+* [book] [Let's Go](https://lets-go.alexedwards.net/) by Alex Edwards
+* [book] [Build Your Own Database From Scratch in Go](https://build-your-own.org/database/)
+* [cartoon-book] [How HTTPS Works](https://howhttps.works/)
+* [btw] Started using neovim when writing Go and I ❤️ it
 
 ## March
 

From 98ea22d830a697c7eac7ccb13dff0b5a44701a77 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 1 May 2025 00:58:41 +0900
Subject: [PATCH 608/638] Update search.md

---
 _pages/search.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index 0b7628d10..160e64c4f 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -10,7 +10,6 @@ permalink: /2025/
 
 ## May *ongoing/not final*
 
-* preparing for potential interviews
 * continue learning Go and and its application in creating backends/interpreters/databases
 * continue reviewing/studying system design concepts
 * continue with Stanford's AA228V course

From dad0dd8ced40881e32ef33796edbbf7651b9be21 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sun, 1 Jun 2025 20:14:52 +0900
Subject: [PATCH 609/638] Update search.md

---
 _pages/search.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 160e64c4f..75c70e8c2 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,15 +6,16 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 1 May 2025*
+*Updated: 1 June 2025*
 
-## May *ongoing/not final*
+## May
 
-* continue learning Go and and its application in creating backends/interpreters/databases
-* continue reviewing/studying system design concepts
-* continue with Stanford's AA228V course
-* teaching Python in Stanford's Code in Place 2025
-* learning about FastAPI apps on AWS - [Cloud Engineering for Python Devs](https://mlops-club.org/)
+* [book] Finished Writing an Interpreter in Go (along with some YT videos) - have a bit better understanding of how interpreters work
+* [teach] Completed my last official session for Stanford's Code in Place 2025 - will have a voluntary extra session to answer any student questions
+* [teach] Did 9 TeachNow sessions in the Code in Place platform (1-to-1 help sessions around 30mins each)
+* [zig] Read the Zig language docs + did Ziglings on youtube
+* [zig] Writing Karpathy's micrograd in Zig for AI + Zig practice (wip)
+* [course] Concluded Eroc Riddoch's Cloud Eng for Python devs course
 
 ## April 
 

From 49df456b758682901193e93ade1615783fefee3e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Thu, 5 Jun 2025 19:30:41 +0900
Subject: [PATCH 610/638] Update search.md

---
 _pages/search.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_pages/search.md b/_pages/search.md
index 75c70e8c2..b7e81be28 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -11,6 +11,7 @@ permalink: /2025/
 ## May
 
 * [book] Finished Writing an Interpreter in Go (along with some YT videos) - have a bit better understanding of how interpreters work
+* [tool] Explored ICO's AI and Data protection risk toolkint + Microsoft's responsible AI toolbox
 * [teach] Completed my last official session for Stanford's Code in Place 2025 - will have a voluntary extra session to answer any student questions
 * [teach] Did 9 TeachNow sessions in the Code in Place platform (1-to-1 help sessions around 30mins each)
 * [zig] Read the Zig language docs + did Ziglings on youtube

From 81763ec856ef0a1cf94a282709b69ce50985bca3 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 2 Jul 2025 09:30:19 +0300
Subject: [PATCH 611/638] Update search.md

---
 _pages/search.md | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index b7e81be28..c3b44df2f 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,7 +6,24 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 1 June 2025*
+*Updated: 1 July 2025*
+
+## June 
+
+- [teach] Finished my Stanford University Code in Place 2025 Section Leader responsibilities 🥳 
+- [mini-project] Network programming in C during my flight. I was on a ~15hr transfer so I wanted to do something that doesn't require wifi. I found "Beef's introduction to Network programmin" and it blew my mind with how easy it describes networking concepts, with great examples and I was able to create my own scripts:
+
+I ended up making 3 things (some of the code overlaps):
+1. Read entire data.txt into memory and send over TCP to a client
+2. Stream large data.txt in fix-sized chunks over TCP
+3. Receive TCP data and write to stdout
+
+[Github](https://github.com/divakaivan/networking-in-c)
+
+- [book] Read "Make It Stick: The Science of Successful Learning" - amazing 
+- [mini-project] Created a simple ssh politics.news terminal app to help introduce people interested in politics to the terminal (inspired by ssh terminal.shop) (using Go, wish, bubblegum, other Charm libs); [Github](https://github.com/divakaivan/politics.news) (need to add README and figure out if I can deploy it for free somewhere)
+- [book] Read "Responsible AI: implementing unbiased and ethical algorithms" - amazing; definitely something to re-read, refer to for responsible AI questions, and recommend to beginners
+
 
 ## May
 

From 601c9a4ab80e7f68a0f23d077c758d09bb3c2af6 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:28:35 +0100
Subject: [PATCH 612/638] Update categories.md

---
 _pages/categories.md | 106 ++++++++-----------------------------------
 1 file changed, 20 insertions(+), 86 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index e47150437..d9dabd1f0 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -1,94 +1,28 @@
 ---
 layout: page
-permalink: /categories/
-title: Categories
+permalink: /esoc/
+title: ESoC skore contributions
 ---
 
-<style>
-  /* Styling the dropdown */
-  #category-select {
-    width: 100%;
-    padding: 10px;
-    font-size: 16px;
-    border: 2px solid #ccc;
-    border-radius: 5px;
-    background-color: #f9f9f9;
-    margin-bottom: 20px;
-    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
-  }
-
-  /* Styling the category content */
-  .category-content {
-    margin-top: 20px;
-    padding: 15px;
-    border: 1px solid #e1e1e1;
-    border-radius: 5px;
-    background-color: #ffffff;
-    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05);
-  }
-
-  /* Styling the category titles */
-  .category-head {
-    font-size: 22px;
-    font-weight: bold;
-    margin-bottom: 10px;
-    color: #333;
-  }
-
-  /* Styling the post titles */
-  .archive-item h4 {
-    font-size: 18px;
-    margin: 5px 0;
-    color: #0073e6;
-  }
-
-  .archive-item h4 a {
-    text-decoration: none;
-  }
-
-  .archive-item h4 a:hover {
-    text-decoration: underline;
-  }
-</style>
-
-<div id="archives">
-  <label for="category-select">Choose a category:</label>
-  <select id="category-select" onchange="location = this.value;">
-    <option value="">--Select a category--</option>
-    {% for category in site.categories %}
-      {% capture category_name %}{{ category | first }}{% endcapture %}
-      <option value="#{{ category_name | slugize }}">
-        {{ category_name }} ({{ site.categories[category_name].size }})
-      </option>
-    {% endfor %}
-  </select>
-
-  <div id="category-list">
-    {% for category in site.categories %}
-      {% capture category_name %}{{ category | first }}{% endcapture %}
-      <div id="{{ category_name | slugize }}" style="display:none;" class="category-content">
-        <h3 class="category-head">{{ category_name }} ({{ site.categories[category_name].size }})</h3>
-        <a name="{{ category_name | slugize }}"></a>
-        {% for post in site.categories[category_name] %}
-        <article class="archive-item">
-          <h4><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></h4>
-        </article>
-        {% endfor %}
-      </div>
-    {% endfor %}
-  </div>
-</div>
+<h1>My GitHub Contributions</h1>
+<div id="contributions"></div>
 
 <script>
-  // JavaScript to show selected category
-  document.getElementById('category-select').addEventListener('change', function() {
-    var selectedCategory = this.value;
-    var contents = document.querySelectorAll('.category-content');
-    contents.forEach(function(content) {
-      content.style.display = 'none';
+  const username = "divakaivan"; // replace with your username
+  const repo = "probabl-ai/skore"; // or loop over multiple repos
+  const url = `https://api.github.com/repos/${username}/${repo}/commits?author=${username}`;
+
+  fetch(url)
+    .then(res => res.json())
+    .then(data => {
+      const container = document.getElementById("contributions");
+      data.forEach(commit => {
+        const item = document.createElement("div");
+        item.innerHTML = `
+          <p><strong>${commit.commit.message}</strong></p>
+          <p><a href="${commit.html_url}" target="_blank">View Commit</a></p>
+        `;
+        container.appendChild(item);
+      });
     });
-    if (selectedCategory) {
-      document.querySelector(selectedCategory).style.display = 'block';
-    }
-  });
 </script>

From 1bbb58f5ad66631dbc26b754d267de4783d8b1ed Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:30:27 +0100
Subject: [PATCH 613/638] Update categories.md

---
 _pages/categories.md | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index d9dabd1f0..2135dbeb8 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -15,14 +15,6 @@ title: ESoC skore contributions
   fetch(url)
     .then(res => res.json())
     .then(data => {
-      const container = document.getElementById("contributions");
-      data.forEach(commit => {
-        const item = document.createElement("div");
-        item.innerHTML = `
-          <p><strong>${commit.commit.message}</strong></p>
-          <p><a href="${commit.html_url}" target="_blank">View Commit</a></p>
-        `;
-        container.appendChild(item);
-      });
+      console.log(data)
     });
 </script>

From 88f55e1b61b066a881f89e63ec935d070ff0224f Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:31:10 +0100
Subject: [PATCH 614/638] Update categories.md

---
 _pages/categories.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 2135dbeb8..480f45b02 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -8,9 +8,7 @@ title: ESoC skore contributions
 <div id="contributions"></div>
 
 <script>
-  const username = "divakaivan"; // replace with your username
-  const repo = "probabl-ai/skore"; // or loop over multiple repos
-  const url = `https://api.github.com/repos/${username}/${repo}/commits?author=${username}`;
+  const url = `https://api.github.com/repos/probabl-ai/skore/commits?author=divakaivan`;
 
   fetch(url)
     .then(res => res.json())

From 7c83c78d1be4b468d05fe56e25a7a09cd02df445 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:37:08 +0100
Subject: [PATCH 615/638] Update categories.md

---
 _pages/categories.md | 53 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 45 insertions(+), 8 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 480f45b02..038edc113 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -4,15 +4,52 @@ permalink: /esoc/
 title: ESoC skore contributions
 ---
 
-<h1>My GitHub Contributions</h1>
-<div id="contributions"></div>
+<h1>My GitHub Contributions to skore</h1>
+<ul id="commits"><strong>Commits</strong></ul>
+<ul id="prs"><strong>Pull Requests</strong></ul>
+<ul id="issues"><strong>Issues</strong></ul>
 
 <script>
-  const url = `https://api.github.com/repos/probabl-ai/skore/commits?author=divakaivan`;
+const user = 'divakaivan';
+const repo = 'probabl-ai/skore';
 
-  fetch(url)
-    .then(res => res.json())
-    .then(data => {
-      console.log(data)
-    });
+async function fetchCommits() {
+  const url = `https://api.github.com/repos/${repo}/commits?author=${user}`;
+  const res = await fetch(url);
+  const data = await res.json();
+  const container = document.getElementById('commits');
+  data.forEach(commit => {
+    const li = document.createElement('li');
+    li.innerHTML = `<a href="${commit.html_url}" target="_blank">${commit.commit.message}</a>`;
+    container.appendChild(li);
+  });
+}
+
+async function fetchPRs() {
+  const url = `https://api.github.com/search/issues?q=repo:${repo}+is:pr+author:${user}`;
+  const res = await fetch(url);
+  const data = await res.json();
+  const container = document.getElementById('prs');
+  data.items.forEach(pr => {
+    const li = document.createElement('li');
+    li.innerHTML = `<a href="${pr.html_url}" target="_blank">${pr.title}</a>`;
+    container.appendChild(li);
+  });
+}
+
+async function fetchIssues() {
+  const url = `https://api.github.com/search/issues?q=repo:${repo}+is:issue+author:${user}`;
+  const res = await fetch(url);
+  const data = await res.json();
+  const container = document.getElementById('issues');
+  data.items.forEach(issue => {
+    const li = document.createElement('li');
+    li.innerHTML = `<a href="${issue.html_url}" target="_blank">${issue.title}</a>`;
+    container.appendChild(li);
+  });
+}
+
+fetchCommits();
+fetchPRs();
+fetchIssues();
 </script>

From 9c8203c19bbd1e8da21acd7292eca4c6ea452e86 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:40:24 +0100
Subject: [PATCH 616/638] Update categories.md

---
 _pages/categories.md | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 038edc113..4829e1c45 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -30,11 +30,24 @@ async function fetchPRs() {
   const res = await fetch(url);
   const data = await res.json();
   const container = document.getElementById('prs');
-  data.items.forEach(pr => {
-    const li = document.createElement('li');
-    li.innerHTML = `<a href="${pr.html_url}" target="_blank">${pr.title}</a>`;
-    container.appendChild(li);
-  });
+
+  for (const pr of data.items) {
+    const prNumber = pr.number;
+
+    // Fetch full PR details to check merged status
+    const prDetailsRes = await fetch(`https://api.github.com/repos/${repo}/pulls/${pr.number}`);
+    const prDetails = await prDetailsRes.json();
+
+    const isOpen = prDetails.state === 'open';
+    const isMerged = prDetails.merged_at !== null;
+
+    if (isOpen || isMerged) {
+      const li = document.createElement('li');
+      const status = isOpen ? '🟢 Open' : '🟣 Merged';
+      li.innerHTML = `<a href="${pr.html_url}" target="_blank">${pr.title}</a> <small>(${status})</small>`;
+      container.appendChild(li);
+    }
+  }
 }
 
 async function fetchIssues() {

From 2c54d33785565c50eb7bbe4e3e34e7d3a44a6059 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:47:28 +0100
Subject: [PATCH 617/638] Update categories.md

---
 _pages/categories.md | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 4829e1c45..3cc9988f5 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -4,8 +4,7 @@ permalink: /esoc/
 title: ESoC skore contributions
 ---
 
-<h1>My GitHub Contributions to skore</h1>
-<ul id="commits"><strong>Commits</strong></ul>
+<h1>My Contributions to <a href="https://github.com/probabl-ai/skore" target="_blank">probabl-ai/skore</a> during my European Summer of Code 2025 stipend period</h1>
 <ul id="prs"><strong>Pull Requests</strong></ul>
 <ul id="issues"><strong>Issues</strong></ul>
 
@@ -13,18 +12,6 @@ title: ESoC skore contributions
 const user = 'divakaivan';
 const repo = 'probabl-ai/skore';
 
-async function fetchCommits() {
-  const url = `https://api.github.com/repos/${repo}/commits?author=${user}`;
-  const res = await fetch(url);
-  const data = await res.json();
-  const container = document.getElementById('commits');
-  data.forEach(commit => {
-    const li = document.createElement('li');
-    li.innerHTML = `<a href="${commit.html_url}" target="_blank">${commit.commit.message}</a>`;
-    container.appendChild(li);
-  });
-}
-
 async function fetchPRs() {
   const url = `https://api.github.com/search/issues?q=repo:${repo}+is:pr+author:${user}`;
   const res = await fetch(url);
@@ -62,7 +49,6 @@ async function fetchIssues() {
   });
 }
 
-fetchCommits();
 fetchPRs();
 fetchIssues();
 </script>

From b6e338d991de07fe821f1e4fcad835f29ad5c5d5 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:51:49 +0100
Subject: [PATCH 618/638] Update categories.md

---
 _pages/categories.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 3cc9988f5..db43121bf 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -4,7 +4,7 @@ permalink: /esoc/
 title: ESoC skore contributions
 ---
 
-<h1>My Contributions to <a href="https://github.com/probabl-ai/skore" target="_blank">probabl-ai/skore</a> during my European Summer of Code 2025 stipend period</h1>
+<h2>My Contributions to <a href="https://github.com/probabl-ai/skore" target="_blank">probabl-ai/skore</a> during my European Summer of Code 2025 stipend period</h2>
 <ul id="prs"><strong>Pull Requests</strong></ul>
 <ul id="issues"><strong>Issues</strong></ul>
 

From d7e89b225716dec6dc8e7ebc0e72ad0c3ee607db Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:53:54 +0100
Subject: [PATCH 619/638] Update default.html

---
 _layouts/default.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index dd6545433..eedd3083c 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -39,6 +39,7 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
               <a href="{{ site.baseurl }}/">Home</a>
               <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/2025">2025</a>
+              <a href="{{ site.baseurl }}/esoc">ESoC</a>
               <a href="{{ site.baseurl }}/about">About</a>
             </nav>
           </header>

From 566e106df12360dd8c27d4eadb296efe02910beb Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:55:59 +0100
Subject: [PATCH 620/638] Update _config.yml

---
 _config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_config.yml b/_config.yml
index ac6ef686d..d76782f60 100644
--- a/_config.yml
+++ b/_config.yml
@@ -3,13 +3,13 @@
 #
 
 # Name of your site (displayed in the header)
-name: Learn + Blog every day in 2024
+name: Learn + Blog
 
 # Name of the author
 author: Ivan Ivanov
 
 # Short bio or description (displayed in the header)
-description: Math, Stats, AI, Data Eng, MLOps!
+description: Daily 2024/Monthly 2025
 
 # URL of your avatar or profile pic (you could use your GitHub profile pic)
 avatar: /avatar.jpeg

From 2d3174f36531fcbf0f7752542dfeb77a52e46dc2 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:02:01 +0100
Subject: [PATCH 621/638] Update categories.md

---
 _pages/categories.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/_pages/categories.md b/_pages/categories.md
index db43121bf..7c4c210bd 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -5,6 +5,9 @@ title: ESoC skore contributions
 ---
 
 <h2>My Contributions to <a href="https://github.com/probabl-ai/skore" target="_blank">probabl-ai/skore</a> during my European Summer of Code 2025 stipend period</h2>
+
+I [won a stipend](https://www.linkedin.com/posts/activity-7350417081953079297-FBY1) from the European Summer of Code to contribute to an OSS project - **skore - the scikit-learn sidekick**, by [probabl.ai](https://probabl.ai/).
+
 <ul id="prs"><strong>Pull Requests</strong></ul>
 <ul id="issues"><strong>Issues</strong></ul>
 

From 8a78669cff5c1013cfcab793002e04f7fd8c6746 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:38:32 +0100
Subject: [PATCH 622/638] Update default.html

---
 _layouts/default.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index eedd3083c..5b00d23b5 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -36,9 +36,9 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
             </div>
 
             <nav>
-              <a href="{{ site.baseurl }}/">Home</a>
-              <a href="{{ site.baseurl }}/projects">Projects</a>
+              <a href="{{ site.baseurl }}/">2024</a>
               <a href="{{ site.baseurl }}/2025">2025</a>
+              <a href="{{ site.baseurl }}/projects">Projects</a>
               <a href="{{ site.baseurl }}/esoc">ESoC</a>
               <a href="{{ site.baseurl }}/about">About</a>
             </nav>

From 6987f12aeb1726fea678452134bd84e36a5a6ee0 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 2 Aug 2025 18:46:30 +0100
Subject: [PATCH 623/638] Update search.md

---
 _pages/search.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index c3b44df2f..1105371ad 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,7 +6,14 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 1 July 2025*
+*Updated: 1 Aug 2025*
+
+## July
+
+- started my OSS contributor journey with probabl.ai under the European Summer of Code stipend; I also added a new page to me blog fetching my contributions - [ESoC page](https://ivanstudyblog.github.io/esoc)
+- new youtube video Using keras SKLearnClassifier in a skore report - [youtube](https://youtu.be/Hpiuwc-Dxog)
+- decided to learn more about OS -> found [OS in 1000 lines of code](https://operating-system-in-1000-lines.vercel.app/en/)
+- dived into *that XOR trick* and how python's sum() works
 
 ## June 
 

From 61d3bd0f0fc62612c64bf566d375a1de4e8e0e16 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 9 Aug 2025 14:18:33 +0100
Subject: [PATCH 624/638] Update projects.md

---
 _pages/projects.md | 209 ++++++++++++++++++++++++++-------------------
 1 file changed, 123 insertions(+), 86 deletions(-)

diff --git a/_pages/projects.md b/_pages/projects.md
index b1daf6a4b..8dd8f1cd1 100644
--- a/_pages/projects.md
+++ b/_pages/projects.md
@@ -4,7 +4,19 @@ permalink: /projects/
 title: Projects
 ---
 
+<details>
+<summary>AI/ML/MLOps</summary>
+<br>
 <div id="projects">
+  <div class="project-item">
+    <h3>Sample ML model API following the Open Inference Protocol</h3>
+    <p>
+      <li>Designed following the Open Inference Protocol — a growing industry standard for standardized, observable, and interoperable machine learning inference</li>
+    </p>
+    <p><b>Tech: scikit-learn, FastAPI, pytest, pre-commit, Docker, Github Actions, Pydantic</b></p>
+    <a href="https://github.com/divakaivan/model-api-oip" target="_blank">View Project</a>
+  </div>
+  
   <div class="project-item">
     <h3>MLOps 101 Project for a mini-course I teach</h3>
     <p>
@@ -16,27 +28,74 @@ title: Projects
   </div>
 
   <div class="project-item">
-    <h3>RSS Aggregator API</h3>
+    <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
     <p>
-      <li>Developed an API that allows users to authenticate, scrape RSS feeds, follow feeds of their choice, and view posts from those feeds</li>
-      <li>The API is fully tested, dockerized, and available on Docker Hub</li>
-      <li>Deployed the API in a local Kubernetes setup with dashboards for monitoring both Kubernetes and the API</li>
+      <li>An AI-driven solution for real-time credit card fraud detection using MLOps techniques</li>
+      <li>Fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring</li>
+      <li>High fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models</li>
     </p>
-    <p><b>Tech: Go, PostgreSQL, GitHub Actions, Docker, Kubernetes, Prometheus, Grafana</b></p>
-    <a href="https://github.com/divakaivan/rssagg" target="_blank">View Project</a>
+    <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
+    <a href="https://github.com/divakaivan/kb_project" target="_blank">View Project</a>
   </div>
+
+  <div class="project-item">
+    <h3>Voice-to-Voice Personal Finance Assistant</h3>
+    <p>
+      <li>Talk, Learn and Analyse your spending habits with your Personal Finance Assistant AI Agent. Communicate through speech</li>
+      <li>Frontend + Backend communicating via a websocket</li>
+      <li>Ask follow-up questions (the language model can see the chat history)</li>
+      <li>Detailed observability of live and historical connections to the server via Pydantic Logfire</li>
+    </p>
+    <p><b>Tech: Webhooks, FastAPI, PydanticAI, Logfire, SQLite, OpenAI, Ollama, PostgreSQL, React</b></p>
+    <a href="https://github.com/divakaivan/voice2voice-banking-assistant" target="_blank">View on GitHub</a>
+  </div>
+
   
   <div class="project-item">
-    <h3>Platform Engineering with Backstage</h3>
+    <h3>MLOps Architecture for Insurance Fraud Detection</h3>
     <p>
-      <li>Built and deployed a Python API, created CI/CD pipelines with GitHub Actions, Helm, and ArgoCD for streamlined Kubernetes deployments</li>
-      <li>Registered components in Backstage’s software catalog, managed team ownership, published TechDocs, and deployed Backstage in production using Docker & Kubernetes</li>
+      <li>Building an end-to-end MLOps pipeline to detect car insurance fraud</li>
+      <li>Pipeline orchestration covering data storage, data preprocessing (using IV and WoE), model training, deployment, and monitoring</li>
+      <li>Focus on achieving high recall in fraud detection using a Balanced Random Forest Classifier</li>
     </p>
-    <p><b>Tech: GitHub Actions, Docker, Kubernetes, ArgoCD, Helm, Backstage</b></p>
-    <a href="https://www.youtube.com/watch?v=i4cQzjRNz4M" target="_blank">View Video</a>
+    <p><b>Tech: PostgreSQL DB, Terraform, Google Cloud Platform, Mlflow, Prefect, Grafana, Evidently, Docker, FastAPI</b></p>
+    <a href="https://github.com/divakaivan/insurance-fraud-mlops-pipeline" target="_blank">View Project</a>
   </div>
 
+  <h3>Other AI/ML</h3>
   <div class="project-item">
+    <h4><a href="https://www.kaggle.com/divakaivan12/code" target="_blank">Notebook Expert on Kaggle</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://www.kaggle.com/code/divakaivan12/the-full-story-behind-multicollinearity" target="_blank">Diving into the full story behind multicollinearity</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/LingoMate" target="_blank">LingoMate - hackathon project for language learners</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://www.kaggle.com/code/divakaivan12/neural-network-epoch-by-hand" target="_blank">Doing a neural network epoch+math by hand</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/text2chart" target="_blank">text2chart - transforming natural language to charts</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/pdf-rag-from-scratch" target="_blank">PDF RAG from scratch</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/taxi-demand-video-models-paper" target="_blank">Using Video Generation Models for Taxi OD Demand Matrix Predictions</a></h4>
+  </div>
+  <div class="project-item">
+    <h4><a href="https://github.com/divakaivan/db2chat" target="_blank">db2chat - chat with your (SQLite) database</a></h4>
+  </div>
+</div>
+</details>
+
+---
+
+<details>
+<summary>Data Eng</summary>
+<br>
+<div class="project-item">
     <h3>Esports Voice Data Pipeline (Zach Wilson's DE bootcamp capstone)</h3>
     <p>
       <li>Esports team communication data is normally kept private, but for the first time a team is sharing their full voice communication records so with this I am showing a prototype of a pipeline that utilises audio data to extract communication patterns</li>
@@ -45,31 +104,27 @@ title: Projects
     <p><b>Tech: Airflow, dbt, Google BigQuery, Google Cloud Storage, Streamlit, Terraform, Github Actions, Astronomer</b></p>
     <a href="https://github.com/divakaivan/lolesports-voice-analytics" target="_blank">View Project</a>
   </div>
-  
-  <div class="project-item">
-    <h3>MLOps Architecture for Real-Time Fraud Detection</h3>
+
+<div class="project-item">
+    <h3>Write-Audit-Publish exercise YT video</h3>
     <p>
-      <li>An AI-driven solution for real-time credit card fraud detection using MLOps techniques</li>
-      <li>Fully orchestrated pipelines, including data ingestion, model training, and real-time prediction and monitoring</li>
-      <li>High fraud case detection through Graph Convolutional Network, XGBoost, and CatBoost models</li>
+      <li>Recorded a youtube tutorial on how to follow the WAP DQ practice using popular tech</li>
     </p>
-    <p><b>Tech: Neo4j graph DB, Sklearn, PyG, Mlflow, Kafka, Grafana, Mage orchestration, Docker, Streamlit</b></p>
-    <a href="https://github.com/divakaivan/kb_project" target="_blank">View Project</a>
+    <p><b>Tech: Dremio, Apache Iceberg, Nessie, MinIO</b></p>
+    <a href="https://www.youtube.com/watch?v=CzkBkwv1Llc&ab_channel=istudythings" target="_blank">View Project</a>
   </div>
 
-  <div class="project-item">
-    <h3>Voice-to-Voice Personal Finance Assistant</h3>
+<div class="project-item">
+    <h3>EU AI Act Graph Modelling</h3>
     <p>
-      <li>Talk, Learn and Analyse your spending habits with your Personal Finance Assistant AI Agent. Communicate through speech</li>
-      <li>Frontend + Backend communicating via a websocket</li>
-      <li>Ask follow-up questions (the language model can see the chat history)</li>
-      <li>Detailed observability of live and historical connections to the server via Pydantic Logfire</li>
+      <li>Scraped the EU AI Act website and created a conceptual, logical and a physical data model, improving my understanding of the Act’s requirements</li>
+      <li>Separated entities into Articles, Recitals, Annexes, Chapters, Versions, Summaries and implemented the physical model using a graph database</li>
     </p>
-    <p><b>Tech: Webhooks, FastAPI, PydanticAI, Logfire, SQLite, OpenAI, Ollama, PostgreSQL, React</b></p>
-    <a href="https://github.com/divakaivan/voice2voice-banking-assistant" target="_blank">View on GitHub</a>
+    <p><b>Tech: Python, Neo4j, BeautifulSoup</b></p>
+    <a href="https://github.com/divakaivan/eu-ai-act-graph-modelling" target="_blank">View Project</a>
   </div>
 
-  <div class="project-item">
+<div class="project-item">
     <h3>Transaction Stream Data Engineering Pipeline</h3>
     <p>
       <li>Generate transaction data via Stripe's API</li>
@@ -93,18 +148,7 @@ title: Projects
     <a href="https://huggingface.co/datasets/divakaivan/glaswegian_audio" target="_blank">View on HuggingFace</a>
   </div>
   
-  <div class="project-item">
-    <h3>MLOps Architecture for Insurance Fraud Detection</h3>
-    <p>
-      <li>Building an end-to-end MLOps pipeline to detect car insurance fraud</li>
-      <li>Pipeline orchestration covering data storage, data preprocessing (using IV and WoE), model training, deployment, and monitoring</li>
-      <li>Focus on achieving high recall in fraud detection using a Balanced Random Forest Classifier</li>
-    </p>
-    <p><b>Tech: PostgreSQL DB, Terraform, Google Cloud Platform, Mlflow, Prefect, Grafana, Evidently, Docker, FastAPI</b></p>
-    <a href="https://github.com/divakaivan/insurance-fraud-mlops-pipeline" target="_blank">View Project</a>
-  </div>
-
-  <div class="project-item">
+<div class="project-item">
     <h3>Lending Club Data Engineering Pipeline</h3>
     <p>
       <li>Build a data pipeline to process and visualize Lending Club data</li>
@@ -117,65 +161,58 @@ title: Projects
     <p><b>Tech: Docker, Mage orchestration, Google Cloud Platform, Terraform, dbt, Looker</b></p>
     <a href="https://github.com/divakaivan/lending-club-data-pipeline" target="_blank">View Project</a>
   </div>
+</details>
 
-  <h2>Other</h2>
-  <div class="project-item">
-    <h4><a href="https://www.kaggle.com/divakaivan12/code" target="_blank">Notebook Expert on Kaggle</a></h4>
+---
+
+<details>
+<summary>Web/API/Other</summary>
+<br>
+<div class="project-item">
+    <h3>RSS Aggregator API</h3>
+    <p>
+      <li>Developed an API that allows users to authenticate, scrape RSS feeds, follow feeds of their choice, and view posts from those feeds</li>
+      <li>The API is fully tested, dockerized, and available on Docker Hub</li>
+      <li>Deployed the API in a local Kubernetes setup with dashboards for monitoring both Kubernetes and the API</li>
+    </p>
+    <p><b>Tech: Go, PostgreSQL, GitHub Actions, Docker, Kubernetes, Prometheus, Grafana</b></p>
+    <a href="https://github.com/divakaivan/rssagg" target="_blank">View Project</a>
   </div>
+
+<div class="project-item">
+    <h3>Platform Engineering with Backstage</h3>
+    <p>
+      <li>Built and deployed a Python API, created CI/CD pipelines with GitHub Actions, Helm, and ArgoCD for streamlined Kubernetes deployments</li>
+      <li>Registered components in Backstage’s software catalog, managed team ownership, published TechDocs, and deployed Backstage in production using Docker & Kubernetes</li>
+    </p>
+    <p><b>Tech: GitHub Actions, Docker, Kubernetes, ArgoCD, Helm, Backstage</b></p>
+    <a href="https://www.youtube.com/watch?v=i4cQzjRNz4M" target="_blank">View Video</a>
+  </div>  
+
   <div class="project-item">
-    <h4><a href="https://www.kaggle.com/code/divakaivan12/the-full-story-behind-multicollinearity" target="_blank">Diving into the full story behind multicollinearity</a></h4>
+    <h3>Sample ML model API following the Open Inference Protocol</h3>
+    <p>
+      <li>Designed following the Open Inference Protocol — a growing industry standard for standardized, observable, and interoperable machine learning inference</li>
+    </p>
+    <p><b>Tech: scikit-learn, FastAPI, pytest, pre-commit, Docker, Github Actions, Pydantic</b></p>
+    <a href="https://github.com/divakaivan/model-api-oip" target="_blank">View Project</a>
   </div>
   <div class="project-item">
-    <h4><a href="https://www.kaggle.com/code/divakaivan12/neural-network-epoch-by-hand" target="_blank">Doing a neural network epoch by hand</a></h4>
+    <h4><a href="https://github.com/divakaivan/politics.news" target="_blank">politics.news - TUI for reading political news from the terminal using Go</a></h4>
   </div>
   <div class="project-item">
-    <h4><a href="https://github.com/divakaivan/text2chart" target="_blank">text2chart - transforming natural language to charts</a></h4>
+    <h4><a href="https://github.com/divakaivan/classify_hangul" target="_blank">Webapp for classifying hand-written Hangul characters</a></h4>
   </div>
   <div class="project-item">
-    <h4><a href="https://github.com/divakaivan/pdf-rag-from-scratch" target="_blank">PDF RAG from scratch</a></h4>
+    <h4><a href="https://github.com/divakaivan/psql-express-yelp-api" target="_blank">RESTful Node.js/Express backend for a restaurant review application</a></h4>
   </div>
   <div class="project-item">
-    <h4><a href="https://github.com/divakaivan/taxi-demand-video-models-paper" target="_blank">Using Video Generation Models for Taxi OD Demand Matrix Predictions</a></h4>
+    <h4><a href="https://github.com/divakaivan/hanja-game" target="_blank">Webapp to Practice Hanja Characters for Hanja Exam</a></h4>
   </div>
   <div class="project-item">
-    <h4><a href="https://github.com/divakaivan/db2chat" target="_blank">db2chat - chat with your (SQLite) database</a></h4>
+    <h4><a href="https://korean-quiz-game-ian6qmcm2-divakaivan.vercel.app/" target="_blank">Created a quiz webapp with custom questiones on general topics</a></h4>
   </div>
   <div class="project-item">
-    <h4><a href="https://github.com/divakaivan/classify_hangul" target="_blank">CV model to classify Hangul characters</a></h4>
+    <h4><a href="https://github.com/divakaivan/team-comms-discord-bot" target="_blank">Discord bot for audio transcriptions</a></h4>
   </div>
-  
-</div>
-
-<style>
-  #projects {
-    display: block;
-  }
-
-  .project-item {
-    border: 1px solid #ccc;
-    padding: 10px;
-    border-radius: 8px;
-    width: 100%;
-    box-shadow: 0 4px 8px rgba(0,0,0,0.1);
-    margin-bottom: 20px;
-  }
-
-  .project-item h3 {
-    margin-top: 0;
-  }
-
-  .project-item h4 {
-    margin-top: 0;
-  }
-
-  .project-item a {
-    display: inline-block;
-    margin-top: 10px;
-    color: #007BFF;
-    text-decoration: none;
-  }
-
-  .project-item a:hover {
-    text-decoration: underline;
-  }
-</style>
+</details>

From 765d1c15e503be3c120936fe5f3ac11a607bd564 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 20 Aug 2025 13:36:53 +0100
Subject: [PATCH 625/638] Update categories.md

---
 _pages/categories.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/_pages/categories.md b/_pages/categories.md
index 7c4c210bd..19c37cc19 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -9,6 +9,9 @@ title: ESoC skore contributions
 I [won a stipend](https://www.linkedin.com/posts/activity-7350417081953079297-FBY1) from the European Summer of Code to contribute to an OSS project - **skore - the scikit-learn sidekick**, by [probabl.ai](https://probabl.ai/).
 
 <ul id="prs"><strong>Pull Requests</strong></ul>
+
+<a href="https://github.com/skorch-dev/skorch/pull/1119" target="_blank">Implement `__sklearn_is_fitted__` for skorch models, following sklearn custom model protocol</a> <small>(🟣 Merged)</small>
+
 <ul id="issues"><strong>Issues</strong></ul>
 
 <script>

From 3af75ee0962da7d2e34280e388103f65e5dd4663 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 27 Aug 2025 20:10:42 +0100
Subject: [PATCH 626/638] Update categories.md

---
 _pages/categories.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 19c37cc19..8b8289811 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -10,7 +10,7 @@ I [won a stipend](https://www.linkedin.com/posts/activity-7350417081953079297-FB
 
 <ul id="prs"><strong>Pull Requests</strong></ul>
 
-<a href="https://github.com/skorch-dev/skorch/pull/1119" target="_blank">Implement `__sklearn_is_fitted__` for skorch models, following sklearn custom model protocol</a> <small>(🟣 Merged)</small>
+<li><a href="https://github.com/skorch-dev/skorch/pull/1119" target="_blank">Implement `__sklearn_is_fitted__` for skorch models, following sklearn custom model protocol</a> <small>(🟣 Merged)</small></li>
 
 <ul id="issues"><strong>Issues</strong></ul>
 

From e1e72d03100f50ceaf0660da0f57ada2071931ea Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 27 Aug 2025 20:13:22 +0100
Subject: [PATCH 627/638] Add pull request details to categories page

---
 _pages/categories.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index 8b8289811..fd2e57313 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -8,9 +8,15 @@ title: ESoC skore contributions
 
 I [won a stipend](https://www.linkedin.com/posts/activity-7350417081953079297-FBY1) from the European Summer of Code to contribute to an OSS project - **skore - the scikit-learn sidekick**, by [probabl.ai](https://probabl.ai/).
 
-<ul id="prs"><strong>Pull Requests</strong></ul>
-
-<li><a href="https://github.com/skorch-dev/skorch/pull/1119" target="_blank">Implement `__sklearn_is_fitted__` for skorch models, following sklearn custom model protocol</a> <small>(🟣 Merged)</small></li>
+<ul id="prs">
+  <strong>Pull Requests</strong>
+  <li>
+    <a href="https://github.com/skorch-dev/skorch/pull/1119" target="_blank">
+      Implement `__sklearn_is_fitted__` for skorch models, following sklearn custom model protocol
+    </a>
+    <small>(🟣 Merged)</small>
+  </li>
+</ul>
 
 <ul id="issues"><strong>Issues</strong></ul>
 

From 519652e9ee3b979ca019619cd8b36d41e71f4fac Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Fri, 12 Sep 2025 18:32:49 +0100
Subject: [PATCH 628/638] Revise contribution statement in categories.md

---
 _pages/categories.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_pages/categories.md b/_pages/categories.md
index fd2e57313..20a41178a 100644
--- a/_pages/categories.md
+++ b/_pages/categories.md
@@ -6,7 +6,7 @@ title: ESoC skore contributions
 
 <h2>My Contributions to <a href="https://github.com/probabl-ai/skore" target="_blank">probabl-ai/skore</a> during my European Summer of Code 2025 stipend period</h2>
 
-I [won a stipend](https://www.linkedin.com/posts/activity-7350417081953079297-FBY1) from the European Summer of Code to contribute to an OSS project - **skore - the scikit-learn sidekick**, by [probabl.ai](https://probabl.ai/).
+I contribute to an OSS project - **skore - the scikit-learn sidekick**, by [probabl.ai](https://probabl.ai/).
 
 <ul id="prs">
   <strong>Pull Requests</strong>

From 6b9d359bca2b04d4e8d5acf52dc634d4a76e4656 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 29 Nov 2025 00:02:17 +0000
Subject: [PATCH 629/638] Create ads.txt

---
 ads.txt | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 ads.txt

diff --git a/ads.txt b/ads.txt
new file mode 100644
index 000000000..2f24c0cf6
--- /dev/null
+++ b/ads.txt
@@ -0,0 +1 @@
+google.com, pub-6334831731501125, DIRECT, f08c47fec0942fa0

From 2c48ff844cf3038a1f1d7948cf5cbb59d4ac9fcc Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 29 Nov 2025 00:03:27 +0000
Subject: [PATCH 630/638] Add Google AdSense meta tag to default layout

---
 _layouts/default.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index 5b00d23b5..ffed4a32f 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -10,6 +10,7 @@
     <link rel="canonical" href="{{ site.url }}{{ page.url }}" />
 
     <meta name="theme-color" content="#000000">
+    <meta name="google-adsense-account" content="ca-pub-6334831731501125">
     <link rel="icon" type="image/png" sizes="32x32" href="{{ site.baseurl }}/images/favicon-32x32.png">
     {% include analytics_head.html %}
   </head>

From 922048438575a516e4748719c749c094cef85d2e Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 29 Nov 2025 00:04:44 +0000
Subject: [PATCH 631/638] Add Google AdSense meta tag to meta.html

---
 _includes/meta.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_includes/meta.html b/_includes/meta.html
index 648f559d4..c2dc23aba 100644
--- a/_includes/meta.html
+++ b/_includes/meta.html
@@ -1,6 +1,7 @@
 <meta content='text/html; charset=utf-8' http-equiv='Content-Type'>
 <meta http-equiv='X-UA-Compatible' content='IE=edge'>
 <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0'>
+<meta name="google-adsense-account" content="ca-pub-6334831731501125">
 
 {% if page.excerpt %}
 <meta property="og:description" content="{{ page.excerpt| strip_html }}" />

From 7a9b9b24df9a05386b695bba0db770ecd9bc2e2d Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Sat, 29 Nov 2025 00:06:48 +0000
Subject: [PATCH 632/638] Update default.html

---
 _layouts/default.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_layouts/default.html b/_layouts/default.html
index ffed4a32f..abd17206a 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -11,6 +11,8 @@
 
     <meta name="theme-color" content="#000000">
     <meta name="google-adsense-account" content="ca-pub-6334831731501125">
+    <script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=ca-pub-6334831731501125"
+     crossorigin="anonymous"></script>
     <link rel="icon" type="image/png" sizes="32x32" href="{{ site.baseurl }}/images/favicon-32x32.png">
     {% include analytics_head.html %}
   </head>

From a7ea8573cac14ae42e54f6b02e2903c24b513457 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 8 Dec 2025 13:56:19 +0000
Subject: [PATCH 633/638] Update ads.txt


From a1fdfba9e47e7efd7a00c7b485e383912b215e4a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 10 Dec 2025 18:03:06 +0000
Subject: [PATCH 634/638] Update search.md

---
 _pages/search.md | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/_pages/search.md b/_pages/search.md
index 1105371ad..908d44bda 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -6,7 +6,34 @@ permalink: /2025/
 
 ### I will add bullet points in descending date order of some things I am doing (besides applying for jobs) in 2025
 
-*Updated: 1 Aug 2025*
+*Updated: 1 Dec 2025*
+
+## November
+
+- While helping my partner I realised that Ridge and Lasso can be derived from Bayesian modelling with a Normal and Laplace prior, respectively and decided to derive it for fun ([post here](https://www.linkedin.com/feed/update/urn:li:activity:7384968272858791936/))
+- I saw that Stanford Uni updated the Computer Vision course taught by Fei-Fei Li so decided to re-watch [that whole thing](https://www.youtube.com/playlist?list=PLoROMvodv4rOmsNzYBMe0gJY2XS8AQg16)
+- Started doing some of the introductory cyber security courses on tryhackme
+- In december, I will try to do Advent of Cyber by tryhackme (in addition to the usual advent of code)
+
+## October
+
+- Organised a small Discord community (~30 people) to follow along the scikit-learn MOOC. It's a great resource and I wanted to help others learn from it as well as it was amazingly helpful during my preparation for the scikit-learn certification exams
+- Read The Vital Question by Nick Lane - it was recommended by A. Karpathy many times so I decided to give it a go. As someone with super basic chemistry/biology knowledge (from high school), the book was hard to follow at times so from my first read I learned how unlikely it was that smart living creatures began to exist in some hot vent 3.5-4 billion years ago  
+
+## September
+
+- Attended PyData Paris and talked about skore on probabl's stand - learning what users like/want more
+- Presented skore at a local ML Club meetup (which kind of prepared me for PyData Paris)
+- Completed my European Summer of Code journey with probabl :pray:
+
+## August
+
+- Now that I am in a country with easy access to English books, I bought some favourites to start building my library
+  - Mathemtics for Machine Learning (physical version of http://mml-book.github.io/)
+  - Linear algebra: essence & form (physical version of https://www2.math.upenn.edu/~ghrist/preprints/LAEF.pdf)
+  - Make it stick: The science of successful learning
+- Of course I re-read them and I felt I knew more compared to ~1 year ago when I read them for the first time
+- Continued working on skore
 
 ## July
 

From c34283a72138419d5a1bca4429ff754a96bd27d4 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Wed, 10 Dec 2025 20:10:05 +0000
Subject: [PATCH 635/638] Update search.md

---
 _pages/search.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/_pages/search.md b/_pages/search.md
index 908d44bda..3656c0ba8 100644
--- a/_pages/search.md
+++ b/_pages/search.md
@@ -10,7 +10,8 @@ permalink: /2025/
 
 ## November
 
-- While helping my partner I realised that Ridge and Lasso can be derived from Bayesian modelling with a Normal and Laplace prior, respectively and decided to derive it for fun ([post here](https://www.linkedin.com/feed/update/urn:li:activity:7384968272858791936/))
+- Started reading the book Advances in Financial ML by Marcos López de Prado (still on chapter 1's exercises about using the popular sampling techniques - tick bars, volume bars, dollar bars, CUSUM filters)
+- While helping my partner for her Stat ML class, I realised that Ridge and Lasso can be derived from Bayesian modelling with a Normal and Laplace prior, respectively and decided to derive it for fun ([post here](https://www.linkedin.com/feed/update/urn:li:activity:7384968272858791936/))
 - I saw that Stanford Uni updated the Computer Vision course taught by Fei-Fei Li so decided to re-watch [that whole thing](https://www.youtube.com/playlist?list=PLoROMvodv4rOmsNzYBMe0gJY2XS8AQg16)
 - Started doing some of the introductory cyber security courses on tryhackme
 - In december, I will try to do Advent of Cyber by tryhackme (in addition to the usual advent of code)
@@ -18,7 +19,8 @@ permalink: /2025/
 ## October
 
 - Organised a small Discord community (~30 people) to follow along the scikit-learn MOOC. It's a great resource and I wanted to help others learn from it as well as it was amazingly helpful during my preparation for the scikit-learn certification exams
-- Read The Vital Question by Nick Lane - it was recommended by A. Karpathy many times so I decided to give it a go. As someone with super basic chemistry/biology knowledge (from high school), the book was hard to follow at times so from my first read I learned how unlikely it was that smart living creatures began to exist in some hot vent 3.5-4 billion years ago  
+- Read The Vital Question by Nick Lane - it was recommended by A. Karpathy many times so I decided to give it a go. As someone with super basic chemistry/biology knowledge (from high school), the book was hard to follow at times so from my first read I learned how unlikely it was that smart living creatures began to exist in some hot vent 3.5-4 billion years ago
+- Read The Social Contract by Jean-Jacques Rousseau (more books out of my comfort zone :D )
 
 ## September
 

From 4cc32337693ead72f24bb7039a8da845872c43d0 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 15 Dec 2025 23:29:07 +0000
Subject: [PATCH 636/638] Update default.html

---
 _layouts/default.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/_layouts/default.html b/_layouts/default.html
index abd17206a..a8b8962fc 100644
--- a/_layouts/default.html
+++ b/_layouts/default.html
@@ -41,8 +41,9 @@ <h1 class="site-name"><a href="{{ site.baseurl }}/">{{ site.name }}</a></h1>
             <nav>
               <a href="{{ site.baseurl }}/">2024</a>
               <a href="{{ site.baseurl }}/2025">2025</a>
+              <a href="{{ site.baseurl }}/2026">2026</a>
               <a href="{{ site.baseurl }}/projects">Projects</a>
-              <a href="{{ site.baseurl }}/esoc">ESoC</a>
+              <!-- <a href="{{ site.baseurl }}/esoc">ESoC</a> -->
               <a href="{{ site.baseurl }}/about">About</a>
             </nav>
           </header>

From 37271a11150ab539d68ee8ef63dd7b247eb849c9 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 15 Dec 2025 23:29:40 +0000
Subject: [PATCH 637/638] Update archive page to reflect 2026 posts

---
 _pages/archive.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/_pages/archive.md b/_pages/archive.md
index c29b9f556..cea018ff3 100644
--- a/_pages/archive.md
+++ b/_pages/archive.md
@@ -1,9 +1,10 @@
 ---
 layout: page
-permalink: /archive/
-title: Posts Archive
+permalink: /2026/
+title: 2026
 ---
 
+# TBD
 
 <div id="archives">
   <section id="archive">

From 448cbcfb080d116e9d1c4e9f88b5b38ea22c7e8a Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <54508530+divakaivan@users.noreply.github.com>
Date: Mon, 15 Dec 2025 23:35:37 +0000
Subject: [PATCH 638/638] Update archive.md

---
 _pages/archive.md | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/_pages/archive.md b/_pages/archive.md
index cea018ff3..037cfb3ed 100644
--- a/_pages/archive.md
+++ b/_pages/archive.md
@@ -5,30 +5,3 @@ title: 2026
 ---
 
 # TBD
-
-<div id="archives">
-  <section id="archive">
-     <h3>Most Recent Posts</h3>
-      {%for post in site.posts %}
-      {% unless post.next %}
-      <ul class="this">
-          {% else %}
-          {% capture month %}{{ post.date | date: '%B %Y' }}{% endcapture %}
-          {% capture nmonth %}{{ post.next.date | date: '%B %Y' }}{% endcapture %}
-          {% capture year %}{{ post.date | date: '%Y' }}{% endcapture %}
-          {% capture nyear %}{{ post.next.date | date: '%Y' }}{% endcapture %}
-          {% if year != nyear %}
-      </ul>
-      <h2 style="text-align:left;">{{ post.date | date: '%Y' }}</h2>
-      <ul class="past">
-          {% endif %}
-          {% if month != nmonth %}
-          <h3 style="text-align:left;">{{ post.date | date: '%B %Y' }}</h3>
-          {% endif %}
-          {% endunless %}
-          <p><b><a href="{{ site.baseurl }}{{ post.url }}">{% if post.title and post.title != "" %}{{post.title}}{% else %}{{post.excerpt |strip_html}}{%endif%}</a></b> - {% if post.date and post.date != "" %}{{ post.date | date: "%e %B %Y" }}{%endif%}</p>
-          {% endfor %}
-      </ul>
-    <h3>Oldest Posts</h3>
-  </section>
-</div>