From a87246a80abaecf6dd037407820a50e014dde614 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 21 Apr 2026 10:45:27 -0700 Subject: [PATCH 01/58] init 1.0.0 --- cassandra/icon.png | Bin 0 -> 189438 bytes cassandra/versions/1.0.0/Chart.yaml | 0 cassandra/versions/1.0.0/README.md | 0 cassandra/versions/1.0.0/values.yaml | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 cassandra/icon.png create mode 100644 cassandra/versions/1.0.0/Chart.yaml create mode 100644 cassandra/versions/1.0.0/README.md create mode 100644 cassandra/versions/1.0.0/values.yaml diff --git a/cassandra/icon.png b/cassandra/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..653cc6c6a22276936d8d19ef6e75436322e9749c GIT binary patch literal 189438 zcmeFZ_dC`9A3y#)W=3ZAj55QKy_F-ILdllB_nt>K;U#;IjOuMXLK zo~rlv`u_C!3qDuZWqUf0b-&-%qo}9q3b*lT@c{t1t)vLk1OROCLrjBPIN&?L8{-l9 z7R&098u-2@p5Wru&3Bp26gAZVz?TgG;GqC;27U>?4geld0N6AE0P!>cpmNS?ekKWi z1IJWF0R~*({L5=ANd&)w>#n3GkGqabg3rfoHFzNb{xYBhd!+3>vpw(Stv`N>yT9jG zbx;3SOGNA5|#i( z;OW1=p_%vL!}rkv;)IN?t-VQPksB4y43Cw;t-B);SU}j{=d<)(1a?$6FqtJ-T++ZE z89VM?y7v70-s4*?kH38RVg%e1yX#BPu_#j~!ek8o9J6(X`1-L%;zyoPk_`o40RLuL zBi-fd9f+|JZz&2y(!_m!eR%o6Gp~WsIKjrK7wHweUiwSXJSWTLZm6OR8F1&%RDW~zGCp5QLb*Hww~&!8gRNeOWx z5+|BIS`F3T7lApM=)S0#-7y+h*(zt!yQY$*6u7p$Dv+ODeRK0`ZyDi#{t`=o_2oO# zcM2VBG%mLII3eVIqX$VzNsW0&>pp1I&Q{pkrn~heEnE^vCPGVTz77NDQR&}27KWKH z?_)z1EC(}I6&p6VG~Xm)OVbomodRu64H`P z!I2vOZ=_PZ)PzLgnu`yr-=}dlxooCO2Q)uIw8gEoPV2|@2@c*xN;>Zr$hY{7HvKxk z3^0H>0o0fPmZYJP(N+7OM>+j}kBUEKjOM$ScV9qVDzm&Susp3no+Y8jX!JWv^RROZ z)2`ZDX<^~L*`l!lW^qw%zaZ7ETopXZ$B~4MeE*xo9BuOqCM#6TSLhWF_-bk%lv5-2 z(obuQ+*PUIO^%(fv0Hox_|gtqSe~$e)K0hZxYXL7(6j;!{ef&*$jfR+$hXq>l`ekK|mP<1sbkr|N z@FST1JSzRs|27ZfJF^-;>h<#AASK5v*RMrmtU%90bX(hM-dy;B3N$smYnh>KH7)wV zmLpEPdv`Il{rTGc`}Z9=d$M&jHQ(|yF^4J&Z9q8w91C9W`|X1Nl1FOIKSO$7vDPyXye_eZZ@s}#EQ0!}bdUb6gStZ~`GP7*I`szSvKM{xi{WXDAL{VusD~g;Q z)tGc9lP>biy~PnHB=J5Kti(z4E%u*THe5lUo;?Z--XlxZJjSFZEbz&HY900SYw61C zzC_%I!w&#P00}!jipKF_-e<5B?t?mRQS_Q+EMgBY#@oq0l(F`f9Qd<&ZFcFMJb%w~ zQJ%;D3|%K06=df}ll0*Y>;BtY>ntq&sRrwk+$1N2a18*c3)BXe^x4ao>ru@H*RM?O z0VMmjK0c>2sxrJ_H6VkDV0ss}w))%kOg|np^0=YT00Sj~sE(^;4E;T*}DJHZ?VY zBEf8N*MRmQjK-y<#fDR(zVxEPu*R|=giFU~%(BF)AQ>s`Rdy%!?OQUPn#CYCKw6%z zDwSR$m!99sf3L#?>Mbm=h|i2u&|!HZ)(lTGmi2e+c?bQF6P(&=w1?kx;#JkgVCNV} zCY$@-Ve#&Cf~Q5Gs?w`hj%w)yYv>H0Z{ zctUmYsXoai6X|@t_rVXnzJ$7F-j=A-f%FlAwqoOsOj$uvy+v*-4fo`KYbSmUc)Zfp zo$gmva;jOB362b`hI{#P;9>RpP|J3b&Ki|gRJ{59yVkYyXCJ1a@@1PxRhO3Im-Qb% zevA?>2>}7|TqFlg*AJUT%&UgHSm9{MRKLqZPe+kaL4y1vI$&mN{Nf>bm~ZW|Gh6M0 zEqp)(<9(diHyMpEGu{Ue4)0J=`3c;uzN315YDHMVSN+~w&Ng+u2VMSm zS>L~X`_|r)F-2*}Pp>orXv{RLVcc-hG0%_zhF29F%?hY|{dCiR9LL8r8}QNAUruV- zSIGhJ#l1bS08SEBo%n@Tdc?+RSpBNOzPz#l0%^d~q`L7rZ0q5tH(X0R0OpL(040VI zz)TS2&SuQ6!6=Ws#>uFAk*Pjcyqs3}l1VuSmCV(?NGppY< zHn@M{H1_~&H12K7iNd3him}>XzkXr59_^)Z_r?#6@J>GHrjpQO7(L6^T5jYSIHQFR zjI=pwWC)hZVrua38$7|X{M1f>pa#_XSADf@)7FXb{z47+Zz6{E*`%H2oQdr5EJcnu z0E@CuMtnN!j$#6MJUggFLOZ@H4_cx+lEE&d{zdpn&d@OjTxG`Fp5lbwvk!^2BU> zlPJjtgg;-}XnTouO0NJWf;z6JY z29pRdH}&(sQRvJf0Ju5jIcpz+g#RJ~E z$1caiRznBkNY75hATQRYKU#@cZFW-VPVl9lE-6oB7*u6G+@Zb*yaw1Q{csVXzlT_M z#^TVeZ1|~tpP3jebth607nKuRbGZ4@(W&ae=RZs;kAiM}#dzkUH)%Y2K_n1FCM$0v z|E0ivNN3`#Q||eQi}?ZZu*2Vu0Be4#c--Vrcg5zL$8MNPE1q%$_OHex4SgR;wALI+ zJAmx4W^eqyg#o`={uZB=9xG1i$z@uOqvz%)EL4WFk49T@5h61_0U=!%ip7iuH8w_t z>DT`IPB>RuIger>pC$&bx-W>}ZW{Z-IOq`Gu9sT0L^=bNrCmOLjQopU9K=z93$yXl zd&e#LN2)*<1RiBsVy731(<1P-Qd_`@CItJ`Akq6ifV3W@=NRcbc13~iH! zy4EP+@SjX(#F07SuO1Bk%%3TB)bg9#Wxqu|dFJJYbO~IXC7!{@Y7cU5bSw8tC~i9m z(A7OI@c1fWb7{a8M6&f@RIvBq{xPetL(~^PUHsw8m}K+dn?gAPA&pfK6z9qw5eztU zUx~yDBtT?Q~J0DKYy3R~lV<$QJ{Oq_; z(CEVK>~JF-BS%!!%c;CYBP`4uX8Vt4**p@6w<_S@NGiTfCu|Ne85L9mi)d1@8zW^V zO5WsZnm;o=S-DTSP75s7I0fe#zv`c-8@;#;yjD!rbW1yBx8AvnpV#9sYur^Tw3myY zfigt;Uq&r-Bi=scIP_;@Gy%aZ2Bbq0!nU&=JA=LSgg6>jCTwvpHQ>RN)ID}K-htoXn%Cl{qi<^! z?>SI$O&dHv8kn$(4I!G+8~QJUjZB^rLJF=AOpn<_64Pw-fvzL)6rllurw@z>U0(zS zF{vGjro~?cr&^t-N@jfGkO4#4eEV@g&L<^!vTQ-FEHj-C*t;V4; zxXj0q=)`*N4N6>$L?P!D{!A;7h+S&V!V3N^)rP4gWUWxev{*emqrq+Mo8=9NMX;~% z1S!wvVDc86Bm6H&Y%9AsZ3uBOWNZzwR=Sh=zkcCY#wT#gbalOgpsE>8s`$dDD&;v<`>qr5qqi0>eSR!g%PPsa&~p(fu%(o z#?ot|0Qf0CAmQfbmij15+m3m=1NcScaBl_$_JR>`eBeIoL6wzR4U4BPJo|#H*s`_Q z9(cJ6pbJ@rckZWdmAWHmfBZ15uHRvYc*jlAk0N9W>!Vu@gG*E5}A;P*&)0Hb_GN0jB&lIk#emBMq;M5I! zE1xd}-46?ETBxX)1d`m>F=4BOp)Y<&rm4Tb>X<;FJ^!50Qm7rdq@x-%a>&;-uuh% zA{njddI~4PyPFP#J@qQ?trOvdoPIysstl^G03l$p{sP6#1{gNnsozOYqy-K^LXEh1 z5Vza(Jy0%!BZG6bwy|+)pk^vo^-v4ia5|@o&_E4lHdKSG*$<5xWCU=1?WA~kd8O56 ze`odzYtQod!8FWgqQ8P@-Jzil`B=X#D{Yah^8KwMx3iE=lfcA@>g9ExZuOsl-6aQ2 zPs+&1h=Z%hrYvJ7D%Zsx$nvq#6}>QVlCm7AZO1_|HJq}bD~eQ7g$tiCh%)HO_->np zzdPy};dhF`2#$ZaKfmHi(I{t|sVMlr0V^o0IR^Y3%>HqFeDauCt5}uhr7H4R1uZluPbDmli`~^* zK#+=Axd`hG2?EfpSlb7_Z4|ET;!P9sGJyfx9xq5Q@iz{-o@-WxnY15cLBe`R?^oWa zM-yjFA;&v%9#r1^WKs6&4Xrm*8ym&gGmL(N0jrMaTK=NB0xS3@w@AgGe*bPf3;njY z@e}rL_|a~@T&C<{2rJ{)(Mopm>EU%V;G1NpUkdw2t8fe5Ma!j+*~)^$HX8vYRPU%bb|39GeBA#Sd!42t!|%hPC;u+ZLo1i$$g7b{}+RTqlQj zIsNW$Hr5u@BWIAILM{8Yw+qteB=T;I$Dsj8V!TSpgjdO{-&_u}8hNp$IEq^x%CY21 zobG>7_qq+Cs`v#~HQ#z)-N|KOL%zogH@U9fcmwT_Pe#e=K9qukOc*C8O233`x`J-7 zj_H+O$&e6wdgIK%UUy=$c5ZhZ%YUu$$<;yc0RU%f&`eaPlk$D#RFN7M7M7D|F+YS9 zm>~e_%v9vGMhDZ~D^e^1G9zaU5syj^=7MgHvndbH9@ND1hnZj6DIT`=F>1-9KsEiL z#ea}iEatVe)ZdAkldPCJP15OaKYuo;-$}hi?fsfyxoyLB8+;ZW(%5>X=&3l)UP$q& z3XodovUrtOQDE9WCZ0g}5I66p$NP@m?8$BG!|x=T7`&Uq5v-|pb+4%XvR-blpS~tE zbC?mNf8EJq9WdLspn!}?9EaLR6MyXARsK~?FC)Em(9 zT^E{JKjHK(Lug>b84}=Ff>j$1cl3fc4LN3=nU5`=~dofU3 z#a7U(F~w`z&d)F^ZPB0fh_`4#ZtZoD$&wl{@QPZ+*X&fSE~7nQPfj1rMOGOz^V^~A zAPvM$Y$S$mpIy5zUfv^{UL3VKI={qBvwo4;mJ~G+OD}->;@&@N0!`Z^k-CitFmMfa z_L2WGtGNZMRV4nS*Bo-NaYvIE>ff zJrx6lB)+-vc1=CkGiq1gm1XB-4Mu3_Rg?Fo5-z34$58I=~YO!bQI3)F1MX zL`@lqQA_-M&x_<3an#U5%x4`$?E%*p@-g-7b+2A(yH7-gD|_C7FoE4DwL~1mO4ncW zY!-^(LO^>IFIH2eH3U))>QSq0rGa`WLY8jN$8ibgLaVT^fr4vL-6Vdv>>Uz}cy4kv z)Q1wjS3oyGLw0_Lo+)s#~#2=@AeZS~Y0s{lUI{!Rh)6dtIuN97MDt`fD*+ zGpqiP-`)=ABI(YEPT3_cHs59QP8pp3{?E%^BV0S3OVizw4Q-4tcFxuz%qbJmZ?!iZ zt7F6k(BNy}9&!B%a4cXEO5#Z^N~b^BfX^!oVhAby&Fs`&E15th-5 zJCR1)U(0%)WK>?z5#n5aKf?q{JkGk1-07zl9M84totL>HJ=JbN){D*i0JcMlM^R3t zljH95@;Z>Cc5aJS*Lxv*y-Nh?jlOqnWGf=$bCzWQi^MRZUf*A80uu`Q z6;_qTFe|@G();#T9pEb_YLFs+(ExRhQz>;$u(_&xKp#juzkT^s^JUs>S|a)xhV&OC z6C0``)U(7Suv!Fp?)9g_2Ap4MV|L&!smZn{amTN9y36c>55vJc&W|-RbVwR>wh_H} z(p)$%I0(vc*Wbfrc?3k@r_x19qx7*Ua=tX5f^Ktix(EpgO5!fP@$COXsk!NtUe|xr zccaLCT4}{HQH_w!J7V$^ZM&><=!hgge8OIU%aZUl$GvX_b#~Sfvfk=BN^nslQpLtM z68~3COdwM(mF%j|jQ3IO_<=>dKblYg1b9#aLZfov=C}0s)rBKMunj>w9pgWUY(rXDSm!F{IIU-BBlK$EkeoZSaDGWfmG5D z9fD+~NfOV_Ul*UcbyMNmYS1M4;=_04d;RI)@m?piQ(av@M0B3dN0Yyyw?1+%ph9ex z{Xh>fRR@#*jpPl4qh`nYJzC52~%mdR)sNE ziz#eD3%xc1&`j*SV21`JkuUPQVhhO76Ga>suheK{iwilSi`9lO8p{N7-+BM{t~a8O z62xxlr-AM!UtJ|UF}&XZFC4wnv1n$jbSFNUw12vM{AjS^NJfrfJ3H%c?}Y>Ox?)Ad zB~ZrT8Gi}x==n5Jp-5S<{^;GH8-P^a1I>{b{)nAM75;hoTKV=JPQS*aKjs3gHW@4c za$EM6yrPYR5b5FI_`sqMAE2deLxf$8_%L=N5{~2%AdlkK9MRI~{8ShNQACT-J%Qe} z7OMG|6!juMJW<#x3Y{h&IIJ^<P2!&%7o%=$2DI&U5IJmW~hT%rSWS`r=t`z zA{-1IQnZYO+o@KzZkTjuG-hyL_r+7(g)?ta`h~YdKCRWp9TGrR&=1w;L2zUrrzXf$ zZ(L@U4O&`Se2#_4eqK(_J{%JfAwh%D;P{-Bts|p|6OzVm5w7T%wwe0t|I~+Ik47(~ z;JKb|B7Wav7o@?@XyDZ3RE)}82=qNc@A$j6>p$A;blnE84xIfGM<%N8-($lEdVx8%qIxx|JGv8#H?Ox)tKc=*Rpo;bm1?(dXGZ$4 z!3*vMt>QyF1bq+JpWAo_z6VQ?Jklwj8(hc!TIZ85Im-S@N=m;$0U~|=_EoS?{=}Jw zeBnF~PmxHm_4?iopa<>mx17m7HyTv*DMgld2IdPyNcb!5!*GI^2d8WCItXsjPW)yE zYtOp&c7+jrgKZDaBV&M7T`wRw2qb&ArLucdEMg(UN`BmLH9HdZ%l>EFMiy%6QHe%e zZ)+et7F%z&kY@PC*;CcS`BEU4i!l%ZrtAF<_x$-%>-MCtr4ui(ln5{r`39* zLRY;xE7fMS_$XFLMCLPc(*5*%+I1-7=B18-!CZA|&B1+Ow7IEV3g#|@xeW{qr0~q9 zMjzf-W1cDly`L#4jx8Tc<$z{>jn8PYl}7znGE8v$BS8rACiD7Ztlt8&1f^eyI1RlO zssnWC2c`ZqceOv~$EYA`og9V*?QBTicuO3pHEIsgJ}KWvx;{SDG@85*|K&3h+=-qU z*YW<8g}qB(7kBQRo$mIJS=x)b+nQc`*!3AWPV)|XWj{5Z`~yiV#%R1m9~+F}+jZ%NeyuE&r^ z=G1<2tE=3a34x&QIdOK}J~yXfp^UvyYcp!dkj7LSm$t_j3!5-M!!9rU0Bzu;^h?&O zrekP7#%M9r)s?D!|Kz*i^{u>j7MoE|ImTAjI49Ry5+P`F9Xho`x2D+W9Ol1P@x6xS zXPQ53%c_SYsNXZAK-JOyzYW4v%|-0}cvzv$XisbhD@*yiy-C=+#3U(mYz&d$^#;5J zrxzs{B%rb5YFg6(AxtHkRYI;rt(TZw*(46FVX3)Mz@cf}Ce+-3{gH2*zU5HP{M>N3 zubKKc(JzwfrbQaJ@LG02$UbAu6{ZQ;W{U ze?6dIr|;8)Ey;*eA)u?-@J{d=qBSLo9;5%i*C!xeGU`QSs|r@WvuG{B#tdfiGEWl{ zxXz5SWbM+?_Rh8hfQt<&A8B}?6=P8gzE_;w9606yf->-B-}2aS)k^AmPlTaC8X%1`Kapyh1 znPoVaro>>TO$HMSiVFDYb$$rtGezkAxwAk% z)AwN_W-vnRj`62+bNayxf~M!uIV}G+%yR*>EPdQ{ti$Cbg&1!EJBZiEs)(?8xwyn* z7}jJWac#xTKG1&he1mHzhARnzx)u^7K_C+#}lxf*6$YV z&-`<{a+=?~XQ;zP&Tw$yKvOf((RoW|Ded%(<8t(yNg#j}(rs!|@q2hTkxh;3t7ns= zCCZ8A83FCX#iY$sLQp920f;g8-LHR|5R=a1!t=+znG=VB>ZEjoW;{PMJN=prKSIFt z{;26_>sLrlUSeuy+PkAS)`HE@Q9kh(%+BO+KCgD=55K8+Ds^UbPe!HQeoMePGBsHG_J`+NeenDr^WQS) zHupa>1i=gkjW`Q`bVnLlRi@gS=n@m5wZRtq06t5jIHmlN(0+dvC7t+V1{QZ#xGLXyk@SGR;>Fz_*%2KSizeHA)%7W z>+tobt-g82;;W?w!wU3ux{i@FU-Cw_xQA^HL+lkXZV<;tT1(fmv&Qd{d)Aj6Z~^iE zO4Kq=MO@`brIcQHlixTQu1vydxb@QRZ)97CrwnuXYAb@=ix0Ms&m%w?fM7mzWj7Pl zb?3*4`KCTElviDL6Hl+|MgGLO27vMBTjBiQ8!Fe@F8#b-^*FVMlu@V@!lTI>bkjv@ zCWaoBW`Ku{G2L8-`j&xZx+ON8Jpr*~+R((`K@`Ymoxz83a+_@IATgaO zSd$JSOCH9OplDV*;*^1kAS~k2t6rIvs=u6e*TZX`|3ywnq=VQJ?dz|otPeR&=Qr!A zK>h?F)PxY>#m5+?YJP*N+`@-#VEBm_>@D?)$32++tIcf+11UCFga=Hx5}LZ#<6WJ~ zcMJc@aBr<{ER#9$?%0*KYqIh6%fJMi928jpXIg#G>Y_}#F3eC}^0rWKpjINhCFzVb z?qy}e>uJM=+ad3&rfhz`?_`#GlNJuQrcR3>d&NN>E<8C&?E5Q+&d{%+*l~G`!otnD zSZt?pM?%B_ktlLW|EZ;jfknMc3GtLWnnQQ^i6Uwf&SH*r!QXA-&g8mAE*s{gbb4RKf@Y?8K<>a5AVA3A#R|0 zU0`L%WDpyQ#-4gLon*@XIL9i3$_7<3d|Jv&)cmfl&Rt)bPw@8bYYgY-R?_d1&h%|; zY}DdW0?ux@i|s?RQ{M09G+`Qvz#ob7Et`I6e&vdLF9wJ5&vu$dVanJXV=$}1K@f30 z`J`)TP54c&D6*qOy()T^v1_&YG)D(zAh3M#!yYGmr@S8bQRmYPz!Vsx{b3FV2HzAh zwT6i>worKp`Dtbx$5?eh&|QJl$4t4G&m-Hqb%rcr9!-0UYx(jfE^bL z4i=yK0gLNDdu?ksxAwM6{d2&mCX~*blHpAW$#CUIZtktss<*yc7gsI>ED1+%$zA^S znUN2(QdsqBPdB+0d-QOT@2I_-z6z!aKog^0kh)R+=ExN7|~1R z%Bk;E2+y&(tsz5|38HKXi0rJ255c_90ruYVGWucq< zDL59m2TUpdlaiu0MoZ|PKYxy?R2qXvo%*xqLJasO);Oj3P)KNOc`@ixs3*ry`diC7 zewOE%?!NgsiGyQzPG#nYSGri-{iwY5`r_W9M{aUbsP_G&J5Ppn>0WMumd+b7gOozC zUw>B99`*@X0tUMBpMKW) zg$G*Z-(m3>UnNY@fox~?T)t|aEz3-Qs2)Rm1c}rS>$XCcu+D1OP4WMv=|l?suqC3I zLo#@+uWXUhrSNozTM(?mVvN2(-=qX~h08Pq^?EXl`pUhp zs1Q#$KL({|kc?)*iJd?v5Cf5?ZOn9QYNEOrHEg~U)Stu|0urBG?BOqXi(=?5!U}94 z#9GHjw3Q{Hh3|&7O-^2iMFj?%VG1A@iNd<*M%xHMp=}KaoQaamCN986wn~z3Z6|SR z;edd3cD+Zkp}RFY{f{ABs_2%m7$hw)^q9)Ghiw024*q+JbW{oTi3__l8R)Rb%N72n zCAsAv9DHu5r;#)#n(f0?nyCtc#|>pp=HA<8BjDGAfOo+=PD#_v2=tnK>%c*tmh0ll&E^ z4h56PND&mgQc^q)g7^bK^!xi|9-L&2&={Kx)(l#@T%2e6ZJCTbVAf1{e!wZgA|N~6 zX{#G_$PYs`ZCyC-^fwymfyFVNA*SsSf@Xe7jhwDW>gebwMk@id9DAVXa?xflo8Br7 zWPWXQG?gz)^^K=w8A#2*WU~YvCjMFVRu43F@Q}m;Xr711iwF0Bn;>muXbb7nYP(B1 zlI*)pcVVax@1AYH*NbO(zGWl7vE2#a___U(PFf5Cex2Qh87&50jAmZd=MsJ@DXIyy z>bjcA#PRlDAWFxM1yKHPM- zNCP_D{HwOMOtrim?Bi>in?V4Zmz50iLex&G;~}=&rf$l3ZkjPJw2)irQ!pdmd+_s{ zq@_BA*t-??4q z;&CM~a|xz~9zc!14;qH$jp@(0a3yA=ZUO*ivwWDMA5|oneM?#BiHgLl_1Dl@cqXds z(w=!@w3kJ5Qkvekp@X>s&UC;COh%)3OI!TEzip4~lH|9KWqYjea%<4z$T#&WO$Nkl zt`VT*@ffKAi1H}eEGQussUsl(#P9L){s?Sal;l>H4T;njWH5Q_&TnT?dXTcneVSMR zE!4a7>SZ9;!0={-myk2rVWH&+wZDOBSl+Jw8dp*&B2;;EQ?4FBOc!4BV7e<8R2&V| zj*c|Qhm&}B;P1uC1<>MbPeE$32pEi%{=_)uF=}m8=4~o{X09vp5ZyW6w@487_~SjD z8MeQ`-hG-ov(YJKme>44Zr-=Rt-f=<-1E=Dr*+MqxUS~RahejNn>i1^yvJEJd*g~hfj5Ng45{2XIcT5U6tGc zUNv0%@7#P%AQ^c!f!GLQiPM_fHEn)QDFY75PrnN~nW1tUt{6tMWsFPi{FNnt6mq4_ zpO>4hi6c&d>~iWg=qnZzVgO!z`RQb@t-)0Ol)mr5UD(Vkl+@tx`XwG7K2`{M{?R+& zn}%LFr8Q(14Vh2k*(o5)rXp9f(TgejeM#%gi z(s;Xv_xBUCnk}uI$xahey8vVM-iJ`QjDoQJ6mf!m+0BduJ26$wxNC^sDNCxb$^Ba` zU6PK|4O>}#i!r-~kbxK%=Vd!+$S=kWWXDlZ!Jyi$wksCPAz%8z+a7Oy9!Cl~aFxxb z@*2*3&o-?4mg*&C7a#?Ak!jJHRqmM=jN^(}n>>FHVmmKSTiuCqbugHHV!k)7x2PVT zV1;?fuv7^qBWE|<#d!v3yW-Hoa7gwUvs>H5*{fAohXmI#>~v+iA>pBgMB_vcpLkk zgIaLk(P@s!QZuXY-W#!RO!vLi|GG-uoc3%V`ArnuKPPlbx6I@aaxftrzTH}!dm3?j zfa;Y);T@H=_K=hwoIb_7F!AyOA;F7s2?>s0=OLFlKpq!hwq5B>N1wVxOsnWs>wd5vE_N4B?KE+Q)&=4s8hQOC zu}p$79|Q)>H>ap_c7PP~HXL?j@B1cA#0FK=XK?@1SHX@hWP`h#2+@y6*)O@!Elv5$ zfk3Caj!{8+duwZx>ld{gFa#brQ*Vs@)E%kf1}6H+-^#JG|EVVhslIjs#38fk%1ABh z5ASDkerA2JGM1`$dKt2_^@0JEV`gh^BOj!5hJMvP?}P4cY)9bV6m#jthIfcqlikeM z?hi4Ewfqy}Lm=3d>1!TE3eTX_^n@rs#6_Qa;NgATDf}Yaer~RjveYclVOkR13tq3_ z^}hfFWH5?6!{zdCr)zqH;(!1Uq^1bLM8SQ8$#=?cMNHn~AI!B{%E20oyf&mD>R?p1 z2n17UmP;zx8eC4ob?svF;r+*KVCbO4W;T#qcA1|~z7N_SdvC~MTK{k0Z5=yqdmgt( z8(sqWJ&ns05qJCKi(W=ez@@FdVGG&pL{)da2lq49PU*WaK6L(2pw!;mt@bmSWH`-0L%SjG zC>P?+<*bCvOk3~3>NYq(=98u6`R`Q*isZ}O_!3j(=_|Fmp^rg9qFViqpojQhumqVV zB2DDuL_Al~lylQ#Ma>W0wH0=V3y$VR{rVm=YLU6L&ktXkXQV!K_DiizUEdu6)h%#E z0KQ9RoCZ8<2G@J7akAQ+#)d+aEC*y{BzQ%teLk*d~L)TCh>-fYyu;KTgd8K z*r3-TNw#-ULPbSILVb#EP?E+s>A7mKH^GsXNFF}pl(LKZ2U~=efmJFt?7sS>Qn(wz zDOt@+U~EgTBfP$Z|D|2yJh2L|N7lc2aCR2uODiE=YH*mvSYi{Oa+Ro9mgT9h(skjO zYVa7u7wp4tdmO2ydZ17(=c&HyvXdJEd4zLaX0umY>~#5yxFvPS&+9*i7|W7}0a$fF zV@y<>-cksm*&fB}Dn${)oW>RllZe2+ko%`K3Uo_`wNz3WXi%0br@F0!@(S0I6hPkG z7z2jEIp7&1PdQk9t-<8_q?oi5>wWVGJ(*htwivV@(Z5pISO;%{p`auFxva@S7u6k~ ztEJt8L$c2W4>aU7E<9=j6PNm{(V5;LMCn9+1b%+<7$4EQo|o5V6&wG@P)Crl$TDmF zZ3DSwQ+P2X4P#1f*7uHOWsMse$G*(q^tzrbpW8fu4Mnu*H1|d@rWv#30 zD{5Z)sTKmidZ$kCK-7r00=TxGevmiY!9T}qCK?2WhubcaZ0jBT7GfEw&%m(-fY(57 zp^OU>y8_~Ca)Yh%wGC@^!gmNkcXX^s+v{}@5_wGelq&X1e*#3R?!mwrnScx&3;UYd z8%#z5d2_qb=E!GO8K^L3Ukaoc-c+_4B751{TZQY_`OO_2Tf!B2X#8IPTGXja<(Pld z44ct+!H(D7hRzMyr-#juq@i3HiikSw4^}X*GKHyYKdRxsD#RN+g9`UHwUB2Ie5kcJ4>&5ME zxMfZIl^yI57w97Uudw9KGbx&$*H1a_s6o?68>P;VN9ax?=dQn%U7Q7 z2N^lek9R+#l2vMKzG&JTeNsj3l0S|_zQ98TJsb{xXNq_59Ihgl|FoqubONz>SR5Ws z7q$Kdb}u|l#|DMn-C8F|uO5@{eiN0oY%!R}zv4$*0s{GC!Q9+jDXAqs`L{q4je}kR z-eB$g0x++EkMANk&1$AT!%9HWgU_PU7-LDL`-*x>4(7#f7x#JP*PP(>SA#OD_r$=PllfNEhBn+zV$KlXx*n z>WzKktd@H1eA`nNd%YU0%on`!JG%@ZK)RK1N)6*9M?Y{h^X6+ZQ2A_O;Xvd^2nJuf zFOG_PdYH(bvFNZ_-jfPj0T1naOoMU^&~5BEXjy94-fMm>j*e%l_L|S@0I-Iz#SA+($Ns zBB*BT18^9QFTNhY7i+_&UN+r&o*eVcnA?p8yrawaQ45p!zlrjhjL>huL~2TMrKD#I z7|5a6H?+RIjq{5d zP81LsXPXH1beMt1CdYhm0hg@8;7!lv`8{+fNdXn>lEaAa(XdzO#3Z4af{o8rt1!x8B~61P}kv?GnB~7(lh#8Cl~Ql z0+6a;!}ZC1e`8e_4of){N@q+w^{J|y5SPDc=EDL zFx8ar#FIP)=&zLCrq{TQ`q~@RlhCE|z{aERTUX5T(XfY-l#t*^?|tiU*Od5u9=B3) zx6;e5%LJPqfP-k03ln0R(~l72RJt>~6|SkG^2hU)DYh~CiWtB;+%S6LCI!chRJ5|N zO1*FPWfoQg9{b4k9<0=lPx1O~5guzb}6btpL?D06YQms+C43g~0dRq+vVqy>XOhx{IpuBYClibaz zdtVMoSquheq?rJQ)0NBi?D44&br@Te5jBCjFy9lryPd4-TH0T zHF9f$56|p!1b%gyx%;7!-rB>ZrpH z_Fw<{6)TlaI3VbRvtN$252}1f^Ujc zTWD<^f$d0wyn-0!g&84H$@ZQ6sqMyVYy!{w0(;GAWBRGszCjg1`z-VgXd<%u{S;U{oS{q&xc7SsjG;1H-9QHu@-E$`yGRO9q+GoTk>YIJo5>iz5g3o9f55!sKFa=zK+e?b{{-n(Y#Pl zuCX|BrJ3LcYixtACM{9WcXDGe67Lh|(1B&5UGQ58i57xp-UO?rV{L>bKis4adbTc! z;REDtP8bX0NIu4yys?Rqr`FHCj`eRSJ^GZ<0yV_OZ{yK65Xnfjjb)-#uvk z*GK3Zp4)?i6t1W1H=@o0gpmc*1S}*g(${PXzEoH1PqK)mQ~%+}!{u!{^P1Ln9zvwj z7(I9DJkfS#GVmi6t`C7q)a6|S&Uuy%<|&}6Y811=?N}@gklZpDZ+>kb_}Npr4s-?~ zz!w8uJw3fdbsxi<@X2ufwE`k3bT5ko1El~?w08E2sKxyJ&P@i#ba-7AM>?zS;jITa zX!}8f;Cnn5&**Qx{693E1yoeu+qMTrKpLbQML|GHkZzC^5b5sj8X5+WPT@y)2}pN$ zDlN^>-8FQ+2fzQfmgoYOOWgaMy`Q-6>mhwE!lgFC?Bks+MYi&Wc(infre~`j9av8f11N!~xT<+#yLqY2T=S(&jI9wbo^~<;xjtZ1+;{Uz$HVfC1I6BcIC?NT_$XmXwV=wY8WyG@oIRCK( z%FgC6p8zO(Y{eZx$;INrJnTnk+SbpXH-LsF+{LWmr$tmn?zkE%H&>{J68hF1roQ+4 zSJHnL7&`S&tV=Jf^hXZ2%1AZ#QPhi4{@T0MY76t4J!_8SIc~Yx^>B*x%c> zlxEdr@Y02A7|ji{Bjl|qCi^&s;Zk*T9Yp$L8#N%AMS)v^Zt1AYGpTbHg9ZsiV?7uy zRe}GJGasopuCM_S{Lvd^kT}gK)4d9g%?QBCncXhY7U@Whi;Z;-$MMo+isTO%{#B?>7t2 z?qTx>&R1;x#$F*8429|c)!0{G2@rUKHY!c?f~t&%wbgHF?i9ncGq0V_B|vIDSwO$~A3jX!o1$*8b9ZVcGFySpd4l5z|mBVJ2Z86jd= zBf!}KZIUb_!!(`8)3Y~CPNo%2U-ZpcN^ORSun|Oe>06_tb5a$epAfE-YtS8R$SeBW zujF_J$rPr&I0oSHd>P!W6Q{KkitAB`o7_G7p0vY|c#<`AcZ}_DPlm=7P*nGnS3wuivZ@Jk_Sd zdw2BwsU7Tbl75mzL6&zoEv)r{(z-gA+NRsw0$QJsu+(}=)d(yq#}&zU^UT-gzo}ln zpE|7}EprS1Yfm+w5F*%-XY-zqP@$aysg^RJD)*E+nMB?*q&%t?=9No{+*@^;T_EJ4ckxl=r$|^7BJ$qtHDK>SnE!P5O?MG{k^Q8IosR5=PV5i#+XN5eQ zd2gwD8xF;~`O%vf^dm>zU9pqU4*{8K{Qt2wBWuUH1ea4qoYHb&x`2sc+{EuQ*n=u#nH?FWp5 z?~JaYhycBcnuP^hFcX`HhvzHZPP!?KQ?*nx@a)X7M57vc0o*O6K~hl-*Eiel7#Rs( z9}Y~XctSDBll;kuRJwM9X=i(P%*Rbdttb&s*S{XPeu^lVm9PJj)E>D5+H62oWmVUu zRZYM!{U%-^Zz#6W&geQhFc=2?GGnpQRA1k}W%W~wh03%=RGgwoKlvw*KQxu(`a>-H z!G*my;&i~x>9laCpJJ{A6I!6CK+1Djy@4XKO|--PH?K7!rD4|EPGn6@u~PsbKvAe# z=SQSw`8?+gt@ZheYoR79OY(|8QwwNZp`0wfI4t{-^i-5gL93~r(KS`YuJ~LjYOFlv zG<}rz-@C>DeL}8it7-AtE=tHOWei?koc!yk!O#|+_Y#0H?4Ta2)Z<$gvaqf*R+;hn(+cLwxa8_vlxsgfL6N9V_E!7CHre(O*Nkt#ZNDE zXy(s~!_hC!C^fn-@>Rc!X1_xfKuNeDielmXk<`?AuYmdA$MHIQZhpx!!o$>$CV!{_ zUI&)2wCEY7n;N^&7vbFDd(F46c=DTm1NE|r>fqvJW;a>ts5C21ZlzJE? zhq9AeM*%O_!SAYSJk~ccXU>s}j1|c-VnR0&_H3yTSh~hhjV79pN1`jfXfH)*i^DX! zXv9=6xDY!Hc_g0TDwxkY3oAAfUDTqJ$#z-P;HkcKGkbo(X&MpT)Gc!CRG z3LVNbWI8!o#5g{Zv;RbQ*^ga6h?EG61l-z`yO9ikM`IM%^?j~2)zx&-EvwC#>j+jP$8()L18i@=01t(#alj|O?sXI@<}OKX#Ebws0fY`JJoT|9 z(-p@cHN7`zFJ<{1Qq4k=TAgOx6k)o88|IDo{_-&VxO-myF!e;nkgF6NV9% zmA7PJ&RPckw8FEMz}@sC*+U(SDBwh{^rD~hgWiq?@3u|J`v;KR1tyL}HojpFfd5-CaKa z0hHE64%@t6k@F)(a#!OJY2Wdr9kB$#oX%_W^NVnSmW8?c`I6OO;)uIv%=AdoX{wzGm^}XCMT2j;u3{FJ9(LZF9V6gczdgDYOdoypiR%PLh`n} zz>seoNfTi!wbR-&c{!OP^sV*T7O=LY9mA*?-%DnOd`0;|pUR87i#?m4B~Acl?h&+7 z4|??}T8f5zyS`*n{(UVcz%)sdai1qm%LM1V+sqz2|4&ts{x2Z8j9nc*WWvKQR$xC@ zt1BrZZ*(4k6>6Ud?p+?n5Ag)Dy*xPW8oBH|-RSsz*8Ak{RodBRZU2f=0r|Shes#z1 zJ?RoQ(p>uSr6wt8|wj+Y_@fKNw{7DMLh(js(6 z>^)geYJLptP7p;Iu)a5x*QZ?;EwifrPrJDKcud+@x@R4h*c z&f!_d?>~Nvq_Y!qJi&-wkOY<+#v#hp^GEjv=L70At~_jIBEs0hxn?4QA1@iY3^=&x zAjUXxq8z@|Dd<6OQiO2iAyN304qsl650uSCrKRt$*cQqSFvHKz?B_NUjQ}wzofUO% zT^*sPW`KXQR)D`JFwr}jYT*p;@8 z5z0^2<1-~=toO9Fu-L7OIs#aiY(4jOpk9F(9W1uxJikw2g@Sx^+ZW zSM>^<3-G-tvS@KT zIIiB9j>R)?iaFVY3L}IboVC3#rgmn^^$LVwW3z7Wi5onxY!BvJ$$<3`UG#f!%n^6x zK)438+kvr4J+$WVPyxlB9%$Sf6ni^}0aggOiC?e7i+?uCP%K6Rn!|Tnz6d)LcsHLr z)>Cfe5NA)^mb5DLJIlVOp-kNKKw)^PyZ0Z#qlm(U@Po!5A@?%gFP=q1^e*hNmG&6p zyyg#d$OpzBaN@RZcso9yGl$TAgkyi>IbLRSQF&gPmYHeC1jd;gbMN7!<5KmL&d=e3 zZeT4nHJQHiH`WcBo@bjKr(3;GslXaB(lKr44#FE)T{b}4{1EC}Q&*o1Vbi{psp8Ye zyJMrf%R^Mz#P!RZe0_WVlNqzGGM>U=OZ`*I!D)M+PTdj5TO03y6o9$);3GZ(L2Wg) zINcu}Fqzd}rJ32M%cBK{9X*@jj0_5ShL7t$KnS>ipw9-1F*tb7!jg$eosXydj69ld z^Y?E~+JVtuH<=~hY#MPfBw9H%lAjieSFhLP@J`y!iNdx2HwdN>2qt zi5&Apl;MY<+@hzKJ&t!WuklcgJmD3-B~%yPwAA$;pQ^5G5nLQ_qt!nI&FFbzTFsR8 z*3M`LwA|k_7YtHo`RZa-wXECE?>GeS`$H<)O##<^VPD4H;cT@L-{sLfD&PDYIwkU) zkc{b}@O9rp)AAXuFnp9Jz|N%z9(wWhzk}m9`U$o)*9Cqm*;vR=9|lvFy1ZBrz@8_Q zHV!pDA^#yhw;j4=@hT_&rJHmd*Zm!orfn!tpgAVM!xNsjMWK>W_HHW};q7{MVCOxQ zcD`G@DZ8=ppik-THP z^O(Z2BraPwxzE9@`*N!{z_acCdQ3;pr(SjHP};j{@%rTYb%XsRs3CIX>X#Ea3gDCN zTi|i}s-!gWP@wfK_mh_XI?yN3zOlPnT<{H7CuBy4F+gKc&lY^)NNNgtvNmS-GX;ri-fJ@Rf z?;|E)r;|+HQ3O52KIoKowi88riWL3&`d&FG?u8)G>eM&)W`Qc4pA=acm-=GoSkgMy zOxw{KiUa`nD{99vUfMPjmOTxdoyEi5|JsrZbwC+10{)2gaU1ieH5?+F{ zy4vmKRM0)>lDLjS{aF>0@g?qGx>B>Bp3d(!arRkAx4Gh6YjC@LMs4e2FsARnRnegt z6nO}xArK6=-h|wod3xEliT{Q-*w0nc2Qmd_#1Ly ziR{K@3kx0&%{0=8pB@`FT7C!Ki)3xKNC6GEd&SN7H{Oa8t&6q7#ztZSmq&$tM|(E2 zjW*~Io8>Tbf1+hv3rIVnDjr;qxB>xMQ}#ix!QSFmTRQ;R*`K{!jwQY4p7%|<5$d1P z%Q?(nJ<(2UQ2P%cEHd_Zp}%Tj6;qR^V1J?zMgkGrx;6j;ejQgMy#IAoL^Y2QzJ|FW zm;Wj(j9KypJ?DXjFdss3yg2o539rU&QZ%k6#~w4rNDF<*IvPRNm|ooESnc6%7-juY zdO;u{_23?v2G3t`{HpRj6a;rR=b;CHf{aaF$2*34Vx#+X3h-q5G`%`&FBxpyN zrYFrS;TJ!zm%r>KIg0A%M?E&@Zic*XYtI$L6z1#C_@3)o#1?QJvL8NuI-@~FIX!>N z%X8|D%SVmeA%FkyVTKN5-_lq}Vb`hDPSmKyuv&DiSw1b$+%!W^yDVp3(I{I=rYFzr zwJzTu;t))vV^hKhrGVAx_a_Yhd$*PC0@k@5yF7aZFaXWbRY0c@?L2{IgT2PDwq>AZ zL=i3^_Z!29+#CCr3cBv^`{IB^(Bqu{d-tL4(6jFrYh0j*{~eP&<@lg}nYn-+tcmPC z_1V$b_-jv))t3hK<`GRZ!1qhd@t(b#bfB`(GtXhx;pl~;{Hwux+c!;W>{g#AMP<@! zg01XU0NqC&@)O$nRG<$5R^Sp6R636KLgQIhr;soAuS^R99ouaj@Yo5utZ&5+w!1H5 zw~uknFT72TmB)P79hk&L1(6qNQPZ;;6%<4YO7_4A9PG#JBq+d^NEokO*J<|=Sn`0& zDXY2KRSLmUqk}|@#wUHb$07;5G)PYM$$Fm#|CnR`Avw*`+UK;zoEIU{|5jSwyNrju zQgrR9aJt%;M*&^;6U(Q-v0`PH;yZ=rOrJi@U`biw@SRfUuJ=4Wt?ai_ClL=HMyFIW zi(j@K4#1iaC7iZ{rlQCNnzWYfaAJ&b_R@Q5PQBc3i21T{jOg$YL5Ab+FAqxk%51bM zJz<0g3vC3E7cP!*1N_5bHxNv6^GJM-&5f-Q1x0%(z#FDZxhbcX`CM;<437gXaWlW} zc)b1j6C0rPPMIph00Q`n`_JsY0nnO;G4={|woyaS*IyRGa&qpjmuf)IVy?wvq8-8O z<7wz-e%KDrFvfcmfXjEomGQ=Dw6-5GBcr*!=*a(Fwhpp2+u{XLCliGt$jS~^Ud?m9|}P75P5!L=deoo{ov?KZ_-=$W=@6GGM(m=OD#Kti@m8uivysF zK};GP@tK1~a4$^S50i$ngB_#7u{Xdq2p4wKw2g~C$`k=jGptLW z7h~yd20a+&D0wz9kSy}%i({HP3%a)9%Jrwe9OknNuEao)N?qVV;aFcDg~H3HzcKJb z@}bpXT_S=~ph>Ct#kTuPOu4lGq@rN!onyGUtj;}!Vz52zgZ}4@vAR5~`5~a; z$(Y;U1_4BLo=MXCk?RWgb`BS^Ri$nf%E9-hyA2X?Ih-)=1~ zrYMVt$mtd#8q&z4rpjRO*xxZgbO&Uof)VxD z2i8>b7Zx7<=fj-^Xp;Kc?PZZ)ihy*#`M4zjAddMfREaWBrNYEP?BuSTq(MYEkV)}b z9&hZcNu{zV>_~!Sf_Q2~45EfwS2Iab(;m`|*{|{+mg2@jDbUCn7D?aoH!w|giheUE zh9IZhVM{*#%paKdU2`D_dD$I|Eo2Epq4I(@I&H5t?U#wEs`m5BHoiO|{VqZGiKf*N zen-vKj@_3*aQv>YHb8*ymtw0s-Y+gSgZ>1Do*YH`6^l26sc=UpBkzqd?-=v(11V#L z*mhMwGIpiD-;O9X)q)v?d_=c()Bi50FfiUj=$r16+s5UyvlU>NB?2{ATM+A;VR@Z& zphTO63c`o)eSQ0P)Ar&SO-g2E;Z7jz5c}7I58?DalBJ^RRJqjYxprG)^MpP$lfT2L z_X0=a{>JiYKrxpvP&8XNy&ou-y{Q4nlq=J6%>%4uk>9%`fxf_$rji2IgMhPdf>sRB zmf-~=D2MqZc3b`myQ_kk&wspPY3Q+---GMQv2)rGXB)k7hj-EG_)w~7XAGXBmx1JN z`_6&K^gRdXuN}YtpIusfuYJ5|19P!3@>PQ&a3f7!Rn>M?XmCk{Y5kX7X^^M&^a7Z zH?a0bz`%L+ZBnA|d3STYZPj)=KYY`?#@*&UzGmFeF$MreClv?0B9r)(mOv{)PW%Vw z5s#u{qiJG%0ws6VEP!kZ1uEu3xU8=+71B%n+VSAbps&J*A(cjn|9yU=<3m}irKt#Cb1D6YSmx>vNjcbO`8-Likh^i`KRdrR&)a__o zeBi->wX(AE%A@ZCI*`5{XztVUVc(6s?|Wp6Q>rqo(xUG%_M>pH=9!VVitbeBXEm3^ zMnOp@R_5}wjx(Zhy_4BEQ1|r`t85@v({Y(ngxTIsWa5PPwzC5zW4-&oiL7oXD_Gt$ zM}!j9aX#=?c6J~K@&oo^w#}UeVmtSpP{W3MHzfafHC3Q#8UMknb3-9&n6q$4nVVYS;s{qb8ImbfmK@ivus~W-ci0iSc&G(xFy}N z3+b5pYi|zQ7QEs~<5{)e&g~0AUT_>_lJsWg<0H$wyWeKBoGgske`+l42y0`239BS1N4-G*L`!Pv{*n_&zo?Ru7&yD)n0r2>@5yy_Gi; z>^F)tZUn-@M9|OC1CB5lfd9}pZHr0XejcB&8*55}2+VJ29XQ{ehy*&{x`&s>Pj3mW z8Y}YirG0j!U(pQ`9WN%ru*Vl3?Y(XaHr`NB)Ydy4|s^2B^kz<5=y{9t%L%@onuyTPbPD}`^{s8L+ zHhB=OFIy5xZL_Q4RYK&fVl8EJy{e0w41z-oAo|rlIUS`0#Hbwu)guDL_h4u={bAGn zp-m;sFtRMQoHxDco>2@S%!@F(&?S{0_=%MotPtYZ2$V%2uqCd4fEs+k1S&QV!^+|) z4AQ(~q~Pu9X?AQT?fl94GUR7~@L0A`FPl!0>C{YG~h{=j|kAw)PpRqMvmLK> zl@=cZ%e4PyzK%(&9VN%0GU9_^*MT`yxC?Y{QW*-s-46}N|#)*C9RsBC9`6VRQ^P>h&&Agmd8iWb#m z7vwc~e?+gBiMxFb-;R7C9mDsqpR*cd)UaxN6J;|{kNiR_%aDoHkcpZvZ2pu`6D?@% zgXfh)xYy)ljesRHVfgBNO}5D-rGan|b{KJ%9& zCq!a=oDjRk?|`AOGS`yg@_6nYQfaPx%3&{3<~4r{PcWYsCg~@vn+IgdJ5`pKTn$bK zqb-dzm+6c_4a@u1E8RB4Gge#2G|~t^;|gg@?M!oW3E36JW$+>q;pVW-ChXlM*s+v2 zW73TXV8#Q=NC2x#Z#G^)!>@yiTM(a5gX@P3@{v%IDl65%pJB{&8JE&puK8sLH{7sE z0)iFXMc$3{&@^=pUORwD-0Bi20%SgV2=}*1o(Ec-@GIbWRL}%ehbe!hjNO~w={^V0 zKRW$8v#2zE}J1dgs)Y&u7z+wWy&_!$>*n z@0~gNd6vq=xCi6pg&ThMW)fxiAA5t7LI+}g zOLD-x?8w2zsed&sH`{0-P*<)OxfSF=uV?c$2z^5E(qiL^j~xEzkPj3f86_~@Kk}9I zxTRI#_CNHaOkMs6`3Bw!6O9Lg6R_zc8~fF6w%FppQ2od8&d&m8p*tTy+y3Ss*{+5= zCh8-h?d&*K-prN`)}EA5c{y#Ze=^D0YwD!-n4TJ2gROROh|mj06TT>RJ79!FL62Ca ziH1?fJqx1X0?p{q(38o^Lx68{ZPfP({aToD(*Mt3LCf4*`2udm@1A-|Jo}vr9;IvWw?4 z(S8W_)P=BTRkoJrnMESX!3b{|k)D|jt0m~D`PK_#cwh7Qw>@`6`+_qbo)uc+Gs#UF-&&IJR=q!lolnQYa@@~UblY! zILTKtT8^?Qv^}IyS87!``W=E@%-po6i9zh+RwfnAg_1kMTTe3~fFrC|VW9V@TKf<% z7~Eu9i+jFP4$TB=!rU70-!5N1L1zfYgNca)uiHgXioZxCAmw;(aMxSr1Wm_v=1tz) zb1xL#FaU?y0H7y%dd12UTC@l*wL=)suA$HG0)~G0+LasqlbiJQWxj%Ag1iL2DJK!J z!b@m^Le!XE22Nk~-2?8?HYOF7QG|xuq(lgDop=1qz6vj4Gchzgc{_hFhpIpJS@mxm zl9Q8Dops@wn$#{O+_oXbMcVuanyv&*cy6p=3oj+eG>~1s1`eE|({(oMI|l3DG469s z&i8q10yo+krvP|dFf$2IA(a-h)1DiM;rj4yV%&S}k+2rR5%Lle2>CgTkRKl=mB!Mw zKh-;o5g+heeXBVqNrP;6+8&xX4{H>xjv(!}N-e34QelE#few0^Nx9A&Z1$&nXV|#M zHC)M2{=G_1T|}%9YnVW@qt5pxVj4Wr*I}SU!~-l{%aE53L0FRQ+1-ljUbs2K!?U&S z0ubwYXu8V&n7#fJy+#IYytdz}89wDL%)XKBr~Rvl5vHh5Nl9jHtxT6_d}Vezg0Xc# zPdK#XqCWHnET_(@X#m?C5GJJZj70L z9aY+S*Fr(d)}KJ6{A;Ej_5M*GFxrA&q|GQ1^7Yh3cBa@ovs?KX1X+jXi+pwTAp&73en)I8dZh$+@raCqaO35L#OeELudhIYeS z8?3eYp_Vb){vI{~j^a|G)73g7*w3}9kSr?#R7MJvdvT|<3BKN|JmoU8aQ&lyKfhjh?6@P{l?|Sh+%dJ8< zHvSPZZf^lr+i`$-&4$!AkA`5)2KI9&FUK9c2G^|>qZ+3%I9_TchLuy9svV9b8JU=L zoEnQYS78sI=IdyzykN3t`!E`Q`rtIi0t*nFJ8iKH<+lw^!XEKY~VsFjOf#GZh!j!6qMY4lu z7xBMyv5_A0Q1p*YDDd(!TRqV0wC`0gP+OsC6#MeGH$gziZQqUQwv!#qYN_lsOJ^0? zbn7um!3|BXMXB(aLExC2dA+z@>{8@5s7a@<=Z%aZk%;=CV6J&$yZH+u`$3A-iZ3EZdVF`L(BlQY~RnL zEN`7aL-iIbV;cp`iHwtE!8%$AF49&C(?;8;lb|+H=9&8sTi6z+(TJAStxHwTLbU&~ zAX0k4r~A{WZ$fW#SZD-->WndL8gLKUUoyFaaR#Yg@HMXsU`A=LJ2ra73)E1_Net4GCbge|(aE*CB z=XTaBbak*4nR$QRpLBZCJup0sL?3v6@3pbLyPshrQrL7x{knehW{PYkE5E2n?Z5o3KIE30*;Zl> zM+|ws2kSHw9VkM#MB2F%0M!BD3&;nv4OoC_nfn~96XdJbNhn)c4Mp?dE9VX&Gpm@iZfDLhkh0c!b*Q=!J zwfdd@a;;3u;a}inLtZTN>MCi;zW;4<+esv|7<9-9b2JxO936y()jX+4{)_XFfz%3c zwe^!Wq@zJdadV6MH3oqhv>#V2Zx%- zAR$fzV&pCIBY}Nj4r1%yz})EL^jdFU@XCe1w86|qvHmuG*3;RQd{cQO8P}TYj9xU~ zaTB@>g*V@RVtz%gba!eTx}7{TYe_BhEGOw6R@PC64+r^d^ey-fH~i9CYbh7H^_E%Z z;LO5ZRmK*^piT3wv$Ymrz|bQ5`Ji#TNV2nKrSOd8cw7bcBU&rr;xi=87hymeGXAY2 zfRx)ZgQuHS6b_IWXJGP!NO5%75hQ)XDLaf(0d*o)Cl|VysrkB7cm~V3LU!wjx5XW@ky#(MZM}?38^LSb7Znn;~+3c&7mp@}J%YX8EN- z+;N7fLVzs=8i{WO6LsEPdWwVWkC)uKt-7Es{~>`~_7}XO`qBH-G1I^6deeOb4nEt) z18r?q2DV(8rdRhsXYy*#im>)V*tC9vpc z;8{-oYg!Lv&330riF{S(Jjq00szM{idZzxPrRA$Z1xx)Awq3nB7~#WvfMvPi*18=2 z>VdK!Ti~!U2a(^|_#{4bk8E#$8@%%gFEs(5aLe}BFOL*xbPYVgw}C$+IA?t4a}Rmu zmj%`Yi9dtI?3={@@?CmD3}h5MxT|GSJEnfc#z3f+K2W^VL_^Mf=liwZ<0d`ilKg13 z#?000sGS{UV>RIWC|K&$pgF@DkO|fFlVM*LW!xCdHp_6QWp3HY>7F4+ zJX`nIms+Xn5AOZZ#l^t4I1fP@GAw;^1$Lk=NHja2?#>z*)nc4ai2_fim)ZY< z+2UZW*`58>F6@KS4-0v-6;qJW1cS+t0vqdHNK1X~`q01WcnL#Xa!%rSC+(N2skzOwEw)Oi;3L%Xxgo92xGTnR)SbvUkcIFFz*-(f4}s z__;a1I!aFL&J4+P?Nt=u;@eNhY(ANh9qpt;SLhzv)biYzJv1WZYJ_xIT*D9IE#OMG zK-BHDz=;aF6g0Zs(l@4BR)lr^c$rpG=bQP9%m+}22E5)J6-(D=!#3@c^S#B(@YqP{C?<#E$(X4fKLcL#M;4ivQBkCG&5k)t(;T;a1-N-ojQ|dg=*U-*9mO%FT z@k<>Mp_&8k8!aO=-j)`Z_bCn^2~jB-m`4-#m(~5rVCBN7?AQb_D9%K{xa203J7hsY zbxvY;56kGt2oWqE!A}EBC2?g7HrnL5a1310kHx(cF5B1CZ*x zP_p#Q<;gh6B`s}+9Q9aq2OAlSU~$estZUV68nU$lyn&PG@bp4`v@MF`I_OLxratc7mC>ue%n zy25rB7ng?BATs;qhbyWSY}S4`d-@J}VPz)RzF6H%u%FnvNo{^3s!Q#ieO4WzOYJ9f zTg!t>SD@OUQ6L%dn(oP-wc->Y6O?H90Z#QwyM72Nau^oP))O1+eqp4{CrJu3`Q3xL zOVkWAI{Fn<(Du&{rGStmyg#p2nr|d+j!w$z=S$>(0+$(wv>8QpFB6tX-< z1J?51n6dCIyrJQG<5NTa$62H6KWgnB6(Z4swp~jJJt!qrRjmW0YrW9dr&FJEn1;D4 z%8^kltsJuIkePH|zHGwG(kNJVhi?+C40YhAktF#8GIvQ#Ji@gXx-%W@k^&PrJTHbq z#oySd#sC8?54QUH^w{AS=0=|Q5d~+y`J3(WiL#dFg76 z{emcM?a0u&Kp^9pCSwrY*FJD%KN-gRualMWLW_@7;3~wXk`VT&@9_Dv1oI;Ev$|O- z>-it)>+55&`}OOV2VwoKWK=`wtJ*oSY+E;jXym80P$l~XUE<2?ReqVfU6rH0+09z({lc8IP9&EG@-@HPW7AsAOQ5{lFL=q;^0XAHMli*U4V*eWSP?1z zogDkte*#_S=A61&?r!~8etS@(NL41V&vC!nN+7_=vMSQg{OAxP)9v__>?BEDQ#!*0 zN`9pg)b+W4=&7Fdl}I%VmOVO?2_b~s*kDApcy$GeP^PefKlk=>@0Yk8FCr>9B0~Gm zj7^L>VhkjaUhG?5t#-6|b65CMrI0Bsb!*}3L0szVS=@Tc%J}J)&k#eR4Cd;NTPFtp_x81@r@+W3}y}U`#z6X zOZGVUmqorv;Lb z-jqThBiW#pj8;!J#|b><@=@QC`WANSH}QsJnr)*m%V{;zaReBK17q2eH9!eq~fx|EN8 zh$%5(p7~9#K+Np6vzK5UR{kfNvfAf3v`M=i@kFOaXp%$V4_d|dnl?2mo`<4u!mQD zV1x8$M$);Rx?bSmG7Ea6Waw3XS|1&BEv;-Y9?GhbiTj48rsw?$cxu-hX7(iHdq!}Z z-oZg2C`ejdL|K{&na%+>sV^zuZ>gu^M-OU2*CnZ;5bK_>MU-!ldwIU@%yI!M^=M>J z`@H8F6>>S4}cAH|dB6kxj6$4;6pdYDXed95wyvZGX6xOi2f zRgX0)aQ8v*>Sh!Sx^=^cT%62mDyVWIV+mk_gENp#CBzo}+*s|8DJN$Rg(j^QOQE!M z@DV%Rb_8zM`C$8zN!bcBm?>Z~^tZ86qn=;*5cq^I+VKHdm;Z=zsD&3lv_#PH-z+qM z1iG#Ni1dxB&=4_WtRuC*Ez3`ZfPkQwo#|K$>?;)d>H##=XrZSHFhNR>{l8tV#)(Ej z(i#Ct=0&nD0ie5%^a(b{2V_?5g>MsOd`x z-BupOR!}9B$Ph#@8sR7F?vE0h?55!hukgBYv|ea9ExhV?yj|@mgnPU#MFIN&`kDK7 zN*3W4cEhS0wt7g^f({P}Oj@kKOXYQa-bq3QA|LM$*ON<)Nw8?XrJ?Gi!4~o|XBJjv zKFDz6xr`PNdbj*Ea;x*6L;wdF_t9gQr(r7@iJlMAoF0fI$hpd){WNk0(Np-@**Mp5 z7I8`BD^)-V6-sqP9-6paV{uabBUcn2g$V*8@vgd3EnEsuhk~nG&$D=YuA^YPnIOnA z`+6seqT^{V_hHS%*Ok`?GP&INxUT{I1|Z{DsTnsi=r`p4sul3yd1dH3Ovy1XSHWJx z&W&rZa-vAg?S+3&)r1CQfSo@HWGjr;dCd={KJWAryQL)S@sxcb1}a%+XJWW_&xLo| z%;P%<$E_3SK`BGt6mvxcIci_V8pj5h2olatzZ@PGCQ-e9&yVIs03}MUlo~~wzQy|X zYx66Q1qFm6RQ%&@VjUncBX9%8k|y2I(9wtN3!h64Ta#<)G5i@Mx(c%Le^yQA;5}Q; zk$sR#%aOC>H(Tr*7m0ZuzeEV!kw=ib@?xy-XobyyfOr$=83Y6J=%u z^A4iG;_$F;s|?jZv^*Ybv2Vg@;91c=qH#b?yNvPXQ)W# z0e+cC7wLcZ!i8PO10+wW3C;Y%LxG}c_%-W+7;~ z9*TGmQ%)W)ql}DoB!AQ`9m!d`9XVlrAyE=eQb}KX8XE@m?Y!UPh4f zRMoV!q&)zNg8WHWGC#dntQEn3NtHSTNMBOv=}(O?ie^{+#YlE#CdQBll&`;|oOC35 zHIouQ!qdMpcqm`8voMzz4(t&4G7a)#KaDQA2lp#3c;B;os45IC zz5D)ZK2<)J%kQn7loa;s$nVZFqR{WcqI6JtWFn2&&823O-@jDt=oSxKd_#V%;r4O>Slj~Lg2#@V@xAHC3s?*4_UEhl^Ph?&XBVcqp zwx@A%S6gRQu=bCApyTEK`}gyj^y`ga*#h|Lk08{_Hm9`yH!JfINvK4i$dr}oWK^OB zv#L`zvKzQcV-iQ?4EEI6zP+A<&&Ni~5C?;Y+Y6~&l4cX<3`?xI_>?XMoU2tm+{>lc zS2ri!*mDgIG=LNOIyI|b&bbC8MCBp3^u4!-#w9&!+H^iuVQgtKq;ya~VEu#cQr-vy zwTmhO1(O{4Vr_hGbKUs(uh_i?9;}tRh@zTTpy#c*XNd~OT@yh? zFR-LAPw|`0!;h`1lKwK-Vclr|_hU8fj=7ZyV8sB})xR6bIsTA6Ofn%EJck}UMbtcX zRU>;WwF^f-6gtX#Uk%t1qKW=YCapSsI40#pwK@9B0^1R^H-nE(Bi=qw&C91faSCg8uO_b=f)xX4JFG#e z;O{?IiSYgPzP@~0V6=afQ(Bo(_Y;{HrfHAo+KzCIjLh!6^x1fr>1XBM`(TLmPs?R_ za!9j>`oc>D&O9&WV5WduR%aR3J2%hv9q1s>;IKp2k1FtW<9F`>pI13z#F2n{UPha9 zSKQXNtp2zDb>4qx@Qx2ac?35H-z}oLWGO;axtxWu*nVbvC2#OJ6ABPn?+kSQNQ(RV z@;f0+8nGq|s<(R7N+QXrAZCeXf5i|H5$Pll{bL9ItmKGD|6-NRAx4!u^3Y zVJ*xP!1FQTDX|uP6vN#lwx0e?aMAd%{sey3YjEih##d>}F|kbmEh)?Tqe>pejf@Acm^Rhp+V=aAz(1pS1>DyqAzSO`_U;4Th_h>Qo1{ zE$QCA>7#rs7T`obZW6nuTKu?i3?$AQiz>wp*Ris!R%83LV#O+WvqAP6b#Cr>go#q> z+Ha!TbTt(?{!UK{OTtl|gFR8rMMEa8KHlXF7A{n$ck^+MYpdl_lB-Gn!J_XOeVZn) zOx+RkU-GvQa-9$o#I-hql^H#Mu3 zGcj6S)fkAPLIEGIL!X=8*RS@qF=sHTWIh5$YHIH9le;{?*7OM3pF5>2)e+X&-ngx9 zvuao<43J#hI0xVcPt`B~8C?tKh#1`;B*cMef}FVlAVzICPh;Z?b-52RFEirm`^wd%5LcgcC(A55wgZ zeA;sR&}aLKRH&Wny+h^3Nzyef_c6c|l8Ye0EMm%aPJ0&66N5xl~QrbUU7pK5M6u5dCycd^_{_b9jZN0}O>X%Cs!S zd$bsap#Cp|DM);0*Q1^*+RB zGs=T)c8UEo*+Y69-t`{W7Io~;pazHh^}Y43Rmq=-A;!y}w{_O=M1Mkq6;ReI)Jw_A z=IbXY4Jl}$zvg_U6``l4t=HLmsEcJY{kz;evlx<4Vx?T|fD|1{;%N+%e+bDOJwO>x zO%J!h4~3dJowd9LzLRS*7vi?cNjyww&PS!!7WYe&ovjrGSNwR&GDH+M}O+|5~J z84*WFdbd72V`>S$Zz|@KcaN2S30WTrc-PnAm4p#u{0;}Isj-oExdM96b=?_Z(yV~` zm2gAt^#iMTXGE@4RK-I82CZguw>6xI0?Ws6i5&Ag#E^9|;^2h1lPIo*R6=vhKe>G) z8wdyz;#fHL@a%ZHvNNq2#0!-vy?=A1>1JTJ32UHrVLo!Q40!u>eHdzh5831vo$^2! zd)mX6+&p+kMP*&4JlEOTNxTmf`0iW%BQU=OeiuqN$jaXT2#hgpF|`Iawl@CG%UfKw zDay_!Z@K7?Z9}2hK`%zp!#k6sGlpn5XA>RW{Ii@bF+ZF3`|lGm7_54`-%3g;o47N^ zykXIwbdIM80kMHsXp>c@@6*!#3wW8)B37mb3KY^5UzZ7DMay4}xOh z$++75woCr}Su1NgMR?fEy-$vGx@jDf`c~d63k7tN9$&z$J((lp9~;j_CdxFtG=&@> zBKfN2`n_r;44CS!a@atcW8ky1dip1~>{gbzThD30!;&`rq?vcZ8mv%lzz5V>&49Bi zw3lhmOOuuo>ld)(11ZrEv$bvG%x5I;G>25dOHOcDLsK*f6JOl630>p#$Rz^QfgLu< zfC1)J)JK^ZOc>iXsOD#wpYqcC(L{68(w1V3AZx}Gb{`f(nf_E6(OG#u%~YNmFt(Jd zO@31~NKiXoE{-VS9K1$XWFF9pBuzmq8NZ)gyn8>}OJ&w?j=b#6~$h1d8(@1BUnC}QFA80?(JO?JEW z|85{QEzOe6e1Y+oRFoI0beilfDK3G+!|;5b$+li?lv3?XDbcLHFagJs&2_Gz%|pY= zJWc7-lb!lA_3+@SS32W&o7LK5Ct?xXct85dkN1$#&#oz387?i0)s}eLt{IkNYrY~a zX_C}3Py*^>7>Q^{#q`yN_l`XjT#$&djVCV|5k-4zli_+)UvBy6o>evh)a8Mn(8<^Y z?XakY@8;BIW=BpCsCXiR(}FQQh237kzEg%(?BdE zYh|U?LtCyzWSX8IjL{O~Vk>2ph9vfS)M3MdVFLy<>LUm)lkA?xh0tvn3s3eaj6eLr z=b-%Z(?=HMW8F^{WCul7s@vv0xx}Ct=KTW(0zCv35JG?;jrYi0Eo!z(7+RkPm0p(S^p*2Wcrr>MC)**@+J~Iehh)JVsaQ8J z?q76u$rkIZbF(Wj_grCJ@Sw-d4n_Fg$+YUM9mClWr=&JNfjn13U7~87V)5_J5sjd| zdQ=W4X>>W5pEVit2(d*wmTlm9^Fvt4|>!yUyv#>11HJNN@ zUfF-a)=nc}*V9r`!XT2BRgCgmS8OYoMw=dMO`qfRA)BNEcE)Axb}W6Y!-9$=#D|~D zP}h#{;S)i#vi_5sUpZ>sEs|s2oo9j>pZY~g4jZvH!O^YH1lVh#mhEpt*KN4>utU)o z62GAf7-R!64kRQbQYP2y2`S3H40lMWv?LYdDFiL$Z~u_O2narWp(XjP7^pe$U6ef)ihJ z-k96B2AGFJ`1YPqeepZZqC>l9iO7x{d!(enTq$!at4|yp)A>VIHH+;6?vU3tJHad= zdhC1x1;0#e=0XMbPNnjs`E~_>4&=D4vRhgayeO9nNEs3Q zmW71;^9Ky5IJo@A=iusa-d=QysuvwQE1Yvb4&w{q>o`z)` zZZHZ2pDPp{j_)(eWcK$PaS{Jdl<@|9)EWIK+_ z5O+nW72Er{WwGvRET02bK_h%0IjmRz#-W(+2sw&`9Lfe@S2mLhsO#M&iQAa$pQ8f{ zOI{y#YvV@Jxa`lVlIopOjFljEl=Ts)r=)}oP@-)g)|wCx_nmpT2|~ch_T6P85d_@O zXH$;vrD|TqQGAEux*EWKMl5)1SoCBYj2erUtyd=x}z5VDaq^l+g>xF zwZIBz?=_C5mA4L8DLrFwEejq@lziER)83V5t*9$Zs|5lC7VJAoN*VT@7!E^p17n)hjXmS29U0iXWE^(+Wol**gVWN<94$ zHQV11;q0s$49r7mo=rwdyDb;7h-)=-FYtPqn4xN2YsAjBK(D!!@8*d4604UK;$gt` zhAtm{R^0lFAVi92BEI1&AJ6KN&8n|?uOD(&Ov=)^*ut5u{k46?S{MafdX6aAv3>07 zy+zq5JqamvOcDzv(1}kyJZLrAUanSa?W>^R?2=1s^2R8Q`W^$kJ&$L8Xq*={RWuAt zc9&d-ASL(L`OY73{x3Q-v<8!MOSfVXe-N{q{x!!{nPE(#ul@xjM9a~I42x2s1sejRlxWi|NW|X4ac3^=;7}TK&|OZTfl?bp zW!8Jj!NlFrh?u>|%%s zmYj7w?f#sig!fLC|-5vzo2dHpnY4{jvLyF`?kk7U@miIj9N-0a%Zu zZa9K$dIeb0liSoJtSyq|nmvW>o37w!0sRufrGH>SP|;FU6uO)y8=A?v%6OoR#9}_q zFC&&Xe;@(WMf&!kPOi06+CRkP!0&E!(&Qj%JwxHn zR!WN|Vs88lG~Dwj9k0;@L5Rj{uu}`b>YAE-zx}+n2Y&_s{rguM1li0>_(s_M`>^tT z0i}49OX@#;JDBn z0mfep2_2P&R6T7buFN@Ep4H59t0Rz^Db0;LeT?gSgyN2bz{#imYaiO4X zvVN^yz+^A8jl(y$wq^ozIbu>$_T#pftBg?_QR~CoJpb58L$i!qY!qKkx}AY{gF-70 z-y?Tv{sFSy=6C3Ea=5w?dHfms&5;)O0qr<8CjlPh5rD|f=M&Wuk^_&3nG%rcf%ADV9l6+`Z z*Ue2HQ!?)^pT=w2S^CT|BBCcv+o8V>7ZjXJfZ!9jSe@^mh#kpO(pp2cRSyQchIFrM zwLD&}<`Z%0)h)6TKI*0=6-r?pSQk_lLyS*5^v@-^HOG(0nExCc9)i@MwA*Y*xK`I> znEL?qeg6`-FZSZ%YV%&nTBtL}WLM(VVb4Kb%^8m>2kCX?@Jr_lW*Q0F;y=E=Q^Q=djmRKYVSe!Q$o55{!9&sXmGH$ z@xmIg((PKFB_~wrB^3VoGMFWUKiT;m(t1pG%LB7 z{(Id;`MSJETV=Lp*!Ht+p1loE#w%?SMG2PfOQMTyi z07onXtKfU$zmPi40nu2*{Ja+%aisM7A9Gz#NM@mO~u_ z;j``#4(_YSe=&Uourt2nG+Th=mH7}4_(S+6Ly5=5`YvzDnA{Qa)6^L?;3L)Lq!99@xn>wQ0cQ-=&R57CV?ixMEC#4!78 z(=%y}`Ut)X(4e*MNmk1x?6!0{FR_Z>X!sVh{iX|hXCn5?5oT`DnPiA%w-I8}%Oe3L zk8+j6!)oP(-Dh6|^>iYkVHEP1<(gH8G5Z^zfb~U3JC>Bw?D4N^vEpl6*HO@juvO>a zbRtXmTFW7e%Rh9Wm2f%4hVl*f=tXz+;CWKB>$9E%r0~?#{1-VfMumSfOouk~9Fg@` z3>%ZxW@W|ts|D|f7pq}Gu75%_a(EuyV;!o|+h7|<8=^ZZ{^iv;CDAAPboj%6Y2jLg zg=q~;$c=`jG%iAKlNzL`&L;@^qf_49_R{z@IudxMh+&0Fx5ww__o;!e60*8V&3)P9 zb_Kh*F^L%g69TLY1(-0ezT`lbL_&{lHzI6+QqcuRH2#&UM%4Ww0dkv-jO5nIQ|lc2&g8%*@kuQQ=-# z#x|R=$F1HoFT98^8AHY>m5;QJlJoE%3h)LW%78RCCbt8P6|42o!* zE6rP%Cn}JMuA2q5Yf)kf*HIbbPfIQ?F6efz$LMnD<8GSDQ)V)Zse*{Nbf)`n#OU*KxcDt6E!Re3d6WWM+!( z@ZG_ zLyI9XXVRjSxfO2i(fitqLvk^xv9nv+(H3bZF>X(Pg%t0;f>e8eLYQ`JNYOyg;oUhjo5~U&lTnNn75$D&k%Qx_>k(4yxs1)XHfrv7A|C_Wg?Z3k z*D9hxftJPl`&O<47asQytcD%RY^ZqlM!UE-*4j!^men5ZA5?xxQfwRY9r2%r5GfST zsjJnF{X+3pzH2H*ZiSBEJhRG*xj;rH%P1yA6D4K@=5j)Ff_Z}eQy_+v6S92r19hRy zNxAP*8M{r5;xqa2Ue{jWe)QUNxv``$24>Eu0%`@ z1=Cz-F#`lwTCU{ueyG}pKdO^)_`(H+pBk-5Y-#D%*Zg4TU_MsB*GILk&g^EF{0$94 z-c=O39hcB}x)otu1{*rcIYx{JJ*5sOvm*OKOnWeoI6dT+&oQBVyv@fyIbZ7u9WO8O4n+kR z@)@pbJTGrBK3Tg!%gj5nXbJ0_5yEZ8T@357yvM;+G!y^y@`XatwRn;Q{l$L1$@WenZL&=fr@}9*Ib~?p|I(C8zl)Ikeqw z0UIvJF?VsgQUf+EgeUPH4sIPAQ`~8=W~vgVX%Tb^n*?Ov1at91`P{M#OwXa&Z=P4# z9(x}$<(WkVD4m}{?k}t|gv3#ClOt)-jiNb@j*cw~2wfoMK}28t!eNT_?D1<|S&C6} z1O_LtEq=_ck6|^u5}%BQMa%-g!4Zl&-yac! zFmVYa?9|~FczV70b=(7)MmXqZDK{gRhse@&@e^wwrWP$MEaqA)@ie9ivsPT3B{_s2 z?6{>g8?_6LsG-MJILLg$2aE^v4*3A=b#}x)7892`_@nhf6$Zvj7P`c{>dK*cnUo13!sv5 zwz_VU(8jJGb8{n|u8TfFQR=&Eej>;TSbkmgJQXC4iaxBOWOJSzoWmv#U?hi;JM5dy zcyu}Dd5&!-VIhFyTCUzA^>GfS+)?Y z|6aK)N%=(sWy13$G_JPTX|uQV?Bg#3Z$S!=p#H>$a)`hA@GdF$(Pb);1$Cbq^{qoWA82eyrk zIXJIPN0$Lb7a;!`+K$+uo){!woMwn|sRJ)YT5F#l;1)5H2`h^PE7V>ZN~&ZoxYaz}>q z7m!QFxZ9JcN)nX$j_GR>lp%p`iXq%%c(-3sDodM7tO^hKc2oL_J_GB9;(LJkxiBNt zQ{3a%%+NZS4!Xc2$lI?5O3rZGul;8To8QWcI7dY6+~D+MC+^@86zY4rfuX!+(PQUx zLJI^bvg0exut7gni|;z|+J3qOQ|;22cAr=O^XyB{m?6;4Bpq08Fs$ykvbEg}S$!1( z9IRHfbkO8%pTmI3fxx~vBy{xNp&`_O4iPCSMD&n=(NRS!E~36@vcjD^`zSJD#as6( z&X+qW;g3v$jv^7AN+S`4v`opKIz@U~%UjGWtHR&MtgGLTHjR-~W4k#_>ECcfF}byZ zaS}f_Onm7^bW@q<^D3SaDGs#^UbpaEXYJ!P9OlMVAc~bfW;%ZjGYHt9qWaKB!~LZL zU;%HxMJEaGr~D^MWi`;AW1KHj zQ$n#Ew{2!Z=+x8wldT&a8KwT5HR8)lr zcaZYU!`{QG70ygv}a|+d(8d$(+9Eg?Bq&95RjO zP2XbnesMfFKtN&o2P{~go#rVmAu%Bny6#m}TP9HeJXSI&$^l^a!`dK$W)qAyMnRsU z#P6r*Vo)4N9j;;3cU$54xsAPM3{dcP3ML$F1Ye#yfLrSu;&6erG_8jki$q-y59+Gp zTl}`#Td4-oY4=P)^8&wcY``@4a#EWmM+g0$)0-4~l74sI(`VEUvC11UDQsv;qGZgl zD!1Ic45AYwZ9B+t9UsHYXzIj{4IX&+LGSjWkcO7fa;i_`U-a>+*vapFoAXS}->;OJ zV3eSg+ly-&F2V5ryziO^7G@>9lD4i%a=#`vwnwhqd*p|*`qq!r=2TjvPNH0JP?AWM z+Np!?FqB)7wZDQO*%Ha}Z<;^#6P1R#dUAN#s^#h7<0sL(s3|B<^$;oe=`72nsj{*B z{Lhr*EdP~_$zK89o9X!E=!q(3o9=cC8?(P=$^DbbSf?4GyNiN#@`;#aZs_<=+D=1* z^NaJ0tH;E~(JxH{xsSy-{-Zx}p6C(1oxGm^K@gC9p#ZK}R%)x6hA3TV=u?+@o5C0Wz7U^Vk9Uq@kwJH2nhN<_IDMyNsW}7#-D3IVSxtcrh415t9O^If&#dnR?juX>v@Bb*&3~uLK_EFM!23NAcgF3@Z zA(J-M{KrrB&^k}2%}zGMQBB!Mg2YaszSUPl=x-=rq!PlBd}GZ6Z64LrOG^26)J~D( z5tdbWq#`k4R@c6I1&Hk*?-vx+rWPz%9JO#I(dBlS0br7Xj@SLot+c#-xMwDmL%kO^ z;N#s+?flG2*IhBv7t~Q+ zZ+7vqBQcUl>pL@Hhx(s4=o&%exI6kTDm3h&2xmivybJq;?$hVQCF36~#dpduT_1ai zUB|32@kjZBCd@txd`{dN-WR9(w0?ZJ>p+0zda^LESh^GY$lOGLekuZkkw`iE({am= z=Vw?HH9Nawd+R(F^b>gKTjMcy)BAKHyw38(F=9uu`w0x6{*&cR8a+OFwWzVC(35dL zn}5bLBY8o0aRgj*jRSpFM#e4-U(~PxoSJTzw8lb)3Va=SOUz4sqQ+X>i&FSaXYst} z-FMMUa2AG1k=K1p&A!hallyrH{9_h(Nr$)5e@TGT-#omWl5&e~@p^2@s86+?S*pj> zsn7YjZ61q6ad$4|d3Rf8QJB8?2N`g7^u|NBVu?sNOp1rtB~rta7!6-vo&oprBzB^R zr9-CNbfL{n=j$F98nB_)$Kmf4Dnpu>&P!d& z#tvzz?}RUX2&4aWe2zKGnob&_7rjO#g?=P=VyG#lf5@&IXtz~rB#SB%l#_EmU3Tp& zO>gnenZ_y2gV^qM^XPgRq85E=-4aJY<6PffX*Rkre83JQwn?e)SpamVdz#+DuoX(Vx?&%J0t0MgV1c#NAc*gXy`2>#3lb@3>TbN4R4(l40p^GyO*nxO zQn|!rdCAmE#J2L#%$3WgoS*G;%y*v+vXLjo=ca3R+Wla;W~?Qn{;R2{=mv+HigQnMIL)$4w0|XiH0fFS#7UNRE&c?W7F-8QJ9tHy}oxmQmWGvcoxv70)XTwe-)C z=JEEFhLO>_{l;_W(|Jgh7>A3(aPwAEkH)9SiI(c}x+E#@o9c6qpJoc_3SAM#(why2 zMmKXLOE18bv>ExCFV#X=B z0H9$A_+_FI3ij>$!#Z$XLF>-S%-BolCV{1nP7St(L&>IwYisf1@=lD6#w$yBuH#{i zkYdi_%x12+x;meSa7E5td(svJJ)CbuN{BzYM;?NcflGRt%DQ-n&T=fWujLoGuYp4P zs_QcA_jjAZo54G}ID6I0w)i950{dg8lE&`Cv;^dho>GW$=Nk?fr_Z!1iSjz1A{@=Fyt&D!(Y(JttPZ*h<(9GJKv zzr22zAGrPBMHx(0_kY$D7cuLu*X}yn7BjLUF0!A%S2#(M-&D`46b?Ga$>#qEzijm7 zkcjr+`LPG@zL&d=S1&Dkz8|t%c-j8}7yk|X#gvlpNTuOubi)ibv7Fdw z&lOxBn&s=@EYpd5zCWG~17D%pwkhjHVew7{2XA_LOLgSv%DJw%w_SavmMJxw3cnjT3|mMn^Ms71QckB1t#eJUavtrg)7o}Q3XA`8N~{ipBZ*X_@nKW{!UT!3nqV1p_-Fe+J!Q#;7>3(9o+}}JadP0p z^EuGjDO2C~0)Bf1h=z_0>uatE7;L;>Qhu`B-_KWJS4t5oBZ_cB$6|ObeSJx)t?gKs zvPt{JVp80UI*Jg9;N9Z2lT72N+81?R*Y@Jv&Sj4OeEWrtgO5_BNYf3qFYFY*_Rcj7 zBbwtZKKe%6t_`4?++U+4w#4BgJRbqk%Uc|nGw;7nqgHWfhz(Vq<$dzdLQR>3e*M`!Pq}}%j3znc_!+c?()%n+EQ2nrSrFsE?48FA;Swu==`k%j0!-w%^82IrT@|gS$36*}GV3IV$J&30KjDSr|BabA{1D zAy;@nOQ?K|5K?n8SH<4X@iq#~eC0*EVRLj;6mS6nD9wH8y~MQB)!BV#f{Bb3vg zP9L|jxJe5(+%EG3PQ7s{gYR=29wi;L1&s(eVS^)4Q(4-_xJ8+J`|4Me}9MQz6#YXPj~K(5&!BSe4S*uBQ{g`XzhhJ>8*>ez z9^b_dEON{yKi>#vcIb-a{+;omZYy;dpOjv(KT&fRq6%1qb!p`ZF5L%*2GyHfj`!P0 z)N~IVe~|xs>82t1_ME7d+7(q<{K3$bsBgN#Dwym- z_ep47@?Wp2}a*+V5J5GdYoKvW;XF%IFoT7{I0*K6OYq#ZWsWARY!O+wqZ)X(n@we z%~$7!UtDI*=yBRGy+}~!Rb0ip-Zr`MXgpmaBYkA`gm)!%CA>JQ}xQX3|}o#x+-(7gy;R~Icy#_btlmt>|r=ARf{-wBO;w`0qC zKD5n#4~dl6{g?i9^2k5`P56eD1z1$}W2aypMc}c4sKQCV7fOu@ZG=&G zrkKBlY8vh{E~$9FML3Y;yd_h#kc|xkLw)ZTuuHCeU7*rN@8!~XvRjyKeSkz({Y+5` zILL&Yhb{4H$vtm)xKEhp=;;wQwF$-dno;F~*Gw*%GcRpPIE#ocZRhFkE5KR1SP<9@ zgt~?qLk`Q^Mqi!*$MABxruYx>D!G~4oE^mz?|g$f0<|tLhNr{ckKR=bp2INv^AVrW z*XUZ?j2_^J*3qC@W>>?P_M$;$&y3V+iqRx>E)6l;4hdPmbQ!aF3oM(#dcd54#7{xh zH)R_XhR+$x7^LMqM+uhRE+%Mdxt8LiKckQ~3)F}5l5!!ho3|!PO57)bP-wEyPAea0 z^vD^gGb5hhf!7H5V4Ndegb3Jn;)g(o)uW zdGWy@t|%HL3aM08^%Kph$9$6!Zk=3g5WImdi$idrcweR4wlzb;zGCO*`wSPI+<8%P z144};)KG=7&I9&lnZFYXQfp;G{MQ+RDi<1im+0C2Ms{j5?})f6Y)@AxqJh+}p{TWm zsHx!PX}^;^Hnlv{ll6MnH|4YV)Wyr%GDmgz^xSp*gKrpCR z^MQJo!PR=!Yf{*q%Hi^_if8HHdpg0V{ll}9lqGyq^l)x;mA0NAStHN0-je^S9QZ`1 zfcQu)XiFM8Uww{@r`c0oL!@rqxp#-czC75GrvG9*Oq92H^WS6sT4gcx3tHGnu z?DbP7O;N2w6d=x7k|k@C9U&2Z&c}i(4-IAtse)TKrdeQ@{^N7#6~#~p;&Z65XiFax z;nVY}Phn-{`BtD@F4L`S-cBR07pX>55=1ijtFg;>axyOu)~tSqOsXZs5F$yhVOUi5 zv$}NtsX7oOL%G(HJy!|U5g>8!AvZj_Xzc*pu3({XlHax6-_I>+G+%n0ncP~}c!u6n zzDR*^O|0eN$G=F`O@7rnR31wiFf(#bAInSXI$5+`w{rEnv!(Rr`cA!O-y_Tuk&tqq zH~hI>n4^>GJp@UClGTzr3H%Vg1@hXi5Qd@0$&3gr1$35}3vgh4MRK9}vgps-!NEcF z;sqkz1gJk>X%XI>%6$7LO^9^1JqpWUSW(Jk&+T^%&s)oYpyRmUuGDiYHGNu!wH}^VukL)hzdGt>O?vGP#TIZ4 zR-YHJ7OKsr{gEYo5yZ_^U1x7JD+6R)pGFIG;ME|KUCvzs9o1%AYUY4r?j$YE!=|!J z2GRV{48Un2qEqobQ{lga@AUuPph1d2tbrZYn^A3N!a+laz~pS6Y5JYr^~2-ANA z8}cI_(d+DE3C0PWN6 zmcuv^NY!X&Fo5$8kIO^qls&Q41Ers5Wa z)}hr1))426-7FvPenY^|>Rn;UbxPq=Ew_sdf*AGM?H=XX z+lJ>Dycy%#bXgIrv6X?5u3v?1jCB)UvZE9`m{5!MtbOShX~#5WxJD?KSNtsTDsqFCq2?wh~_el{R8=o?TDx#>be;Y{ekJq zLwLNHwMK2=W_F*Z9y#XXH&&$_leb{C*o|1;v-_RD_%CHbI}w-n1$Bwy;RPefB%Xll z##Yh=bg@FQi|XsAh7G*WkX>8aj|;YUv-KG*ERwsa|GB=4jEOcr&Q|E{aCs*X1myJ5>LSiI~NgY zNw_eTzNe=XzhsHtj~6Wg>FVg9@BPK4L4P|lmGmKH6-Ib>_tnYcn--d7@G^Sty!_1* zmA3aTp3LZga~Sy%oy1L#%-7Zw$ z&r`9V5$Zc204iMF9B*QYlF$nen5NW1Q{}P}Ua|qS+*Vm0tXoQ!v7OM9eI9Q--|RM} z-3gXVI=_1U7N!C*?mQ$Depk<3w%}(sgwjn)q zSS1a`$wXCBq!<@3s*AkSOZ(cDzgvV~OsuUL0ey+-HloKv;pC!S|JY}hFjA6(+f#{f zdeB0dHHA@}rZ65jva>rQG|8#Fq=AiXh0)*P4M(HQaU6zGa`p7|IEufOI^`P6Rxzvy zZirztBaV-4lai9iR8)D+|6obyBzNNA;4oOBpe%d}yu(`CwAqH##Gk93JiXVD2v6TI z(dS43fHYOL9mO{p^Ua+`iuPk%uEIYmFyv5-boXL5KI^)#zZ=~5FtUNrza;EqG+V?y z^3yTQc&5^@&uuf38>+(d-u|Ow<>KtK((KW8MnEo}Vim)Vd_fyqb|pe zM|HFC`GrZzfJY8013y3TQC~lxJGMcYv861Xh^6qiEeay%4#*oBd?tWL4V9KBvX84h zZ|@WKeUdBNxM*sfey!A`iuhK~E_wvh8kE3H-d)kQU@p0tQ=gaTI(xjlSoYaIZg%W1 z3Qky2A26E0ALa6r?1OR{LeF@X{jhFo~e#Xtz2?HuVcgCwA12}$7KU|N+F2#c z&`&l!R-KaO3cNU{JTg$t42*H_ZW#!iCAZ$_k;l_n`yyen)4pfZ=SF8HNb7Or_1jnX z?={1QBW39VRJ7y^`okC@31szt26brwwMybi53g^MNw4($a2OcinR%lm2pHeLkUKwz zC=VX2lYR=alctd6wwn9;0CeE;eCJdom4*6FTYw+r?p}AZVCCt@+^~s{iHki}^&jpP z4z<-gOCB?=^}vcxlrVXL5jDgfN5ihhcQu6`H~Epnr#O+W!?AUPr|ag|8=S))oXQco zs>5oj{(B1K_&)T4=+S?BJV&e<7-a|LB%U{kmoaL~l+^>4C<^q@TBwpyDeELZER3~b+4*i6{2U;b zQf>9t9p|5v^$Hx|`33c7cfv|@4Yx-``8z!IY6LZI5S2jPHaQ6Fa{U#$rIq4N^|W)G zf1Gh)-HzPv_wO9|PC(e=Y8nmxTi}?v2vqS!mX~K-k%D%2%s%I+=%hTpHZ02Qr;@48 zl&7R6rW&$vO4{*KHcpr>Acy(}><}lir0Ieaof~dvK>8d`Hu9$xrsp2}S=iW;26|_W zx3p?BUcbX2*Sh}RjME;H{t1^V7;D|qdC2>9j70VM`7*Vc0^2|0p?NX-{AGP2yf`UO z{H&ZPOr51SkQbwTc?gxixc3yS)&Q&WL+&-~Z+Z09l~Y`twiuGma|dGci#~Fxj3ae9 zCXD}W2^9lF#Ks4RzAHQPy$)!+urRG%Ij;7g1;N!B*}WGn0wS7o|M zg6URb@_K?Bi1eK?->zs^a&V-5jLHg1vIc5ge+;N!A;MaqQZ}SUq_@W_kiN8}t>R^M zgqM%WOfFu~^lNqFSZvfkABUkW*mt~AD>a!x4XErwm9nCM%gAfcQAdLbPm5|Fv*P1hnjL11eaN%ZSeT1<0$7WS z{w@s4{5-0EF;R2ga+=1?o^iHQp6B=^~+pNA($1PR_g?`VvOkm-Sd^ zWrz>tE2u$3gw5uzQfFO%44HU(opnuB&xrL0M`@o?T{-=7jrprZ%wH}L;u~?`#uC)f zfEtlnOK|tEN#RojH9No(G;EYTN~fuC?39>OhyO~hVPLz!yK2&ucfjRxb%8#qq(5w9 zH;*3bJ>le$k+QXA_Ic_e8+A_yGVt7o>JTUR8&0BoQT@LvgST~snW7M9VqCIRHaxV4 z=hEkYIKuZ7#xn|VnhHxo^nTnz|9s;-6VHIa(0%9G;6?Ow*0o$BG`W`G_=Xxi>zEBB z#oHSemrP6zoiC@sb=?T@I9p3UWTnb8;s9mEYbnh9TS64Y>}tvzk1t$qP3THLCv@5l z?NbhFx=9ZWppNr-*(5t!Y)TS3&6d$|M#&pLJGWPQ7Go@1Dyro0&ouz`a`L#Fyl^TH zD${L4J_(@L{FPLehf#R{l70B#8kpcI=~036yRoQff8?!C@`via2Q8{dP$#TLh2EF1 z-^IIaF*oW&>iK_xkiIx5GNG2Wq$E#Au=Z`SYnF&p=f|=zdMay!$Aap<={a?&`ZG}} z{W?{_m`}kbQGkULx(*OaBNlLrxd%jKTwJcL%Gg%52V#eqv-5^09H7T0s=?~b6uZ?Y zEGa9qIV=|}uL(15QhoUpWhjm8xG9P$in(EJ^34GMH9l!rLS`MGv!kOx(SxMSbm60) zD&j%IXUbHO!#V7`K)198UK-M-G8LU=fvv(rBwhOO_g&MlYoAsYbccLgF!(ch0N5z3 z!xa$1G}zf^gXYgM zzF0b(#Hjfy%X(k#mixm`IaEpw7gX8tYm zY)^GgULPPzG?e@3DNliLjU7MBv&c^NjLVBYk=ulia{%dDXp~R@8P~6f{j!{QbBt+C zw=v;XhE5mu;X6HD`pRhs^*UqGq3*9np}*eL+25vW>SSeQ(f<%STPMJ0i>^}uZ3#4e zJ>5^RZkT7Tg;iSM=j7@u6cb%1f)#IQL%yU5BWeb&*t=okCx3rX@5ZTBb7UtYJ3OD$ z-6Qq24O#QXlnDEpimTnY5@6slGBSBjCVcgKfgBF?y{Nm*BkKb(AJ8qcwg5O^@rugI zqR5`lLiQRJ!6zsl&!WN3+Qyc6W2b)%JnCP`xs`IM`;o3UA4>3nMm3T2HoWT$t^hURHVDu2Us{>5c4*y-RowdG8>)==ak8!&$P2Lz-Dfgb{A!2=K1Uf1-xIk~yH zZC2Fuk?hR+Rln-io~gD*2K6;PVmkIV%F7CLp!7;JG^#44RLGz$*iHyfL?Jnm*>6C%!8xg zXidbg8*isY@7Jb!^aEoB^9n&6kSgAd2)Vsf123hl;by_{W{8}bU4I2vjih9A?8okV zBhKgs0VRPK9<4f?eEv^~*z1Rk#i-8o67*;pY0~ zGG^f6*!RAn2sm~5tsWlcs1!i^yVhV;k3g6uT9Du)8&R}x`%a|nCbQ<(Vx_{JnrBXre;EVM61O%Te2S@FsDhi`%<}W_59tk zVf1-;)@yoeYjmXG!bbIeL?+ih_@mpY<__*we6nEQvG)IQbd^z6c1!qxNOvP3AbF6G zE(rkv=|<@eC8fK&1*AKryE{K=q`SMj;cl+W(Uy8>42Nml8Xp0qo}2{o}OOBRC?krXZwb%QKYP_Nr!zp6%+ptX`3f@-{|ss zQS1fL8$QM~LjUAOPNb(=Al4S4;6O8DUsd9(_fS_?KRG?6przeT>Dv~CkHXyO^0^sh ziO;R7iaF-Lz-(!0nb_9hr`v5WKSKEGc>s4zW~pnVPYAOeTtcw>!|2;j?)3K(dX}1q zX<-~|8ro226@&%wifRvMqWfY57r1?<>H5+PI* z^PR_~2}6%4d8-ybpQnoUz_>3;M1 zFyF_M%d)D>9K+G$aJG1Z>ohl7pQx7oug%sl3TERr^~sPRYzRNvkZwEIhlG_N)2Lsi z!&f$Vuq8{aEF9ujZOe;e9!#Ny#=7C~u4C0-$t7xj=@ZU)Pww26VP=JS4E&28!HAno z*T@E9)lReq298PWl#~a47|&O03x5s(hJ;&EpH`gpIiVzbgsG4y%&(6+Ix5cB)0!}8 zOS>zv>`Q&gaW7jkMwS#FlJ$R|-G#V4`W*Ckh#y$&`>qZ(J$mrAhs=5S7kE&Px1Vq2 zrb@J=Ohl1oh){a%N|UbMYR`P@sg@T6WWkYVP2^Ad85qFr=cVms6@J{^sZpB!$B}uM zA?38KORB~Ph5nZah!>#lrEQRtO|GVP`yL+>{tJJ2Av`x!2F)sXum4;Gc0%XjD)Y3P zX@CJUHWggstoy5bXUFunmRrSm+4RIjcTrh$r^xanQ(5a-zghe;+b*(92GllF>LPe$ zk<8|A&j6yNgotC;G@u{z+pU_juK~J@t|=UPQ)*i2uuVe*CZXHotEPy9IhQ5WV(h)K zON<=M8dz+I&ZJ}r#m;ZVQf$R1u}&8;vs1^PY16vje`sh`+W)92RJ|5`U8+F05C&{E zvS_qCg@u0e#m$@7{8A{7y5RY2PzeAiF$qHkETq4L(#Cb>BHpeDmRR%$gAKInN#4B& z)S_`ut|x1aMWN!Jk3T$avdO}K5Z$@BIzQcR!*EA$l1OEN$->gPB6b8n=VWC~dN2S* zV-EhS%mQe>&}J)0z8`JelJ?z9N6^!2(2Aj&?;ULp$8()+<>Twx{Oq^5gyW9ftF?Rq z+Gp^$3|6D1ex|$`I5h{Sx!YCEP)yQ90kj11nNkBY!h>KTCx`0o!;xd4;F81_0IOhO zVZAC|&W)V_%xUKfx|f41E6WEo6`<{M#tY2b?I}G^i{=GRMwZa0)MD8QN7j)ZFcG-* zqIRxf|KW29s_o^?*P&$R^hh*jZ!MfZynD@?jH_5G5`>10k3vx@E@^~S05OGG!Vzw! z&OL}Vu;#}eE!w@C`X!T$?$eL3GyI}{w8jAt-~ML#d>delqj6O(;R$1=Cw3LDN%f}8 za*uS})61Q7Pd4~s`=vyJm$0?rgRd3SpzaR~6i^V?{8Lrf?IW$G z#u;lE-EP}7Zo1-ZmHX@zmU!j4bg6B?NgJt|BoZ`z+s?s$Y;n?mh_ll5M`SXKp~Kpj zPm)o;M$13kEvP`0Bd=oH2l^trixcf?0AlZ)FoiQbfNu36$;V`{iOytSyr>rSxFQ;T z#eH%0Yd%qUyo6SpBw{Abpb{q3h+4SWH};2@0MAqzR+2;az`X1E-ei%h@?oQ3|Ld=r z)~}FlZ3!fx%edAY84XemzS;T-o||YnLrmS=@IUy|+jX&7Z4pCdNXDHtU(0w!j=9Gp zj9Br94VB`ypO5VmNVboah&eQ-_(u#no{V2q^7~$SWKn79R`T)# z%~h*_omPHRQ}Wf7gNuucHMbb2z_l}iBAX-izm+!*Fv7fuOI5#JpSC?s3GnL4~t(=m^~ zO}O90tE)XhQ4Phk#>u&a5el7k-S`g+Z>C-3g$TUbiHnba*Z2AJ;~SZKQp`Q3&8xFZ z%Hy@71eInItF|!|OnfVy7Y>rFESL^mq?;O>AUB=0ly=vDoZD{!5NKd&)%|bk!%p4Q zZ+t(@jjc_d$3{>zAJ!YTchJN0p|)c;fPs{%c15ynDn%gmtCNLD>`AM$z|7G9w(6$~ zL*0-hIoL!7FcI664%hXoyVplI*idvAg|#sGi^RLRPjWIzg#v~GE(;s0^+H)gNo|~~ zc1imx{ZpcF6q_x+PNVr#ImVcMs?~z0An*hR*wUEp#!Ykb&bd9Q_rj!JnOdcVf%Z-7n1iTAD?l1a#^(OzlKe;pzC=4!g!1V{-jovi11@HvDCPd$Xsh^*r77Kn<)&*#{fL1eRB9+WJy2Et=iSBVM0o`Z+}| z@lH>hQ*rSI@flP91acZ}8cK1ECSb-Ak^6^<`k1tE`>y!w14BP%r+u$*#}M0EJvFVz z0@&&_-jQfNyCI=zmBv(m=tM1Bv`121KRJG$HA!2YRMRFpugM)PfHtQcoruQo5r}CJ zn3V+HcFWX_PVA>S*e#+jaC&6wAAD_YMv4gQ*gF()G@(UzT{hjGO!_;p-(AtdhY_J? zx%A3>Do4{76CY~igDLF%r%5o7`ay%6#4}bnR?5`2Uwv4Yd&`~ZMf(RoqGsDs{zDZ52X^pCg14Lyc^a>vjU&v- zp}PvZ!`wud3yfV19)*4Ln@DtRC#k=~qlocPFWM3p8uP|r}*x6TS@`)Ys>s_6l&T6z;UVa^*ykGR@ zLZXn22Ksz`mG=$Hqp-y=Pw#RcFsunFQr3S_8YO?3=86SwvQ>F`d2%W>+gUHolRw~N ze$|_(=JF1;#wil4nv@*wG}_Sm$+6_|l$}hKI(U%G?T-IMp(l$Ys)+|)zGXqDl%!hA z%Xaiv8rG;DlT9Bk&`Znm@@NK)*{Y039QtcOz7JBZ)$(xgE`c6ONYdzi_5LHigSa|x z109R`*Ka<7RZfzq7pox`bT8-@iF$)KToN^^+-eoSx*NYT1LrgOW&@FT)XkSN`so3!@@#-vQ30+`FaR2^2ZXM z_I#hKxhsp4bTddasObisVlmuALqwuwA_3v=>x{*fb)47C3KYnmsi<624~X03g;U`%!4;>6K;c*>XvY!y_m`Y9^vDGm?Kr zw&T$Ayv}n!$k*hLeXl9`jN0n&|b(x}w0zumiyS6>y1* zc?$xw!A?bcUiuuakMXjKu;C+PND*7sw7`ceMo3xd$Z>pHH_$Ds> ztyXs^X=u=!STmKJ$iaC0U$-(45&kLrI2b%fUoWJX6;&_;yK;C+7s<4#s#`>}$?(T) znj#k)mNt^tZ-zWWbo#RY31;jz9f!T(pmMg>e7;tO`X<4rKqW<(<;%v_7Ca%xp^1cZ z*}b%Sm?16$Ih!X;l8s$gAn;95{>clIiKSF`y12mJK3&WJ5tn=%l#(##4lBpNr6_H^ z`z~#Ko-p$K&-2N50=1fIv$6Yej)ZsF%@N@TJ@M_~W))I(gj{+e5z&8x{|U9aD&QAk zaGe0euDPM?cPmQ0+T|{9y2qas41tye37w+7jJJc7`&SjEl{WfI(Zc2A6tT&b>i}Tx zHw%_&Y-+q2_Qm)&&BaQIwrHBi=JK+Q-$Aw~3sn=*!C+xwv9y3Gkk>?v>5-IHr=b|L znRv@1G#`9td-UV-G!>r&Z+%%=X^~TVb4(AVvGudB`5!YbItuGf(_iv=U%nMeHDFNm zGK;t5O(r%qag{B$wtQ2$pLjX-dZ<2d33@i5%R$DD6t}?r{~50xA-A%_VC-j1&cNvn zxcTh9$bx&rPVWJG(BK+9!{+X@pN0E(!j~-+@t#; z_Zssets$M#;aqa#hvGv9_;3($SI^64EEv1=O&`LqhXZ zL~$}jwv1$ki(9yV`((0m5)AyGSa#sAiObMhv_$*5b*ELxRGkEX-+5y#|@zNU}72SU@1kSffz?=a)a?|BHIM!E1B-pE@0kBAhPF$VWe zfHF_cVc`m5ms1Pf{ndM*xHu9oCawzO9$8%kHaL7AKqI3C4+nSToSe z6ymHrRd_!zfbZ^&`mgholBIApgEqB3Ks(FI?t4O8iBr>t$>6a0Ew$vRqQRvm&=m2U z8b3U}m7g7M>##;%lsIofZGB%1cg0zI-k-MqU~tbuuYo{|aaxyvDS3Y`q3(KjGx#If zIxL3qvzr&!w6WOvpF|`ogxj!r_vVLxQq7N38n0?O@^MEKY%8{zWL+AAA|kdDh{bai zyF&DHZf+2M{fZHHJxBVEewdwat1A!ykSe0~IV6UE_&Ezd6(P-%$ShL!hYFDhoH+}* zC<>q21}=lgm$`avu-|1Uuq8Ug#l|WZsmiB#^c4bCC{2Kmk57Yee;bUM_fHB`rODR{PXXHNFkOm{4%=;fN}WKomL_rD9v6B0}7 znST(}{ykLadpU@RK|V83%$tjFHaI2u)H1*Wj{!U|Up___z6k2SdI@6D{uO;k?cYge zT^hj9^L)R>wY$5^(;|Z$4#ux&_>r_$@f%l4m_WWhi1H~4o`DeJCu>7?u!UluCe6Rv z`Q+7p6Lyf$)l{n@qgWB~MeEs@raM%kLAvuZ;%bpP3M9~rlJ{U*H(g}<8j(UBLc{vY z8=lc!@d$W#7AVVPaTC1smc7vp4GnCKe0=Hf#tkCQ9KO+mov#h?8A!BJ1Sk-L!>CzV zUi*GwEas0Rx0xr1xV&UWgzLirj^*J>?LeIxv zs%We|Zahm_{6Zd`%kj(;fI{OusKQZXINfNoHg50U8%rmS6akINC7JV0C`g7%pcIcl zotgS*@s_`Xa+6D81-lqcQ^GiF8V{8h3+T$tbj#RJFU;pOlM>s{=hRAVWw@dfPm&Rbjp0^%k< zzP3MIzFrzR(r6JOhws07g#Q)fwo~rAhl9sSWkmxg$|TH3H$BXQ5(Y@$+!NcsU#UM! z->Vk4k1VG*J>O!{u7Q*=yZ=1mSM2*_Px9Yy7f3fF=a24>Hk`+W+g;3w+drffkOK_CB zN%NhQJ5^0qUzgWbM#qgC zh`Swh_#)iiQbwZA(A30~w$0%pHc?YoDPouZnGO|ZaT>O$*@_2BgrT9KMXRyuU392Y z4vJ_>tS|`#9pd?U+BPR|2qy}qK71eul8n1nKYIDbuRyGy9w|<-|887=8T!`>Tci|m zM`Rk~8mXcQXmzCc@j?_XD1hT1RkcrZ^SvLYAnd}s#geF>=Y!O0B(mvB){8-rlc;`u zG}v#`{YKd*?%I>voOlAm?Y)oNt^>+sg2QLa56SD^NR~WA z00T*U+2fhZuLtXIxLMbG$Bo3J4s`MY!?S-A5V_857o6TPwu3JO_v+-a~Oi#h)twvYdzE52P!yTsti~ zmq5KxCH>4|{-(+0_JKHg~GcV&QSUk zFCfedVNnFFy&^A(C#=tcK`x^d^~cS_!}*vh-Q7rk%&xZx()jOzDh3fzrPRehi>{Fm z%R~F1)Jh?oM>xw}S{mb;tnU2&?xuW8qt5oP0drN${D#A2RYRWtTE6Oe(I#61C!3f( zVNZGQIG`yAYl)An_*Y`$rbutuXc?n={60;*;Lz!siO6)+NU+$GIKZNz`(a+}HO^m4fimZ!CP4eTU8%UVz!Yw2U}Pi`&KL=*WhL`Y zwCEs`;H%4)?K=bpg@EUcl$PUNto5*p^^4m@3)IPK%N1)!xe;%R`C9uzfii2Fm)aoq zMNcnNu?X z{4y5qy5GygRgnMs^(&SABM}VLQeG3u%`0{Ub`iq1#h8dig5`Q@y~HTs-O-(Atbcl? zfL3spfp^^g*UgBO)>79 z%Ng*+3$z4?GYzcScMIGu3Qul}&IT2Bf~u=%oOBf#r&_=2%Jn>k)4{JKlww(+t0 zcmio@d9~i)D3&L?5 z=C6*{3zG#b*b|3~gyFCY-A$ojr$iwE52pgcbrtEqj~-E1iMxYvmymwd6#Od zLXCWZh(<9;YmEUVGclAo6%X@`gDlMW-nzKCI?d9Pci~uHP58p|+jnj)qkcka9VCj~M=tBrOUzvO~7xe1jod1pz&w`BP7jRyV7hNW+e(h1n@8|isKQ)r~&;Ty>9)xseFC+!WST5 zv9bJVYx@c{>57kHlhKlO`9Wysj!khKK3C-8>bfpxW5eBLC)h9gKR;Yy{WUD5i9I0} zR%!U~4`lD{N$EBJS~q;;8-s4o0^xIGX_f1>Wvkn=`@>dUThG9Lr|k_8z@D-Y-E1}S zEvVKndS>}br9A}&hE&yaF@LkC@A1>E+IIuy#81_ynE1ZEU`8fY`n}ipd`7SYIJ}?c ziH#ZaX`S|r+pany%4Q`>5E3ATRt6z9M|!6Q+&9%ftO4P_a`X~aOHxNAU-ml@MUp=z z2Y%@0O%lhCHwszlojos@w+%Y55!ph9ro(7@#6fe-Ay23W8zeRzTRQgVGBlw)2M32> zhbx*Z-DR#q218riBFlunp!Fjh=|mNF#agGfF*PMfC_m-LNZL3B|7@!T@$+?w)#=HI zN5Egh#+3?SEP{1lv2|W%l{dYb7HXcXwVix?q`O#w^U-CN0k(u@vIBR$cNGYpoc^Sc zURnUb<41RQf%9I>f%8cP=y`@+_1%rafR(>I- zwBz3U6|j5=ci<+O-Ek=|T*S&UbI6Xh|F_D$^uGHP&bz{)NxQGBaMBTjhFBD6Mvs<7H9lVu5 zJTp+bM4C+KA|IRCXLsybY;wf7aRn6KkJitR9|2J z*>Q;;Snhc?&__NbkWc=_mE8ukH@Bnkq3Wv(EV7lMb1vLpZ&~lcCaCwP&rjq$SWR8U zrD&zK~JRs2J3iv7bCr7h}x+9Z<=SMN_1s z=-?=#D@3XDvx$^HJbuw2|L^IWA72b^@kgQ9c^ygJG05qZLdbSYkdv3rxf_+Z=xL-pZ3 zYm{d(1Jyzw>z?h=VpX7(`##tsM8M3u`egqZYI5nPc_jQGOCvgVX#IRY8i$)fc+8Vy z`?`I+_)kFr%=Z^d?cIH;7%>CZM#}>o0mPyxRu~Fw>TU{Auh@TbUSuoO8P|rh2iUxR zm_v&F#w&iKm+F~jaV~9rzg#oT9Ax^9wSwS%J4R=wrl!il>!Ll0f_D`1Qd?sH*?xg$ z#nEfO=#AoUWEoe@J(q2N2gb&SMk8b9GB`+49W%@Bx0?bgvY0lZ6E9Z@-Z2H(v{ z88NGcr2*!`k&6*!u8ItV;S2Ki3k8@X(xYEfNvTV%?F_H63Em}Ow!{N?Q;5Equp`kG zQ~to876lPS@(QsLBd6>;e*h~w3!$Ee=4SM!yhZ;~9h98^*k1Wd@vQLGlE}AKc}%lY zk^Aik_h$DAD(lVsWR@GWUA2T z=OirLXN{5Q+GEdk3-5o|n`fkVsh}S&#fbS)2y#)EfOJ~>GfQeHZWwTke((H;n4hu_ zoH9V>*J7jn!gVh5>AB7WELl1ViXK2x1hqH0ra6SWP7Kb5ivk|4mSl~wUxDo3 zeO6jkmP{k%acN1r^{J-J=HkC-?=oAN?AjGxa~Q0`Gr6@Nn4)*zW*7IKkTd%AvJxN< zZ+|JeN+9O%4FOfa^sD9QD7H`6dYDl@n9z0(#4*2EJVXV0d-pn}ZXuZ}Dd>Fo=oGC{ zvPysL)Y8&bSXfwXOt3s|2=v8TQwdKYA^lk;=K*PD4@#vmdP$puiT4u@<^1BI;q;p= z0YRW>ABD0xMU4<4Ew-5` zbF-B3X#-B)A>~-f-vX zL+EMrNvm!UpM*l$WNWRj4$8G-c9|udFR7N*J_u`~YBI2wd=XCcip+@>519Lgy!t_? zF1AwqyLx2#d=onb?_tuE8T%W)SHy34pIuklqr4;Ptd>h%qV*-g60avV8=SbJF=u7i zs`W14sAEx_G?B*yvc7qD$h)qqwKP;aGe+ZLWA*AQO(=OI!$SQs&X&f% z=U!SuYX>{ev__i9au%w42w;vMdmg(MKdtyyX%G9{CB~1vjW3~pW-#va`?pR}%MD6{ zG@L`IEnGXmVBfpsy1_!VhNyVHw*S;^fS=F7^dx}q3Kb6GEcx@NtZ%w5Uw^$p+)S?#lsfX^z~lw0_VxHNs4e{Cj6~&DBnod!E`E_M zmMyR!HY)+srH|yGmQL_lO0i$3>CR6XIr`Pdccc+k6x}-ZWa-Ol%E~12?w}LkdASXE zqCnIgZ|4Lz?RdKw*>&p5UScVVQH43lskQ86^!an7w!w4!_n*_fN(^wUov)M)(h!E0 zq|_5^_X;j_@ftHdZ+Nw_ct>m!z+=^gq5F< z*ta9&jc_L5$Csso0lIU*HR2Vpy7FvrqnS494RepCr*6Bs9dX_jd?vckQ0j$E8s92} z=SG;tda`qoFh@0lDM&y>^VxXsU5|X@C;%Yi1hi-QtJfm7zU(!i*<66WVg$pyo5cB$(V8^1Zp}^K&uLi<=Ttvs(*(EK zwY?X9NP*&457BWO6ytS0EK~?=|EDQpY96FCTw^>E9qAA68@zsQN#!t9^;P}5tkv=E zZf0|F*jXhjq`9RfF)dXFJi^!h{?f^uG8`cwwW*~vPOhyvx@#yExQ-#OF{^F=BHT$K zOl^B!Ja#(uCxExGlKHzXs#@{g%T|{Y@t?>e&%`Pe->XmtZ#~k|WJ2c}2)o*tbKOS6 zt#o3_g?-zJ0mOJUZsG`f1YzpZ!qJN#MZ}5qq}W7~6&a z{1T*LVzNyP5-;K)&vVcr#l2-E#^XDQk6n^NI6l?#Mz5Wi7#M4A?)my;T$E(a+yA!x zm7MIEhQ>Zrl}smdodO^f6VBoV(x{9TTY#p&lVkpE`=@ZNZy!l7xBUQw-|}p~ffITyj!rdA|2EM7>a5!}m@$_7y!Ss?W5SrXcYQvdU$2%ln;QnA0! zKW|p-WK@-tgQMjeVA$GkU4<(qQO$8FV_1`Y{0tdHR065IoeN7r&wmG-4Kn4Xc`V89 z;!2W>l}Fl)X5Nvv7dkGjF1a3M*5-+E{OU9SYR$hra-QAe8WFJ8o7Hqvi&}#qyDML7 zoos}^FT3sOGUKoW>I`y=G=n|6GISuy$M*jEhUc(iv0P2%<)h2w-EA;r7-dO`h^PAC zc)06aSa@BSPlxMU4d>r!`yYtN3@NpOVRG2@0d2yH2Wc(pvl+kHWu47>BiL@DVxuNU zUN0iEez}CLkxwvHeivnHuxI+N9SPsd_0613$~O4z49X+VRQw%Xa)!XxK>MhE9i3A} zxgbE%iR5`vXP~mpy&DYf!p(yZn|qp1E*1hHst^4BW8k@D_14{P^eXQH{C>^S?zYv^ zDN5g+(5{2O#CxJ?@Y{5&f{m)~;YIUk7WEXQV;<(=1Eze!(xMZ;oIT z5sBqnGn*e~Zm&=Tj=n;BV($vKv1aUp^Zn=&iSPyy5aRfD6Tz1ON%;l0AvAu{N+6M) zxNd7jRrz4#R^*7Wp8(<}-#+ci&4N&n<(povle4$q6ycC2YOjH+rq^7484oWe64=PS zzL8H9NZ&*8wQ4EM6G88%+=1G z1Cbzt+^$nzVN_RRQ_MAPF@tZw5B}Q!O3S|4>Q#m+=@F+h4yN&CnBGlps`%|?5^15r zNc-~+epcN>e9iteW$zBM(|jng)4I^(iO!KCcI~YUFqeBfW)HxQn^QSYNyUHvWg|`4 z*_}_zh@Wy2gYW_xF5dslzTo2`JGL87@7D5_&yoT?)!4sr@yNwbky`mR`H{cVv>Ola zyp}aw?*cyN4H=84!|MNw{B3M&Bld3btv>IxY0OLJ9+3wtHWL{DKNQL2Q5TO|RqyKbFSKr^o z#NCXM&HZhd3RR>b?;ZeBrZT>j)-#Z&T_^?3Lk8_hJt??l6ktg%1!cC~^+B!xeu(KL zUjlTXqBV7qaR5%R1X+15mTIK3w=$$qct>R{hB`aP6 zcos4iW0&QC%}|c**%BCwpEkQ~Y9}$4v_W!-wT(13RDa0Y5GsL!PJXXx}x6U7}w1F!58;g+h!)U!RU; zs!tZ=YnLxtBuE=rE%r}Yy7cK%0Gn@|rpd^He*|L(T?B|rz{%X)_dZ@q?;3oS76A$m?5=vm$O~>*{yVyC5BGx^uQ@{rKH+XwC>Esh*xM!HmIl3 zE_lcO^Ty=B{_e#JL1RZch0WWIqS1waobt);K)LY%84@Lc>83-+$mlL^&+IQ=CZ~)? zx8M(Y`1zbz&u%0gw_X$^KEKlEPsn2*7Yx`VJfeigc^2nLei#X(ftht>$InzsH8&1f)6X~tj`x7^TvD=m zo_7Z-WQrDY!SWE-#)f_(Y_Qm}xrDR@^5^n~d; zw2dc&ag~1eG7;nTY0#DMegaNt$=}s?EjHF?=s5D2QF( zk!vM7JuhaT5X);V+KIJd;r9VzBas+{Y3s%FC{-LS^KJC94mevq8&`jAt#mk?GoBpB zO#>npJSI6%*J|>!*IA&OT1=XQ0)9?df$R1>gQ8@Lh=F-#e4MerkBo7(Daj}XNLqrt2j=KWQ#nWqit6ffoO7YP-$~wk(M?J%Bnx1uf1nDy z!P@$fo0)CWUsu#IXsac z9haPG^RRlkCj3@FcynnA{*aXv#z`_7!tj2lQy#e~rq4kTBm_s~TMD5V;F+jqvwEo3 zGlQf8x}xv;B9ll!DZOGUH67dd3_SKuJDyU?og-jpXycGF^!1~XJ|3$zJ6D;|a_2l` z5idxRBspLywjNhE!c>X$oBDB6kVFKcI^1iaS2;&xuBk)2RU~%#!p21@x(5KZ@mwx- zn;Q}mA~v~?o96TKVn``yFR3U_KL=oXm)L3$OehCI8T#_}77 zpO}!Q#bG5mfxp1X`TP)pjOxeT1(wb-5cYe&9tUjgv}N;|5w>~y3axg#UJ7c!i+owW zWfP<8PQZj+<1- zoWn{YuxI902dVE8&o==)qugcKKjEiOqJscFXF%Fyml05)hc=S;TdCVB4{%LNm! zQy$@0KTFsB4;&NBv2k%N87^s^l)V_BAy>6HD`?y(&WpjR_&V@@M_nC%^D}MQL z6~lmcI{bOA9b|a*<)&zd$BF)KW6!Gh%I{;BS^u{=aC@fKDl8<_UN9gJLU{~~5N`Lk zDD5u8KmO?e1_lP7e0~b5)uS)Eqt~WN&nT!)b_1_m5I^`QgvWPTCvo5+K_aptU4S&Q z(pb5kvoUX^p8a2?LBTS3<88+-S;gMIvQnH@5$-43{otV!#W7U6e`l$qv&>(lR#r;@ zBogb^nrI6_nDN%DDQ+V(_KzrFFf1uVpkE*Y!?iVZYbcpuX?*rJ#VZ#a9fv!2Wa7Ve zUdiz3UtTfkfGl;Tvqhe;t7Jdrwe&s)5-O*VM@4OOIvSa5xpWYH1h3IHejAHMA0ylxCTsuUdWZ}9o^ca^g74zB9EgO&9+G81^&n$N9FE`4 zJ9X;Odul$#WaRH6kQx+|L3+pJe(2?J3YB2Jq|_3p&EmhKDb}V3rEd;(M>QSDxv7 zj+W)s^JVeYIOPeMXbb{I?1cDR+uO^k+S*UidjZIfpT9xqH3J%MZ$s^GXjcl;{qCdA z$Q1kQ0!fQi$6Zk2MP9nk8a9-(;nP+{nAe5VPVlO);F9=FxS!67sLchI*mLLEosM0E z9a`l+=~1-Z9$NL;m0pN1zj1W@L_yurB>1B9Bs3x^YHt46>hSzXMMD$uhXEKwr7f!d zEt9&!NMzWMe&=4&>@nc>%g1d(C@bN^u2 zQI$qKo9P+by22GW36d$*S)sIZ;IY2zs)mOSn0TnDNDEZ#Ie^7~xHN5#kY8MHk= z?&RMJb$6AmCXm~7G(BNP6YKPhjOut11k~RZ+DPWh_W>vEBcSxLj zX2Kf?Z^NuTjm6+u!R92ZUKcPHCm~scj>F*70MysCUI!JZ!OGL?@vqUTdkWtoR+RVmB#ga*AdAh)Xg&Dinvf@iG zeo8me)OX(sUF|EpcEVp(UBwC(f}p3FZVwaW)jEpa;%;-=a>nvZ;PP#eec-zN? zvpYa3F-*?pN5p-<^|la0Wy=9*h7NbhG# zbo>J;O1R(My}TtWOJ5%K*8nh~8v67p9#SV+9}tPAI``Vm0oZ$06>88gyyl2em3{-7 zE3=(DsdhU8M*X_jGbG(d#J2@v2Cg6v+x0pk*}1)N-96cBwh+M?|C6zIk8c+5(pluB za*c!R>{S^3BUkHWQzE|!y64UM&h|PebpJiBH^uzK@Q9lanmNPo6mU7E6c9)oaeq2z zU}Ac!;s$u0&45CwckJz3{G+_$$S?I*Ax`yGfgkDmXT=PRk>#IQi|8ZK1E59ffn#14 zw6Uo^T{f|VrU@+8&Ck0EmJ*AZ#n#d9QMsv%S6?=kCi^d1#)%m~Fv;x})p#`qMyo9i zbe6X+fk_4bG!uXT6ySYVRbj5ljp)6i&VmJ?ZOSiG=Usm1q{0^=+!$d{B#jiO(_g`A zRk&fJBSXnv?FC%6VhYD6)wWbu+c`e3u6H4*uues||K9tQCY~N4j^q{ki_+QhyaPI) z2CwlwPfkN9Ce-&VFPEO4c$9LQqnkyAw7^uba z$Ijlq!nO~+))KVi7V|d+-+q3~CR{XCjbnphkc+&aJ3XGb*T_tuvn#P0pPhGcb88KV zjehG`r6`8z1mE;1x3svZqtjx$y?aCTUagc}Sw-bF*ec4)On2GIpomPj^r|*A#xLq7 zsL9Jmz4My^*`;$1S4`mK=V%}C5(nP2@rd;V6qX!F3SZ^rt=&N@ATTL~R~mLlQkHNc zp!PD3#-8=n8@sqbR}Z7RQ%C)FcE&G%f##z3lxd-<&-Gb+jc4mIB$DL^ruk_Kg|nkO zO8V_cBe)-@$AaFuxdB8xJP(_ved6wE80cIl!P`YxG9NNU#wpb#nuJdv#a#Ja2I@Wz6V~^BRmZZFK^D` z_wWx$SWpVZ&Vsgs6!xPTQZll(bAJMD?&BourDo?k`{f^qss&3MiM}@V@h1tN zzQJ+{UJGzoEGkv<9Q8?f>2YwX?l4s!&fo z+y9jvBP`tRjqgVTW1piMpK3pLr93s8s;4fl&d4`9pZmeP9J0?4WOfTi=Bf2%!?l0@ ztDvL`!F6drcIxZUsu97XkJ$k$XrAquAB-_(Jx z{^Y0ES=g7D^DcFZKaVUq_lbYo@O)q!slAZh`LaWOVd#kSUA|j;80UC`?{ebY!-4KBp{T%0i--X zb4GwSsLauV966%0AFTY!G{5z4*=s^t7a}3@JTAJ;c0N?@wc)ii7%D70Lg_rM6N9YC z^JcfaU`7e+^%cMW$a%X-C)v=yMC%n;TB5c5jjTIF*03P8^{ySz%7%m~zh~j)fj4mN z(bFvBHoTM*k}T%-#DwE|lM83D>|}c1yDm5P3F0Nzgfdn6=AqnPkP}a3El3N73D@WC z_qV?g2=0x4 z%Ed*&R8UZ4u&Jt$<*+wL%J~S9!BOJ2graZ1@xXkwt%(J}lyeB-w~p|bIz%8)tU0*KvzPdXa(gk4eH1vve!U&9XZ0+HO;GO$aVVPUd3BsWkc z#D={LAwup&IQtmC4Kj@|&ia$(QQ|Ov5o_{gF{j|Vg(x8q9JTq#&La|9N?Xm>8Jd_} zZ!%4rQ{X}@-Wpr&)y(Zve7PnJ?_EsO)cpF@lo<{kmYdt@L{~;wSeUH$>zlu4JXQ2D z3>f@BHzdjq4$OT!bPKSw$aiEBUuU#$OAr1>Xc&`o3=@XlzBe1RLkWv&*>sU2eWAzN z&{+8Ie?)y%K$PA0^$gwJ9YafZ*U(5wcS?8n&>$s9`O=`$rF0J6rF4gc($XRPAAjG) zcf|!pMy`bo80v4mR~e8O95u~*oyk+;wX^J|)H7E+%*z#1uY zmU3qSNs`6KKW^1zOmnoX(U4B@D8KXx{$guomP7`5lsA)ax-O>!h*y6+x8&VsZZFgLevDil`C#?h_$1scZ@SsI=DL7HYbnd{Cl9Qj zoSYn=y+rqm)~7M3eO$wiL~K0Q?9sJ&vf?g{PlSJt%oesBnDv}ZEdNyJm(#$yyxa>~aV?;|H7f^W4RpVHS0USZV~pC=qZ2tfbTd3oe?}9#aXL8ZL5aH|~M8 zr}ZVB>FxK?*3_U_h+?sO2v+2z;S^Wi3sQbtHC;CmsL}Xj5UW*N`IA`&h(=fSJ11~H z2QwNQ{X2bsNaFK*;KC$<^)+C`Fkr)FJZSv>2$ z9v>|v(!8pR5_*QR)yjaN56T&iUkgT8{?4mX%=6ReRWryD4(08WL+ntG*G0+BolEKs zmDdE1Kw!2WQ`q#r*AgxJzF;-(JBOlV8ughGS_Dyie3i^i%qBPZAuM6!kTCTVT3V(7 z9U9Uc2Kr(uZRF7~ZiLysp~FV9<{v++;3^7<=!^&gn-TKw+=nX)c<3^~H~RwD0mZ+I z4ea$1x`KAXbr~o|;&;%s*vN7}{rHvs=FOXy zcS!V+G7799WNNvxAK3MR!r{0>LnvE2F5>YUjBw5$j0p|QEC(o6^TyOL_vZ_SMyP{_ z*PCMW5*{5De5;Y;Y$O{My~C?gNJPHj!?Lk49wp_Gp6G$<0IvWq#mCUyG`w25B*mC_ zU-ZdTQyA$@V_JCxclz~|(F_>X*>=*9@10wOg5-iJVN11MHK@FEPhFu&iV^ zZ`dj%-{awjocMUkkbyA7TwlvJLaXL)P;E3L2H=-IgbMIipCM+&Kb$tL(y#nWXP2y6 zG6*jaZV{KHgRTzm&d<+flP(2zLX=0h`z5b}{=T(XQ=&T;{CDmCtDcxpfnHI=euAq0 z!`edic%~-vKI3FlQLw_t-+x*&H7FVl9L0Z!YU*e`5QB3=_XdOjpRfv*R|nsDqY4QL z7H4Pg<&A?pUM>`mr8fes2BUQ3LUJfPttts$D)57UZ`y|~a;Qd`Yg@%*8eq0qMZh%& zlpM~F=RaH=&%+mXYyz&!CJhZZ`-`!rczO&{4r%kw9IaLv4<#z~x-y-~&#B`}qD4BH z)4{1xm#e;F>z9B#3+>*6!)6Phoihey0TceNl0Ju87GHF67)sPw>}|<}(X%aOVe~{W zgt{5~w;QraU8rCwDvAj4R~MXUDD8pVXU}BRIJ6slxg|RU-QM}V zH>g*Na(8ZXOH{?P3CH%x#-X8Ii-WA6Q`mNnxl(`+s4Ebdl`NuO;b5Q=@W*?)yk5v| z+i{Z#T0ZUAz>0@9^)^IRGvq#ev2>nkw`LC|q_YcY{@qBCqlH{61^M3(QD4uF`<^ss z0x{pkXde3X*um%JNTiBc7=xx4)}G=u%7TY%`8cCf*QzyI|B@iG(ntpRNc8oi+P!@{ zWcVrz0nLC*7DflGEj35bIcrB*dv+W|gA4!tQRWH^=5+gOobHryeI1y4d*X~``;kK= znW6Xvlg-x2n_CAZjZ2~Cy~nTmu$-O~E#)ivd_U0a3>djcbyjdXtV&H$;lnZi;5OS) zpWiFI4-5iY%GN1_(Og!l$TU4pd^9~6hAoeH=B954aDqk3b2;YfXg2ElGH2)q8$eeX3>ZLAvQjMwm435ga`)mxQ-_Z zX#)OrstB*a>AJ-fLy%&5`=WC@=@$A0Gj0jVEa_W)xnQ zXIg6N0qkE~M&3=cX#V5~Wup<6IUb;eC^atS?C5C@m1inq1X9j*lpnG=eg2 zePt$J$%{wPK#hf(PL*QTrLfzw&PKAidek5qmgF%Xy$-qQSaNqm+RTkwwv7>_;d@$)IR;xQ+H5E6N~Q9XsKfG1Hw0Hv4-h~imLlqHbbKu`E{rgY-xF%xU4_o_iraEF&{L*Olk1p`pAkmH?3kGW3Q0TAwF?t z>w42HA}aNT0(g%@wsm}g^LMCgkg>UVIy z{^L9Ix>S4ye574EQnaZAjk1mp@z05H)tlMA_d}ii^DNC|7x|)AU0HM5T3q#ra}J9+zH%K~9|}vWU_2GdyweU~Ab~^tmB`QEJH- zs|M_Zzr@du)t+Jb`k9{L9#bqX#=X;U(oR%GxkOFmBk!F?m;R$pfbGwY?lH1gcs_!C z2wF2--`-B@95${Pkj*NgLH1~EVz2_zTZ|y>ynUZz0kv?ZVyUNa`}Tjy*`?f|OtQ4% z1T&m&*H}60RN3D=qvvSJnsA|VISJ)I1WVB&d z_yqxfREnPnktCgBq7Q0wWBWBXV8JM{@+VjliEyJ*ohZJ0R8ZrX++UV+>EI;F`py4X zGQ!O?31Gsmlat~=1b!uBYe(c0J}@QcLGpq5^*bpYT>)N;nsk~Xm*x$SQMx1Z{J^zcV4UXOJ==CN=$|O2&P7Qd%mLduHXQi zW6v%v;zi8l-M2g3Ht8NOSk$_z4SD#{B*4F#?beBu+9Gzo=)SPEnoe>KQYIcyV-f>q|ZBQpFqlB4aJW_@G!fPl9I?&i_#zlRhZ|d7Y@ink26`!In0W zoQDrqsfW8IiFe{*|HTgWM!|N)mXFv!oHKHcP`%Saj#zH>Eu+N9+uyAof?Sj2(yOA$d!W3=A?;q%PGW5$K*x82 z#5%tR^1gL6il8M7Sf$FEkQc$Bs1V+~CI09x;N z+)Wvq6VKu&O!e>Nv%(J(6VwnD?vLtUkmVqer?Ei5(MZeuD4GB}10m^bW##8x-jdn) z{x9zL<>m?>kPrzV%NOG9c6}59VL2Mx$+hy`4Z*Q(P%JMB2s|*RPBN;WjlhNm2B>BI z;LMdv2Mhum<)oP{!}r7LKG|f+7@8X7u^oIUGS=2^M0u|>j1wZzwOg;n36HtL7B>4c zFs|NxFEY?2+v%H&$9_Z4rJf)~5i4bdb)iaUUJ7+-MVXz|R#sO>Izq9j375knk@i5G zh#U9ZjJy^Z5#OXP0Jnx+6%~ciVj?};);TgZ*P1XR;UDq{#r`j02&q7)$tL!%E$JT|pcCQdM`7N* zJn{t=$Zs5u?^HC;R9aq(tl-Oez#WbDdx(8NNWUbzi)1uKY@mv~q_h;D5J)7SAXkU8 zh_kb^K%No3HuFlQF=u-Wjq#Il&!Ktb+^=rgiNSuH{q*j*F<6swhS2e%*D>=S{C@sY zQHvto`KU=Be=5kUzAsa2UbX%Ttcj^mR3r}Mbowbk`A($t?wvVv$uOcl*Jt*jrB$dd zn!m<6Q-vPm_&sHd3mb0pvlW)-PN(R=A_>}*3mnAbJOMA#1GV=}tsxoc=?B$;lCb1W zAki}rQJp9QF7jmOxLbU}NVqE(O->SbSWW*~w_yHfB%fR4<9HQCJ#{_}#S7i0Ir{2$ zGWqe!l+LJP)NQO$q0|Hg<8$)$z6rqD);72T^Wxe06rQcjH z#tFCJ@q=prPZ~*&*+J6XujAZGIy%GYu-K}MyR93J3oS=tx0euNoZj$-K%1aJ8=tf2 zB5Ji7t_BjqyVOnv3#%R`O|+6n{wVy(82~jfl?-FK7C0iv$`s=5hU+0+A+AJ$m)p2gvMI_xws{9i)P;$~g=le=R5#-gCkg6Y(imJP`VbULCH%z;rq_ z(9_}~{taX{J>{hqa!`>jJFm?U)+cPTEj^>mNk7AJr`a7W>X&b_=-N5KT_i7M)bSpl z4J*+)%feldd_s2{9NfwCq%^+t&u`Zc-zvU&(JrU0mUj-JmC=z|{tikAZWc(hBj@Dg zEU&DL9JTE;xIEaM>5U>2av`&}wgD6tKXa7VBm)Kda-A(jj4_vS7ht#=s!FQw+h4p8 zDDrlU2=3x+lyJ6>!lzLV`i3Rq$&{1U*L&PIugpUqj#)W4#JQjhXhcw-20WmIabnt1 z0(BwS5Pj{3jGw`HTN*(GGgOKh+0IS+sKFLB?B36yEn{9MWNp>h@+~z>bOevZ$YfTD z4Ll!L%El`}Q@%ikXP;?FWfhfQ9)U4}uMLCC38YUs;w~IsYcXqSt77?UkiqP~B+Epr zvOXhq_Xjkh^ccNQRqS|~#v;e>b4p>T^uaAry_IovzG{D!sT$*4+eH%Ix|@Pyf`lNW zAkQBrZI9l(6iTY*EfYc{)$Qpt!@Te42TsiXT18fnJXXGj zKQ-fjsB{dVN_nJe&}<0EEx(2@rsPva#i*{9))rdKv$4^>=_pKXV1$^JY=-fbNZGL< z>iBtI<<@0-PWUL@#Dwy#+Ol1;P{`M7G^tfTqk_2+tS5x;;Xb?H%|E*c3M#%xy1P?3Fe* zT^5ouci`!D(`qw`;|Ctok=>eB&6O(Qt_3cd{lj+jD%2)Z^Fl2W)9RQC3B0>w9H2&(MGyL)n+F@?2@E$JeL@Slq{Gd&g z4nl7CrNq=0B43WhEfuzefXhFgWMs5#G#;{L4ZDVtVSh7=)iU?rEigl~Pi2p{IkkiX z3Lp*x@%RCUX-Yh9YppnodnbE=0n0iVW#2EO<)_UhaQGva9_o708{?7a5NLlGzDf3{ zX(yh<_LUP=0S)9*gtbl*~G;(TnNL zIHsoJ$$Ibu4@8XcB)^qHti^%-9sz~XP4yt!oYs7SRImyP&No{2M!ru<>ZqJ(Lx8LE zrX#bk9Mf3!e#D9<+`g$3-cmMu(^4JYiWK6Jk7Iy!vNBV7ujK6Ul3pWXx=1zi=N;x7 zE59k_H8C!|S5PFgz9~Xg_^z3&s9N;-Z^n4ZPU5t^M-4jyzwFTuEW2q+vl>9~cT3_!f$px$(d^5kO}2_3kuQ+(IVSDrwz)@7y^r4LFpj}^l04`r>=WH8_*?qJ{>UPp1 zc=pey*#>7dWPzSs*WEf@Ut#5lUBAQVui_uRTUY%$&t#*p&08&Q8dFw=qm-(goFjsC z@DqN)Y$_~`OZ>Vf;+x0;9}wUfPK)rZ1nt}`vfsal1)p``8kqYM8=?|zs=Y#PJt#YI z&{oR76qY9_z39b$6KULQZo?u1{ev5U%eMBmZ>{Gq`M@Cj_tHZ8-!yL;o?09h>v`Jp zs;X5pL&GJvFJ-`~8WyKYQP~O+vpKeIY{PDmjv(NSk-JErb(kRS;yk~<1a5uEAwNGi z*aj0c;OD7ljDCjzbzZl)`tx^))1`G2#`#eClk%f z=o;tmpuHlRyPvVuFb&fS{V;m-k@fY@zVFHU&xXHj#Uy^m-z)pXnuoUk-8iJH3JKFc zyKj5v?(~a``f(vEGgfbm@Sfo7)|+v-4nhIsDFA)v=77Gkq<)fR9(yQ-GH$pq00Ed6 zxO(WGPo8Z3Esg0ZgTHyYY0+kycs%#Bh?IuXX8kD>rM9MX8!(DD)dG9jExnrKdXhXLR{J|D65 z{G>z40Gjx@MJnc!7O-xss8#dW*F5(o<8cM<=ZQ+b#YfOOCxl*vIvF9}0QKzZ@4?`G zeHIPwp_sW454yMeNbFZiB-^Lk{l9m^LtSQGXZVsO7yEm=@0VFNxcAPg+y)jqRC&Vf z(b;purF>ovm`XhjhCsYVXlwhHuL)~&-DQkQIq8nd+{&SSVTdIuB(0f}h84Oy4;BCe zu@=nT*WWe%M-B8WV2|Hjg>={$m2%NVc)s7XEGtR#n;Q0-1Vbjm{e3=8$lY9*NPBn- zUaY17*@S9d4@vzzVje^V)z)akwaGZ&?Q|Oz)Ee&@#!J?LCNuqG#o0FRMRL>HD8$-{ z1Wk-j3c2@tHc;PqL$6Fp7o*S&d*i!(ggM3Eu(J2{_hSP(iJB71X}YVS4}(W(|b2>c&*2mo8;a|lP|uQdz&gWCT)b?%~Yq2j};K~+utZV32gnv z`geXsS;EcM7u|dln?{k;!@s8=9;>gy?l$KM2O#w4IXe#||1OKtZ)1e!l*%lzfs<(^ zmsx$DuJ#~IQs?I8>f{AwzZC7FAzR|T&(Eixvb*QUTb~)!tFVW^h|Ky4qtdECkx{CANe3Ji zJF3~erLeb{=+S$*6SVjS$|GAsr|VWCeNhpo=|of*>0b&y#^b*6=Ll>U*>1lN6C`k7#BV~=_?#bbO{&V!h#oY(Z5+$JITX*+YK|ZzkM%%DW(HY?QRx34v znPuiD!MIL@`%<$TIZ5|@30`y^{faD{W4)v@X&K<6a?{hlPE!u9=kVGv^!N2~SwRgz zt$84E#OF*c6LJ4#QEMCwqysrtP=}~kK35ukD6XX{sev8K!IR{WVZOL@7VIsHcEP+; z4M|skn|`+VDUtBZf9^bud=LIB5jTP(20dRj30(zr$+bc?_Z+B2WB6&k2KCs zgTS-orR{^#&V&EI%D;o~!qlT*D^*@x*OnS~)?sl@UDkcv?b9{%eiZmLs1T1XH`8!m z)*E}&J<-42z7MObs{BlY9xTl8FoMG)_7Xx45UU-)yxBzig`#r$K~|gHB-e3`tAdwq zz|4(g@#I4)=HEg%>DhWo)ZL~{c=Qi0z?1(g33E^`Vys)EpL?1JH;>D*7aDauFp=tn zstTV+a$p*|5wW|c2AVD0iqY_O$xlVO>5^7`mbF^1ka-$0tG1DBFjca$fqEfJO}&`a z(aDL=j8&u(WT?0QASiYXNY=a~`9SLtdNnEBcpAhRkB4lGAtzbgU8LU$BVTmcPtqZxnKd=<^gsW}NQCKB z=<>udPV^eKLR73kedGsLAO^JaRmTz}0sc&<+s`Qd#RuX!cYNaF;t}2jqoUjbxs!49 zoxhPtWq@PDwrqSNfD?jR#PM1?r9dQ*&C|X4gDKQj5cne~Zu{_mUU4ae{Qr)OK~SR+ z8084M;o!fsQ0Jv>bM){nk_LmkgQlz2Wbm&zsW7*IHIQ5jGcu_vdN;q6?G9=G71Pyl zjKWsC&9>P{u|BSJIlp_bFH45^FUt>nqB~~;O3;6?n<8^+&lWJO{Gj?B9;`F0< zlW}s#%N5I7$T+8)5h9KAUbz$YbEsWYex&&*+qpC}n)~GfaziGJW_4$l+glA8uahtK z;2XMWGpz;q^_6WhoPXpfm52wO@n;F2?bj z;^A6BEy4M`TmMqU=~~m7GO9yL_%Y-XCz1OBPil{3h;gMt}*wm=Sj~nWqk;c zCNf3C%fGEz{`m2u6d>StGV_MpAZ+HcM8grvb(k3tO4xcuMFnaMgVvV&Ku}w5+UI#G zWjH0Mo&Cig5r!M@kcj%hSlh{|QbpaN;8G$@pki$vgEn6PJrXw)3E~v#&wc=HJ$sZm zuB|@$D)vdaX6a!mS(a%rF3Yu_bTzT7U5Jwlq0T~x~Wvx4CB+|B1^ zGiNyCBvmr~7KzCb-O`|{-i)jtr5Epu#Wbvyhv@`rIbf{C%*HmJ3h zy#01D^24K!o+?l~qU_dvh+|Oy^>MkPt&s#1BW34N^0SopuM{1QhTqL>Qjdz%a6lSuvASL(=iqp+HDsDcM(%)Bq^pR3XeEmAnmJZHFQ$d@GQ3MZEMFoq!Jhr2C6Z9P?vwjTyrulURg`(%4nYId7 zul;&;R?-FlRW`h}26?M{w668rtLvY0T1v@Ko&O8z4*2M8##~Fip`;Fy9sARV@#Op? zY)y!ZSoq7Wd~4;A4fR};m}mbYiVJ7mQ5GMzw0Y#u8fn>eE#Fg7?pIj18~(IzIf9r2 z5Gu5Y0a}pO8OY@;!1i{QHQb0i-)47cS%j*il^!iseRp7XTWPEi!sEy;D5yqJ@RV90en!)4$UL#7G@$r%q)_Ul}rrQU_`E?67~#mKwSL0 zH7c!tx~J*|Gd!dCkiId#0~4$OEdXju5eQ;c(lsMUY{S*Nxq@>(nV_+>CW~seSqHz+Qk;CsQ;5? zZ%LP3{A~Go6@IA4O7yk_xylf|3%YTc9xBJL~1KqEQu+bz}ta!?Cj_$ss#{6cxJB-XT)UU={#ngn!3o&2@!|99 z^vp#hls1j~omH4k4@kzgTS_0S7Rtl^;WCvjiert52Qk{LuPbgGCVJkS=`CFH(7j^6 z7pV|ivFE3i$M;0d~ z!(6%dm(l!TbShl?UexIV|E>3pYiw~^UsDy?+S_6sq&H&87?ZAblp7r5?7_*)o~}Wy|C@!RO{<}Qmec3ZCQ>V;^FmM zab0%yA*6sl%3GAt^Qd+1l8fLUqjY4Zba8T8TJH8Muc9a1MPy-33Oixj=>i^y_Agw# zkQC_dV>$Ds8x zOVE^bwUlZCq^%NwOr#7%)u7Y`!EIbj;l5dP*$RSNx4xU(=U6~KQj~eXVv=~CO)Rbv z+U*xW`{cau08wZxY_0vY5qox%v2%H%i(^Q(Y|1_!$65kBH#!O!F`4rq!LR|Cuu+5d6VSFXa%YW^bj@dl; zjdBvF)nArFv<{(~-o=*!YzE#~1U!UUSH;A=OH0T1K3dA4%nz0_U5O6g0QgKQd0?TR*@gk8 zfI#S5@p|2m(Lmlcj~5O1u+6g{o9D#MoDY2Ec1BvuunO9P5}@DpEa)vzOSSS?y$OW2 ze>8gcvL3?j?0>c6Lm`7AYZAPR*6--6_-}hw{GN) za%lvsRdmzknR$sK>%)iA#$}f-h|y_(#0FNm{>n**@X4AHN4uGaR5St)?4Q^LJhXvY zbUg7$pvAy@!*$TPbP~~j|2*E}viu+cSD3yp)U!(o1a%nwJM?nB7&C0%YkyA^ss4Dr zUAEwJnHL{1Azb4P6~QksZNi-1Pm;-aI53zVZ4I+HWA6n%rN@Yw`21!jFX*Myw$Wlt z%2)@%#F--&i=2V^J!t=%tJ3+%pA)y2ZeYX0_$z7MsIy}mMP3RjR`U4V?iZr371pJZ z{^ZbT2RttRkh4}tKa!4Ev?+Tad%j)lm^EbVEAU68BJil`bup)zNS+uOiPvM#3}wnz z>lOIV25=S0EIC3x0hnX39H49!!SDJ$+QNjO&r`yU?$PEQF_4qbmZ=EHyx47$`>guwiC=n$CUfPTj`Na^TI#o0Q&tfeKUFom zON4a>63k(2;{FE4RwSMmd#pE}?%zXH)M>z2;MQ#pL@!C*uPRAas!jx9;+4|19l>7dt{(TbMDCUcdoYc@LBzAObl+EE4m?^~+BvUR&W+oN zCxdry7qzKsopH+BujA(|*H3=Th>J}e*wM1F_l4eWB=Xt}@0`8d#pQ^!g9ooUb0krK6iayxSYO&>cLgh5LD9#^ZdL~{Mx?yUm-nJXQ3R!bb(%|>Cc9ccwTc|> z-Th3xzdQI63NCwkqe39n&GmzOtmn@kuYc}$<+Zi3`c=dq6ou)Tgb*|}ucbKr0hz9M z3w3W}s<-EC86}VZD!xEqg}MuFlUyImqTSBB;;#13E@yMt*a@CLcksrfBMw$srW&@I z4r2b)JreZBlN?G7J{+9AZ-&D626=5rCTE^~FV^?2#GSTg!ODIMi*b>;nq+3v&jFV%2rX~X*N z@-db~DqszH&it#d%L&U_$=x^@%2{IxwhwseZ;?Zz70$XN*YA&FctTj_SR*zw?|Zrz6rdYf%hT~8Qn&G&F!}Tki=0B zpXDE=qjTxYVw~X~T@J`Z8yBJh?h%bDJa-6tXr#ci_`)p%+)@8b7%=iXkQV+mC;=}z zmCjNW{4L>q4O)-=L)GS(fNbucd>%hp10e=h+&{UCaxdW^nO`5x z`%xX3W|igWn6J_=7KDGbOi@kgJ<3i2h=iJ&n!EAR)E!H!?TWML>%y};ziqjvR$Iri zHD_gOBAQ%*iG#h<8n=NT&VE*N;vAmia)cx&oR?y=?^V-z4%*e%ayGb&9$lecubzvr zld+bN{soS_r_Y!TVZEmGXcm%j&`HR+Md9wWhP8{Z=QA5DIHAD+eF%65gbzw8e*wtpZWtumQgFy}Obc z#2yVnI%Oc7AwbB&A+QN}WslXTr&^dfO6&x3)qj2D?H#4#mghb3xP$5OS*-bd!_s8of z-Y*8#7Gr)RRO6T&9>cgKw$JX^=IfofpRo%}L;$J1xIyx8HAoL3wu&g=X6HgHRvE=5 z7t_NF4ax1B^QG~tzlX2s(P;a~#Nm)%k+t1WNYrHw-2;o~Gbw`p-I9U`d0ySWkINmT zfTs$uh%7)AY|aTD2M_r5o!BCD+r!^bv3ff5p(x8Xj@$Cp5VG|n-o&UN8Pfv{Xf!j6 zEDks-F03K#?lMDLiX*iuOhbbAERj1>=k3u#?O?Hd99>l(LWvZxL1(|wm6H?C9owMM z;ZJ|hc*v*KKXIuL@bNv-Vwz^zP2^KPEinFy`hiE74HGqa=W&0K>+xx-M-Bb+!1)yM zZtbCt!dd2K8V;jf8*ne7KhQ<;$grw z=+vsZVqoszX#c(w)r3P)JWe`>7MF0y(0_di`iW55bx?XYp&VX-fGtQsRKB!^KkjDG zu0O=aaX0B<`E$V8MpwOYO|KwV{Q2lGEh5M?%@eF$2DDg)lOjf?xNl2wr3bz<*$=x> z0K$Nsou|#4mMqty;@E|v@MoeVD=#OL?^!3rzB#=9hzm5eS4WaRKG>XFE^KKy(z5hD zl(4T|L?JV7m`6x+Z>B_!p|sdt-em529z-|7e$@fv7#GVS%hV2jO>AIf1pj|v$l~d* z=ZHoeUhLVgcwTix*nQq*oF)YCB8Gt{1>zW^Q8rFu#)yU-jW9EP$j5)ivc(Uo)L1&G z+xz;21-fNySJTlpwEg+jw5rMYN`GtDNxNF);twI(+7R=bkyc4bK0BiCJEh{LDb+Yp z)z9~U{%FxWYMfYtUVE|HJh`t=4$sB|%HkIo%0>Gf6W8uh3TJtTHhds2Rm$nsy_CCW zsw=e<$%H#(-fl%i$N=lclo-N6KJD~F9t_U)UqL$pF;byP7vFegw4Z7~xo#N655 z$k(?BU|QY92_!V6{GubPVwviBFB*=N^cSh9^uHBD#YRBA{u+Os5zPuIli0K^A33xn^|YT@K8HFf3ta_G;`1yyMQ$O_XM}I| zNpUNv3sPelRU{CyKVf>sT=x~$!nAX5z^ro8UhELiY zkf(!po?1K$Os*T|+HwDm!2nI73@^-r9S?}b9uc}y`Bp7U?{ZRvP+uBKd|taqD7fy2T>=4P*28Z{knem zRg#F*o1_JLK>f1lv`JTD@DxIeP@m=bEP{XH`N%NDZU&X!+uKV3ZU$cDRn7ZIAYf2A zWCAw>Kp&nj11WW!t}1NyAcyI5dx#3%aF2}|4wei56A-N~; zAfm+M)sGWn*M+2<$RS=**+uS+I@e&Ac@3&c*Kc$xNm_9O-?9lPBfy^a!x<*N=VV?| z=Bb=}O<`^BiiH7BD&qBOs{knr)Rs)G&m4$ffjYY752ojuU-S15;Je2p?YkbX%Cjy) zCF=mMS8_30LAU5)ioQnZz*f-xcjO$y^VFrC1t~OM;T`&oc83HfWci8`-1Q24pazIH zGfT3Odf&;#!(0B*MyOvthl01b!h6cdW=)V@q1w0^ovS2yCX~Y+8py`S$MZ{C+F5QT zo?7}f5;jrk)RpD8&^2(NtmKAhjlKb?A{lc+yB@AKabpQ=8=k*6xw0{tKcHt@cp?k@ zcj`E+7j(s1kSj^~b*-7PmO33<$;B&sE(RR%@*6!TBmL(OuiV_+3t&Tcml|yB|JJ;{ zv!$Z_7*$cB(i)ibl2NDz#Dve+(uL#e63oc`V#}|aszf|G(M>phPd#+^p7sfMq%atl zLKSH@IV`?CGVMHe*_At=Jna1=AC3|$&m;EmFe)kVNWaKAnf|P31N!R$mV9}9O3$-E z0lLT}|FEO^uuXqC#R$5_A!v9(BK=fbi9jp;NJml05yK-E7e=>&HWZT2S&0vSi@v0k z^qKc;2fLEY!{((sR4pYd)0NPS2~7wPDC3R-g$gitatDUH|LrKDXS6w1!8!h8S*lm( z;%df55b6`od}A*J>iU|7_%{nwB?`_n17Fl5*cYwlC=2?vI;(Fqy1Z5s~6U1@Um! z#BHwVEjaY|k5386oM$2zO>WCLetwa^ITg`QWsdWomDNZ^yi+^o8~#w`zujBd+A1{Qix1IzMHg@tls6XnD8d{CPz#Kv+YiuqnQAk6-2Ae9_{8<*NBa4kz@`a9*W;AA36Biw_9z3AejHn@qp+U+h=n z1(DL058x_Xhn*E74(Fa1EQ|Z!jr~ zJk0pbjeAf_H_c-Hq1M?AJ`vI3tg};(U+*#7+ zu!551P11!q3!1~Bbo=_*Vx2{*eL>hmhM-xI_u&kE=S$B^{&MqNgPBhDbHn!C;fi`8 z3qS^ki0+483<8^lJpsSCO4JC|z%pQz6dmGr4W14`qkK2jJfrsRDj9jpKwPc9Wafo( z-Z&TWj10G=eem?6slkdp>5SUHck@I-KlN8(^>BJ;EKR_4zEGT0?V{65sI?d| zTDv3ziY4moFCRq6&;pq3SD8s8=f>^jD&XM3^HDuVB9IU0$RITT zZO$T*c-o38uYES+Wr|ZM_Ie@)9j(dY0gUnL=G&HB))Gc4i2%IRzt?s4Ms`9sYD#<2 zRj?l!$=SHSVw^*dgRnrznb>!`3>>y%Z%&Fb$O;O?5UQgvroW?578rM70?NKMfOFYa zec{{&@KoPQziUJN7u<6RMO#N34k1#DyFl+ek2GJ;zJUy=vXmBva|Qu0&}d`TI}0n! z{V$)d+OFx&)*0V4k5RP->YM0}OME(m1H7f^NPnI3ZB*tPCPHspVOOkJ2;ZZuE-@`h z-=Q4YdpchIQqMLiUc(wDz>K3AsIY?p3g!*qOmOk_>sQ9lc6PyUblx~a7(9kbKDMB! zCkcX=(2DYSg@_6Ubqbg2EP9>^Su{-Bf#(K1o9PZ009=kb^UKGEd{zt;mH^R!k>|Y- z!oj%AI%ScUocAQ-&Hw#JVRcP9uy!|t*^!A6CJkzOANwy6qZ#li(|Z+&w)-Qb%n22j zFZ65iBt=vS#Ivuio)%WIbNus-OQI4bkPN=YX!Rq+F9<#rby@zpmvokznisn;@SJa6 zD9lBfNFEjR;P-9wdn77tz!nxV3#jpmD&D63NcQ_jwfvVKIDI+h-II|{*`i_J=xF!K z2K~iVbfQsk55-USuF|e~gs3V|SS+EbWIaZ_2=JZ1d-YlK){hNAYXa@kC6W{0t1}=W zZKX>mo0PziU)?7ZKwQqKv+gN8A90Z;qO%woITq{2uar5r8dCd|D^^5GQ@|(oX+mEF z?27}Gdm_TGeFg8XmQg*AM|x{pLq+5Lsu?ZGU7}zDE%2&fFqVKP{{CpYxc2q^Wy8QgLi|p$xrmaH2xgw1nO2Ny!vVTu47ElC z(?%US^XxajU?W!cyEd=A7Wa#nA)$w7yFb@l11e=T#P{@{=+9B^qX~G3EJ0Unl9pn| zF}pPFf?_s7bb!8=DCOqEyr7_<)vxcL%j@ct-o9lfdIt||3X`zM={9RUPdYQTnicud z)%>I8E)uzVU1rMIOUN+}7G0 zZQTIN^jhjzdWnXK6TYP#TxTbQ3q}hflhp+k<$&iwzsY8>;2Ka3kPwPMVZC7$SPNwc z2Vn^t9x1f@VGyLxTjI(x=7?m&4$?k!tF26`U{d*xL(cc}5_pj1QmPApo}Jaz#{rIx zzalc`<|uebkQv-fdGZ`Stm`K<2!PPt%!R*`&=h0}S;BDvvwWNjD0(|114SrrYs2$1MoF@pMITwV$R=P)r~hvnB!LElnn*nilcDMxTAF_)9R^pFapCU5VjO__U#S+eVuPssG`miwTp% z0D9ECm394KkLzxqvf8lwX+kg5hhglSn~T9c@N_2zTSK?+;uL3PA8MBT!BYn}ABZw_ zVGDjyXmRF@W6($cs5$PbGm8eYaN!=>Hl9xRWo3$0VJqNH8>4V-b=qL=%(PhA78;1H zI6wPnBj@FoifgdDr^YYO{tFc(Vhd+ZWJ)UvbxO_LEoC{A)pPW;U#pFPo%P;F(FH}t zd-WPNO`r8Hxgmt@z&oH0DMxE7eS`oqC}w=aiiVpLCjJQ^ z-$j|dG|hTZhKAci6BEis;2pnyxzzamS4Qf((cX0Yi2*NNclW}KoRPs;Ax|2kWYWz% zQ#Q|is({C;KA;_G{U0KPy~;lYD_)6$EFN(15f@;G+8~ma19ljsB$N#irFsc4Y^XWR z*9wbYc9z3HaNkA}5RI(3psMw_<#VJ+R(#rSUiFv;o+uJ>T|=7^ zUt~FnqYV=|Dln}WYDnTIw7#Rvglz8~@y!C%ciG;GvGMtn_jK>Cr@-^{Ptd>{_pKzc z7MAJ8MjEM_;nMMz(#BrDjZKJxpE}sVzlmw=j%ghCj(bI^=0tt^i@Sfe^+7{W>e$Gy zqH5ApsdtG|W~rg`$<*uzo7pL{I#7q`F*^{=GBqXG}LApDn8xaBNdg<0#`J}{I58aP5;Y?C~YzU!rvo#WIhPmN7cNOH16!$-j zDzADqCi!{hzTgJ%R9GCN2w)kU{YG{GdXi6@g65%aYaTe}evLKya3yWjU(3!mB=g0M z4ch3uVa-v|St6x#SW49+DQi%c&UqMmgdXzQ(TiL*Uz6nArPKk+FZZbM%67L zp!YW4iMsS`d2Q4TBlB?>a37T3&hb;;_N*EOi3xx!IF%*+wy$&gaH3$>zusmiK^Ap@ zw{z8DUd1A&vOJ#~@#c_KPo$jBOTx;B(U5?yz5Utd=Bg)+R)OqyZ$2!kcNuX_M_#9i zbHdo!<_afZIyrb)hD}EGAu>uaza~ts=*y8-*oWt$6n>Fi^?o>BmW2KLt-LG9jlb4g zX#4pH3&`DE|7aEDN7c|QONm9bKuF+XC;1I02$`gGrIz06ms9P^*9uqHmmjvp;C|B3 zn02TxFRymbeus`Eb6)b%wsxKss;&DUCWK?>uHlVD*U?`rw8&C5}q{{GN08oCxM*AOg`xuZGB3~StY z?m)t67($HIuboc*I1~T4?r5h+JL#F|=;+3#mV|X^-$-N~h^iSK3c~B5iFpRnY?**C zp4AN`Tu?ayeYzwAwo@3$&0UFbYIqW~2ZCK69)CWi*xA{_!j+KQA!f}p5Z~W?f2TPf zunMF`?b!v82#ykQdVW}#*u?m#4)rT~TT91>R$#4GZsesXpWw4gQ7p!Zj*bSros^`f z0BGxTXm5QpK0Z!PNh06`D==M5MM3fFh!ZRtH4YD-NeKP0x&^CLH+%oyJ{Lt;Xce{a zugmD8VE1&mhizqSPJR~-ijN1W@cel=D!hA5}G`J5&6mxnR^Qt-H|VAhKW-+-V5?XM=K?u?&#eD?D#6 zQ(b4L;E6j|TyZzO`aKT~ND4J5c@;tvLkEMb(1};+u-Np|kP0a;^X}_M#1b5y@E(Jf zzv_+oZ++n>y;kgC!wLRH_>ETq4FHm1=hmXfa$*BvD+`c@RF3th(LNCz_ zh;-AVF*7p*;%hbyhzhKPw}SvewJtPsY(b}GU99k2yp#Eh-;?q8q@>CqnF$6&WeL=7 zx7RcXO8K&<6e#z*-B4u(iMlELfSIR!3t20g0p2ZL=5!Pkl%qTg*c}sdvaK7~{{ZAG zD4w4H+Vp136UBSvlg*7>*f3@rqjH^k68Ycl=S?hVttALJ3+wl7-k!?lD5^cnMy)?t z(~=)?Zz)8N2Vzu&v6H{6{b1(L>(1(Sd^_JJr4gFT{J^(1g3{J4Rf0QoxOmnvI(shI z9_QYjZ|&tm4!_1$_tL1DJz~DfieaTYNrt@Nza(OJVztquMmWthTYOb!SF`!G3eAJ5 z>-XmJjP2$h!Hf**Si%h4=w#OM039N)QP0!>__!<0QO4-5E2Eif`V{dkeCEL#Q)b;FoN|r3^D_mZd+16EQp)2%{!CBFOauz zQJ>f~ij?V-(#N;4rlnWT`g{V{>yF zFDEKj-2KUu>p5$p;0Lv(pe4o3#t+Av?;89f3LEi+>-tgeoWHQ3lZd{&i^REG%BvV~ zdyc+n{3gh}taUP^SmZpjcICGHX^deT5}%@ zKsjNXv3?o%LKF@e_4~S6c(o>1gj53?IMP6e@b^>(v7lY!n)J__Zb_k zI5RO9X1VaexqnSLpZWq>VyTnc<{T*%CDH{P_`;ngFDhH6W|!LSMsCkU`lcBW`F#U1 ze@&X2no9ib8)SnvnY+trwuQWf*@=u8I(ke)bztP@BL>V1s79mpl3YXLx-UhA_yL)N zi>32Nv3#b>1ybUofv&2*c@{q`j?=EbY^rcRVK_+9-8cA2gtTP)?N9M&qMZcS16kmV z;kyEYwe=*3}n}xT27Y^=No+#bX|rEp{#Jt=wf-)j`cIu$C)(J3hT_Ui#a4JH25CG=XC&ZCGt( zOOugF0LiB-Mm?ByR(Db(H0byg+3bKO9%pW$yTKR;ftV(vtiJ4+m1l|;A29X)a~VjP zqiCcaIK(#jh^L88EvE$WXWPipHS_e@#zy}OvN|^rFU4}i?3+aSW-Wh!8m+Agf zUS7^kmOKhNB;KWI(7=MXua;TEA_=bHnSIzFPRRc`FDkCDb1a@o!%HPXN zdR>V=N(kxI&QB_%cj3HV!h0}Vij)a6R<>9ol&4ARX4hB&^)NdaYD}1Vu^v3Zw6IkD z8r%z~zrVw)aeKMJ_nH!H-0V>J`M;%}7k#598_)hyRu!AJ{VeFQ*Xgu$-!WgsNt}Th zs^0YReUubj>O$?%GhG*Es~?iqecLdpj*ZXlFNBnfDe2;OP1J>XHX|%=TX^kTodu;X z!>?P>frxx__Cz@O^JEG8VW)r5;M}}E3Dv50pdcT%MhF8Qz0YoK9d@P1Xj@TmN=M^b zPgk3hu%A%t3!MK>tN?sh1Ial0OC}@uleag8CJqV2S5OPgbX8eRM8F!@y4p@s`73>H zHx8dHTDfmH@J-v$Jrh4i*FPP0AT?g=PI?R|#epIVv^;|2U4va~mQY)V?31EPZ=SCA z5`!suEoLhGM`mV4oS8sSI3=WyeZUy}GLc`367R+?HcT`vAEl3+_sg$!{2jqgLWPEP zgDCQxRl^fa9ulmC$M0h=*+Ry3);Vv;L?XpOiiAt_FJq1fIt@%jL^$#Q$IXHMFKR!{ zAhm+rt6Yr{SiTb?QPJIy;=ViH-XxaLJU-Y?BlVU8Tg+uPYKmP%Y?yd%)K7?=3^72T zmrE91Vt4jftv};IA`5o_jI0>Xr-hyS`(FcgxSNbTKf?ifcKk?{qqX>kW9|BQIkN4z8HQCe{aj{NBIhwWxJj?fUdO0!B=}jW(-BDDUYWd-y z^ayPptX&hGcs#Z#c_5bSf@7rwcQ|1is;zd2uWgi(tD|2H(E8m-FFQLq{rLFtV?JIt zyL%m+d=!s>z-rDX`eW+6WJvvv)SsSkyOAcXkmRJCQKUO&z?dm2Bn9w5I6Dkul{rif zT`6^B1EhsuBWf7UOg8V^_1hTx9B^;HrDxqQnum_rhgoP>!Ul2e@Yc7$gfVChW#adT zl*VPIMO}q48aa==)gR@-vfSsx{`e#%zFzxbSq)~pmt7zXowO%WQ$MtX{w5Lfgl&Af zoW+?v8u9=6u@EqpKkkgbjOb^K!aN+ak+pECY zmD}FFWKY#nP|&CH>C-3w`u6rmOYQkwfFg^)cwd)0FyMZ~#(JNN2otwz9E@qw02!00 zkKvd_&cT2f()8_8aq!y8_&m-@s;j@7y!@o5);Bsb!sta!N2V_KNqDImo?2|2ngK3~ zWkVucKFSPoyA^2&H(+--S3G(4rQGgx{bIog^u(Atp*-gcbazZx`8w`>mi)ay`22D*T3XB8f^6 z2`_bS<9FiswSOUHzvG253wE0=Vt^V30BA>3JjrN#(VuHNzY|;!_2o93dytjW3xAd0 zwi8aQr=-w}rYk}{QJy!PNDq5@xX!%S6QUl?Z}z84oed;(dEp}K+efFb^zw*<6y%lS zwS<)kx0=72a9Sj0>sz~uy<9K+xLej27E@E*o7DfVZz#=~WD;}wuv;SzTz^BkZK2o6 z?-8x~`&k*0=|x8`;A6@&5`;vg8sS9rneOk8WOzH82w*zPCt7pBs+izPe|5s-Mq2gq z*)8js@us6hX&eY93}04!sxbHvbrknV*!G=6Lt0`rX!kwj1|;d!dL z++@3`VF6%9h6sf*w9U6t1yBxgQc1+BfBqCo8+)nlc^@B3~`wS)!c84FGXK=IHwoD^sL=wXX!sw}WLA$)$}SK%qrh z_Hu`_MlET{ZZFO8<|s2ke`q@%Xvsfe1{^*|7?t}Hn@4b%UPcVIAN8+6rQHi_bVIFeWb+;#zNn1O2FDkOL^XgP6w

2BBZ5KCjR)rot&7cPQt>( z1pZed^euYP%8YnX^w^ykSOZYF!Il-Uhg1U_M6pX8>y#g%>t6`~+}XcF&)90Ro}RXV z8^a=ugq^yh`%D=@&CpPpn>%dE%XX9W;Q2KM}jYV-zy4KD+$8etXfE`&OUJb3*X2^xVh!AzAjrnJGdAE0odxU<>_e&N5&# zwO&q)`@%9`crLn?7EiZgHWdBc*hrUfUOKFLJh@cbh;ORvWD)1qX38!JNqUm}A0?JV zSug!5CI?PByGJHp5H^;FFfuGlr;J($KQJX|k!;vGpI&qUG;elOJ0qgd)%X(Mz49MbIW-td)kKlDvU61V z-gR)db)ULZAiWQptA^~8(evTD^uJKeqU*O=}c)W?i~@$89yJ# z0hFTw!%F}!pOK-m^~KmuSLaNGdo%*uZQFzk%yXe$k#9z1#g^;NjhEy>55=ju1TOm&J}B{*dl}iLGN}1g-L}ubZr=#=-mispex3 z{pA}uN*v&sRy1X~&V?5jM9q7Vi4qa-KF9go(4I&W|QLQ}+P~L|_(!+4{m^d04CT&)X*a>ZnhmZt`&Dgd^QLBNqQt(@3fS1MYaUX!w z*NzTq*~vSdxg2KE{`3%ey5)remXq1AaV)aoCDCDUE6UM zhJ7t*m%Tv9$o9p_suTQWKws;?*?*S7Xqpp?A(KsHUv5s`goeE6aiAmP6UK0KeJmRf%|lhx_|U}m?uBIO zLu7cDzA*e7w$*3w{Q|>cerJO(Q_W6svAfLhMc2zSfvc;l?UBY$6@hiR?$L9#_>f_S zECz_RE@`vdBL7?kNfioU zb`Z_;IGH$J!5tt>PXJ`=NhKP8DJsZG&@AIlqlqn8j-^h=pixuG@c$@*!D$2)E*!1y zq|z)^Ctfw@J|aJ0!>bu0DaD!)ew{ff7!SiBy5}MQ3LN%Hh1|G_$G1hXOf+HmpUg-< zM6t}8nXj;AdA+y5#6}haWEKO0AM{!!864CQO`a%Dbw;?bE}DFNGy^X21xwO+XeMTY z7_ zyMu3;^!Atf8H4}H+^|5BNlV0Rtp_K*B$4f=K>sE_=t}TuJVN}oKVjd_(Gj^6pk#5} zNY#9J5q&g!m0C&#UaJFKd#d=&(+Su4T|Ckl<$>Wq__#}|f_ZzMwO&9G>YI61b2DU6 zfB6EC=J4Z{*zt5ezBGlRBVaI%6E9qV$O}K(T^)Y9LJ^HsvNdg>6=T1m(=8?xFkbd?#WT29kvUi7&_l>w9*T)TRnYJ8WP`?e}vUU{Ixo z?aWz`js4)^k6);=-cWWmq$qowLCpE}=Tvh2L-FMH#8q=M!e#Ij-AGeQU{>0`FElb` z&p}m(+elgsPcGX&g1KACyR3ni2PbKc&Q;ZY)l}-QZETw{%`aICDR`Rs!|NRpcF|#H zFN%W{&X3*~ofprvbSwhoS*supf@5|fOoD<0D8pQ`_#eaeG$!{?A)ZLe#=n%dUz1=neA3IJWXm zLJ&=9V!&4Hmb4;0^fC)yFqZ8Q7`~Ua#er^SFdBcmLKgpTlEQ@* z;pjMK`*6QC&?XE6^CP()3mnuSI4%_8{ShV=r(uIG?Ei>+B;x}u&PlSxMr&MVT~gDP zjHJoSWnJfLhc{ojM%ke2%VIwWhTsFy+F)8bzrW8;V(o9VU0fJQ5|1z;(hL?P{LV56 zW8FvP%Hm-)&YFFl3@{?+tg*Uc)ftG?f&lp^J7C}42xmsyesa*&&kX_ ziB(Z3EJjH&;jV@X^vCP;S3U3pYT2wTgKV>_-p=u4F~)wVzh2RY0>OGTQAOIP*wAY> z>bOjFW}gd5pnD%bPCo4Qy&sm_Qpz+doYls~#pOrUZN%NyWgg|OZP)qr56e&sE~^Of zk&7@L<0i^kijn4vs|6zzlmoaP)43Y&GiC!a!kmjQ@)(L|fcV|FXw63$%Cp#S@Sj2< zwJj>8WDR*Yq7GaRNKlRE+KxF_g0IGp5|Qh6Qv7eqv>)<`Q-ctns70aR3?EjI%j+vC z<-jvN5Szg%qW=~6u=weNG3s{((oh82=tJDHT=2*SFikdM-Y=mrbstDyN&;f=@O@U= zM#q(*sw&Qk?@p#_wsqLYAS%Jq4_6VNm^eD!GrY^(D3`iinBE+9qS>Y5mb!T>ygKSx z$qEJa4Yby;8c5Y4BNDG+FF;iwML|(!xOV9MKPmC;j3&HV(<4dEmi_ef6kn0^l7qjH z8@c&p3Z#Lv;QYKnKKKLm6QcV5PsY=A)f?l*RwDA%w9KclO{-Yy2JD^47G_b{ysC&@@$Foy#Q1sx5Vw6Y)(Bqv%~xKUeBipCfH)SC z7+muOhl>+F^O1GZN+cyMi8qns8Z{un{5IdLkh4o@Fd7&3bcKeHxNv%}5_+l$mA{G% zj%s0)&^s;s9c7->cRFBIr%ALj18TmBSM$j_^)C9@VTE3IxG&@DN9@+ZqH0p&+`9|q zah_MBEcfhNG4ffZgy`K}`AM`C)%D5hiK8c*e#XlBSTW86ov{-_`OCK7ITBE|K_C+) zuz%6VkE=6ii)r6)nH#nzHIaGJ4EnBUW|=~s&67{^zLql!!_Ui4FQv`K2(dHIYoc3g z9|^oCajTCQL{buUWR? ztzsuc4T`_3$G&X$BB0@4Fk*c#d!%AzZD<;RrNM`1vdBK4T67G|ojqKZI z1LC>Wil@!ixqYL4SuHb^s%b~nxI-{uxTAI>ffVBlhY6yOq%s4fu6}`7DFZ%(X*Wip z+X+(ysEUlnHfO%Z*L-SQ?)+=XeGD%VW&Hle)Qi(Pi1`Ki9r7#_Cbu9Gul5i{ogwz~ zLs`TwxW5(hsAWq?+=)(6%5rDQH1jV@c9KkA*-j|Ol^J)5@0NPPzaRS0PdXl0U$xG4 zMV(Z?m#nLmO#h34t_G9VFZ8<<_AKEA0HLR?9Pv%Ns zKmpZDDirR&3+rToEWHkt8|D@0bk%`TE~%E~@5g^Y2=hyeAn`ndNPkEKFVi`Ev6f=k zo5t<`h4*P)^*{U~IXUS$aTY66Db+ADHKh&}_kIOK676ej%)zk$X0bzyWKH`oAeRw- z31&?dazrY9qt4#=9yl%b4zF3Ys>K?$n004r0hi~}*hI1O1qAni{DG&x5xKvNs<+Y= zFfdJ$Vd!DH!*^+X&9|3E!4eog|KYbvkQ7wO!uoBnf366T^W6+5;f z@HrH&hitQPBo{lAH_hd*+HSAZyGy7iqbd`!V^;d^pXy}``;}_>R$~g;Lq+Z`gg_&z zf9kGOxe@^qPW%q?`@&IlA%dncWIh?5gpee%Q&wun=y7*}vFD_=;$_uO@J2d1MklUa z4E#XxvK+9FFZApgaT323wNDgwQ<)t?A!6Jofz;i<81IKD9{G(5;cO+}-8kDb0aJ=N zRZe!{-~8ErD>jIv0GZ5-`Vx^T;x(l4rUx>f5`X2Qe3=*$aR{p+7Q}mfTzPn)52Ury z@?@Ywx9k&md^sj2V`h5(ZvX?6t_8CmaF!w)+n^Ha{<}UmZJO;HtU_&sv48g4=LnZr#B+JIt>Oya*Y#s|}Q4DF<3FaY)%49PFVWj!(@Mz)g_HXsATrp^7lym^;^fu&H0atAA3HUd=hRgQgY$<;{i+B z)yt%=eV}6B&oUxGtjbf%?4f?rgD#LRb=rSz+!X`mZVX`+RSw*Ir`DQj3Q7`4eqKG6 z{~=)Qrknp{;H@a%xA$cj>IL;gbR|4B+cu<+u=4HfnQ447+EMUa)r7dQ1R<;~t3U%f zb|p8(K%?vE>9gTWc-m{2kOik6l+6fu$?@f)LFI}fxa}X16ZWKO;!5?I1l-q!b=>$r zrF`-v-V`S8{tCJtGq703{#dn96K%Ynw4iuT!`Nn-4KE9}PfAM4;9Oo_Ibqmk$&z!N za=`9`uG@Mk>JUbQBjqW}vccdKr@kQ{zdypIDGT|xkWDc=b2{4S!VBg zcM}SMm--3{L9|3I!f}lM2nk|CPn%ia02*<%?IJI*8VSE~{os?lYqtQ08!*G-CS2>& zw)!ACjE2%(<(7DR^gN2;aSX4wX=3)@=)p};Q$j4Yg&2lQSUo|s8^QBw2a%kt_>bfo zkwom?j``lRSA(nyV_%=gg17j8kBFqE$vBa#FMJ@Wi!$R{gCf?AoLth_#zaGEAVk^v z;Q}&Qr-K0=mqO%3tKd1L4}_5v$<|UP*4BYjHdzA!){u-WeW~}|2w?i3ue2(gx2d$W zvwKF0S996!8|tN&ZRKZTVmeBmOHnAA46iwZppw5SHyX>Cv0}qlita9Zl69~2Nm6tt zAbdC_IC6l#73;m@PQA!9_}EPs?*n46@n{qFnh9xxa{Tr<2YS^<(g~~efdwZa^jUTk zy`}~D@^~oWstgmNLl9TP?7r-az=FX2jI-q)I20r6FK5l=x9kmz$1&@SaCSronN8R` zBxHW4ze*=(^34Nq8MBA-$2;l=?|8MEEzUI5cov^lP9oNKPELNB3%wEni%-b}Uf{!> z+h44kSXM{7tt2Jb0c?R3tvd#v7WC;;)~A0NS%ZAxK&5%bbafjobM5$5_<*G6lkg8< z3K5h%0`z!*ec(B~(Zh?*^L^D!=c0OREWj4RhD|wT`NN}RHdr;?yk+IV8L)mj zG*k*)zX60uT9x1_RTAcbR@!Ss?q;sI{w{I(yk~3FjV+rb4Im#BUfDOp;ll5%)HyXq z1Pm$TeGtVXhDEff+!?CK#coYON6&?X4IM{BBe1Iqzw7*rJaBX6yPGuPnEYNNEc<@x z_wNA7NPJKAS)Sg9wUJx`H+Qg@l7XmcgJqMepX&d*x>!&Zt;NZ>uZj1#w4)Sa1gh{h z!;(H|c{kUp-am>DQ*|=%BPWx6G`oM=6k_lq` zh~g8J3FGRP3o9(^69QjlSU7l;`oVYqnckWb;4mXxo#`)I$qs_;&O1J?AYkNcwm*yK zUTD%i;lnH5ZtY`XzL5KEi#i5Gb&PN z6XX?EX1R6RnLopKhQCByuz2FjS3TRFnU{${4qLN$Ln2ltgNujFuVZ`roaw(ds+lEj z?VOJp9>y;Ap?eNw?yrWmGCNhw?)ya2%$YZ_fv(Qr8*I$?5(M9g`B>2*!VU?5de z)dGPAItb5VbAJN!2VdXMKEj8HhPFU}BmzKn8mv4Ltg0VW5V&i;7$ zsA!le49GJf5mpVS8mBCZpkfmkVO?5ZOrkpGc=_^RyR*;7x0%5x{Jv@qBaSr2Cg{PN z2_6&SS_)7v9O>gU#xjWFsj0qy@u43`AvF>Nx3EI?b+?#u-8gLe1E*nQ6dqg1EO|bJ z80Dpt4%|gi0#|pyVt2b9M%K<)W&zYp9afJ5h6VFaI21F(IVhmAIj*-Db{e121SzA??UlLk<~{C+1c4ql{y2&n1iQL`Z+8g|fZoHVXE z+{|FpWY_RM7lMx0Nl#~cWul(8PCn93JLdPFGr=#7LNY(aD;%|;qrh4mNw%t*)w#5a za^>B0YjK$z+ONmOeE?W50>ooB1c^!}iAzM~8OtQmE!nySB;o*Q_yD{2JvqeuTE%Zt zjIt=C+X6IUAbHF_Fw+n@`*ERt*q{RWkz389Yf(|6SDwek**$hQ6NMM???ooRY`G7H za954E2xi`TOk^}-M6KV3m%d9Z)uEthGkgfUWvf~Z*3Zmw5b9u6Npq%)PU)(1o*~U2>KqOBjMqiK21vgOzVH}lkd4bPgEq01xh9+yHKCyc zz!^#io-N};DfXguZS*?al%?L?A)qlzJvn-VH=1Li<3WpJdQV9(n%i2Dr{ za)m6*35*7Y5h|4YWXoV|1qLf|i7LpgSgsoS2w!T}@-NbK;a|z- zL2EKYy&fJ-VVa?zdqSpeNYv2Rk3Q*qnmsr-Xs3T-mhS>rnl1(= ziC@L2?j6E!OVNvR9)N_=XKk%U$@TkJWrzG|rst@63Id-q^AG>`O3oihJ3a0kneR_T zC6f$aPQyyV!y9_WFDyX+bo9K<3Pe_?v^{__7Qzz{2{IVi2;Q1< zRLwkBGA&*ScLVLttV7<+p9qgo*b z$_!vnzk*>_^Pj1BzUt7r3odpq>B47^9nFN;g5z7A@wcl84ZXCP_rf>RN>hI}^tF_t zKlMD!nP#jPd4BLRw8YI~Bjw*ox#=gcEwiOvzdbE|mu#lEF#nKuuR(XL-455Dp!zAB z?Rj;i$E3v0(M3l;iJ&!EqGn)wHLPx6yW4*pVe;0_i|;#33T`eSE3F(){!>wA3L4Fv zC*VffGL&(!48uDSb|Jn_>n=A`f0)7sv4+VIlXkhdd24qS%0P;{v-c5>2$V#X# z))fd5k^Sz!gT>U(owLf8Bw+gedMFPhCE>*uP(sK9;+QSW&CO%%tL$FGa)DNrLh6!< zvQ;TNRiKIqCj${WIY1L){-uG7dg3GaDt2Ul?ZIW2nT2T-t@hgkMFEA}5^`A`^QGE; zs|KBZd^K{1;PH+`f2dt~-~+6(bi{hohw6jBC$f1ue`0Q|)76Zgtj^`bs1v`qiW6KlPz;yx7Q+@7GWGlDA{Fv8+-osxqsI> zOa7DVPFfU|jEE^q;aATOgW69tY~{J* z@BEt3DM@j0Q^&>w{|6VO6HQ5FD&EXhy0x*=(>tUn&<3Yja>0;5$f0^0bdG+2D08>X zsQ+}CMZvJd&pw-=S9vu*KkwVWiN@;aN>pF8rW-i3Wt@%{LZ@qL9wY$Fl zcx$RiMb5)mM=&crH6n!_4fbgKOe|lO{v{~&bxrN+S_-{F5ruJNh5a*JMHpW&0|qM= zaa;=%ASwsE$d;V;L@;&c)wzj8e0)uWV5Z$}%g9iZmXn#b9~CF3lLF{P{8_eGKWxR4 zr1mpAMO?)(56u_Y+buFidnR8EN8{V_h__jKmF=>>&w_T|l7L2f{CrbSA{VNyc><(vod>Sm= z!#veCP)xryY^XbMvxqxl`ihTB5?Y7CJIb`=0CU}haw@xRBM|jxx}4MbPwRJL2arkN z!)uMqN)|^O#FQC~Pw<-@9E>QAG105Ofj4hEaJIrQ5CdJXP=8~dwkVeQ!corZwte8S zKbWNgq(EtqGZ*z9swM=Qkbe@mKX)#v>%p>`Fl>s1JNf<<3V0KJfSZvX!I6s9KU(Ya z1rW780<7!cZ6)J$fZ^bkE}mYd{Td3{HP;3OC`?o)(8K4RiT-i}az}0tpMqyvodIxI zyK5epW3n`qqNkhRn#jS7{|8(xwCm_Tit&^^r^n;T8)p$8a<^A#O6_Ik`p@EQk&Zeu z_%MLT+F*_jXmSjrPbVV(E4U&f{4oAI!Y^*Fe)@+@1|r5|{Z)AgPNwXJ05ha(LHiHQ zmG*HGbhYya_JTgi%fzDp{>3F!;E6afU>Bx_sZJMW01?H64XAB%Ft1zvl#4w3h2!7z zt>q>3&;@)*B2iG^54gB?0MI0X=va?J)M&%-ny=Iom>FBZMF5}iUHCKUob8X2uPnOA z0y|8&Q8Pgo4%$gjdmppz51sDJ^+BEs1yIHotvbr zVk4X`!0-*!_?Gy~OqjPZ#M7z0Qw14tMy0?@t6ATSctt66D$rs)hUw$_P)JC0oEc&j zrYPZj@)=|sdF&iD33WWbmb4u&w?RRoeWh<{uE`(@q937dzpg`g3CK?V_{k%SE^p zS8CE&LLswnY%sx~k@W|td~{7$Py`Fc^QSt`LWz2e3?_=v1Yq46a(-F?DS#eI)oUuDouMCwqSPSViJIS#N*I+*}cGHZPcJ6)Np*!^?0r( z@`&|-0l6RiY9i)C4KebTBsEyB?;3q5?;s4&H+w>;*Osgl^f+wq+G9o$gTzLhR(*U> z^hNKp>5j9a$;!-%@i)v2Ehv%Uz{(7RFP6E*D*vmVw7EHtVZ0}8-?LYZ66m3X7GNuJ zi38IKWpY7uiUu{Rts4{;jC!imjY`PIl!y<;3H|W($o}ChQqJ&V;L7Wy zk(8Wn6Wj}<6UI38DRYgKOPWido|gsM2bb2v3kFWW#tp)@LncIr0hUB7C1Lr8G=_6t zqJno?lDj!`BmtJ>kA&nA5J}kLEi+)cps!*wY%u+YA?6bH`aoIYie-FzPdJ}p!8L2# zyV|7>Ws-oMg(x?p7x~-|rk?Lr}Rv*`I3hXqtWb^TRYS|1hyxfDg zsVX`63%9!&81*^Oa z-(-I9C2)C`5_W@>GvEs16jAaUEtiXZ7AFb~Di%EtD~dx&Nm>u`Y|Nd$ioMar{wf2d zEp`r4!1dqgP&=UvWMhQ!yC{K$pU-vV%6fi?f+R?z7+Qa1mQg9fxLrw5^b@QM^8I%JO0v zY45S+x-V*TjYJLf>+sO5ZZ_j2NrXb)c^a;t4L*N6@PD%Inmb#q>iiapf)9bn(Je;- z@%ocf_V>#1?kcIveyY)?%{p6Y=sbVs7k3W$z9$==>*;6Y$7dnc5U%3%~1WEi4L9S%yEuLNx#w+NB1=QiG)g?ht5KPEILF3 z>x&*}BDQ^h6t3%Clx@v;4cYfK0Gb=q2@gMO=(=n$! z#J`HUL78i7Yv0%X@V9k*<^}I*gZLdCrpr3a2%-Lx?j54ZZSm`q$Iq{PK<-e3D!`^X z9k%iCV7@*AiynHIbzm+(8Cdex8R?zJi!V3#Eg!zVi3wE_tM;GSVh#}?*X$OJBphI; zX+TIKEkPvQB7jxDDs3GJ2~`p;r9X+{N$aI7Wz0`CQcQ<{JO8(&dpw zT+B;ck_S|Yx%5RJ*fsF**#oKmlcc+z$L@}H1p*aN-Z6!{#Zi~`_V!x5HiaMAp@Gl^ z@6=B%$jG6iTTbJz_oluN0LL!)PZLjnl)`Bdt#k>%d2t=R21Q;H-pi{=SfIs3C>ovqx17UM}aes7< zW_etL>-po^G3`6fz8W4}PyZ>bo?mEJ+J`?_XX`pv5ix;azSZs&F}jz`|AASk0gxCI zXQQiMBLO9g-+D}TJhu7MTIsFNSNRU1ApaG(ekAEG{UUODG_LB{W(P|xOF^52(v4K& zh)J>$O{S(sA`pI)eT$Ricl*`h?r3)*W!G8pzwcVOgAu#Z&{?M4AEIgmj1GNnhJQdJ zHOev>y|#K@g&3$HTXEtJ8zK#$rY)Zc9%wv(BqpS#wJOAK1+NLqi+&b|pD+=ys?oXz zGqC@#;?B`L4lOD^6$R{eiCB38NC|`9`2rD(8jh;*tz|xW1Rb@=nH4EEhWOri=aAqjWkhru+=s%CyeS0H7 zL>3D*crj*<(wZ9ybPh*sSbNk1KS?uo^ktIs<_;`4we2RFw0I-2Fxz1l8uwARvSD!- zkFHz1RRK9LaaNxEz-x|ApD|O8e7`5K7UjdV|am(?-3k#>8C#3B&%^Z z7)uL9yc6Vtaqxc4NXz#gpyXJ3K_aA@krNjDLfF~<<&D%8-iWBXz?W2(AO633b4fMY zpVm%l{MIeR?{>bKKUS2Nu_N6WZV#+hW2tgpN}-qF!X&o3_+y{Z9i8u2n@DvWKU?&0oN!5Z^lyjl_XWb(YJ zBk?@mlQTeQz}1SBQT#qWru67~3*c4TuPUoZJ8Z;Ogyd@#+gQSE=-*i26jz#UqWclR zVr;*GFIR|-vEl;L+4;n2s=`C9`Jinfo7+>FOR!nv-@bhz=<~AdPUj|$3S3N&3oFho z^=>r9Of|@i#MPTxzG5@>oTCH6qs-F@b-7uoXdzMpSY7Va>9gcQ? zRXf)-IVNjphnOMK+KfkQU9S3XB+O&v6+M<;ZCOu zzF93R(orqD{wo(k%fMJsF*A#vJuPxe4 z9%XiG8yb@4jUL2BInm`!hTM)7hkI8C?cfK|e#tbw3;b5sS@y-z#k0d;rJ5k&F$eO4 zu(zT2!D#PDp!uT^G<9j8@J0l0FWg@NnIUGGe+`~2EA2Z?>{VK@lv8D$U#Ux{ZB3Dk zu-O5BYX)s3RpzSNIq%b;zzrYei@tV=llA!(>*D0H$L`pA2L}g-4$c^l4HI`^5lXwQcsW~Z$!)*zac;g_wgAF>K zynI)_Fjbx*o6{ox271+20_h4p=wDs)Wb;%$`_0+I=~4>pWvKwFl*RGKjJUH~c|Qg> zg16VhPd!wW4m+9d(2)Jq=5B-?RL$JSBa`&0|;spsGpe0n_C(LdXC@G zlX(~C=Zu*dY;>~w0xD%Q8_=!fb@M($doCm3<&wJeJN>=&#s-W&gjiTA+3D#_B<-+4 z=@W}q1PDPNhK{#?+M1*8utzMC!__16kjo8GP8@Tajw z^{BKysBR=lBPD4TW~HSl#D<@}H~&;;0wTA|17W_!CY-!l5Kab))~-*+K^>1U`2oL) z5oc#7$HCtn0_U5x@kbaa6}|@Xpbyt}-yMN|#Dm|P{+_0~2=Wk<>8FZ%d`LhX0hJBq*#g#Z^ZXl>;fDKaoY$khvS)}ZPAO=f@e+wG{Rl^7$ zEk4v!rvI%sMp7Q%Wc|9gWLE%}nt>yr!RR8NPka+*e7`79PEXXI%M@RS^l}T$YxlmN z>gg!4dNi<9a*Ce%(0`XxqI9u+=B+^^X}0}IR!cfzKx(k}M%>}&U1G9eo!r}tHwgBs z&yEz`cH=d9!CH^&2lNXK>WKL$11P(PJVCMt@YtUEXl<9Sa~1;fC#`lpDq@#b)ES zkJD!@*RMl1{Lfa}@sBcU1Y}m^(b^K1R1>;#$sM=J#{wRMf@W<*x7bMDlQc`Kfh*@g z;6_si__g>TYmMONjGZ(4ZAY9&wi)a=OZlWxen=c8H7Nat(w2^(X_TV(Si#yEXLjrb zaUi=BE;$sv809V`TqCn#PlGdTp3*%>gSQA7~q`<<=z_&wVl@T!<}wKf%d=$q~_=bsn8H^eeW* z=*xKbYdW~Mq`uhP1&Fadn@F^c(5bw0;(CP*@Xw>wKF%XVx|Hs2kX(=k=`Ilk=|;Lklm(=vL8K)8(*1n=J^u?|dm$GLJMW2e&dj{5NdkB` ztxEkXJrD=#aHr2Z)H$JLS1N2Qto-HsU;uqn<4_F;o$(_L$GHQSp=IJHak&3o7G_bU z>=)w&cG)sII;JvyQA3U>Q8zX@DI5i#gDyw_ln5CO<<}g}3j(r+AyFDUW|;F2g(1j? zqZ?hTx(}d~ItfsEWS@`WsyJFs*cZOd!#n`kKEdFLj^xuQZpH%+`=E68K3Wd~t80q~EdUiPN9$FEvzkilGMU#yzAm)hIOH9XL$s<&e&Ezs`%n)=}%J_nC zoUJK$Q!1D*S6yMgtk=w7b;=j~s9Ly_#HtCXg8%xvn!aVuW_K}37h+EXap?Slh`{fQ zH|R=i)qa6g;izNhZ)l~NBP7;?*F#SzYDhhHKAGV>+%C-Dbnf}8WnMSRbP-L?Ag?~P zvUyoqGINaYzu@e*n6DWbU#Npwkhe3sS9w{@CE4GRusA!<_@lh!9oI7@C_`{0AHkmwk_6NfZzMOFtw(mfa5b8YzbZ ziW7aBuuxDOeQlNrhfJhA5Za3}BdeNAA|CACvj6;j zDJ}9|WWCWw`HL8TlR4F>xTDmTtqY_`eH?M2;jq?LTfXb3aiIzgUp_v6RI2iG0tVT= zLBHGAPW;nGf5}}bMba~)Y5T=|k*j_W>Q?&1FpN^2(1}XLKRZJ579K)nWs;wt)pZ&0 z{_m#E#Bri7rix1K*isv@{q{lIM)ZHY(&@xIUfHCZ+9F1qer?(G$S65VLh`?gDMbX8 zn;#N1^za1;2O>74B0oRvA>mwJ#>u9?2g9!m5a#cv+UO5*0inw*z)>xj#R)s^wiMu5 z_R5ovuOF;R0yN2yYd)RLE}>mxNJC|%lW|qR!AlDZ4^-_UQ@&FDoME(Ak}d{vkf=|V z7Gw_iu@ldr8(u|9dC8ZkzfCZO6>QVMO?zH|%{bOOJCL)+y?;>6?BnqMZBmQq%U`PbV#Q9!WuJH>K!-tUU^sY?7=E%Hy}-S z!>?`HgF z%N{L8@Q1+nU)lK}@_tX${hXii9A9s@WB=Y<_rAcC9&K~5*X*y)?ZAX_#yJw@21K%R zVqyp5nf6VFZuWm*=15xxI9D`f#`O>{VAVQntI~> z3(nb~#Dq+CB)x*DlW70rWjhL5@*5X)cALPP72J$Zv}CAq?P*`T+Kr=`Hdr>L(_$d_ zTra9VrE?9$kGMnrn+3n&Ge$d+xno>!S$Vg0K~1oVoMt5iB^D+Mqk9VusCzLC*ta!L zm0%^rWqq7useyhsyd$!+k5%|}R_2K`Sx87FGNnMIu1n~=zW!e0&{sGqhv zxEX3*e@*p`^{WI;azd(=E3F5CN{p8nVf@kg{iDKc(JA2jxwm+;g7~+OPts0#>R3CT zqBvd{S_fW&e?Wo3aCQchJxiO1(EZ}dX!28a1Ekw$+4 z-Gs(EdhF}E*U3gTX2}wqSXKsDC}|Zl+BH9RQ&KT6qDRWa4aol-4*2vv5J;vxQf~dI z0NaY^PVt@L^U=Dz)%9TAyl&`UfWt1<>{IsEzVVpH4q363n&E63qSx4x7x{PfRDSqi z#hiT}aP%{c1}mDs$K2zyprM?KIjOBIAm#92+V!wRtma1S#oZR53=Q?AM=?awYmA&O zx9eAriO2=MQ&EV+@PC_XgQil0v{zK^MbTUvrSTSoEZujC3jB)#v*q0<&BL2pKmqkU zgQ15;mP1Xhxi~nmQmno(ox2H697`yUYH2zn4*r$Rvx1D`?2ydY{j*ds^Ob)XCW0<^ zS}Dg-0DcV`Gwb(`C9ePgKetNyz3<<@x7dk^QGJ*TGIX93oUjj1;u!$O+Wcn)t2Jgr zbgWUZFh+S;R;$L$BcG$GF!a1CAkXZKUL z5TjZbeBkK6^aQnmGM@Bc%NZ-%JVvwFzM3JTq?Mob2(US=506Y62qPFn#KUi2dHsI!CRO1 zR(K~2A?b-|tpR|PN!g#7`bj+Z`6#Ad5ZLo`F2cD`b6Y!11VOc8EdI0miZ=cM8|f5- zKDltP@LL0F>sazt8E>j0Vn>{?yN4R63WHexokWaOCC`nvEd#Fdk? z|0==mXA2`4I)NaHg1!D0o7(5-XbSQqS>k#QS;s(d@mIRRb&F?w^m}Hb!PRA{aOD0x z3Xi~e9Q!aPf*yN@t3I5hbloiXHi;eUS91#s*;KMZND2+clm+?u*7@)N;JhDUqwxs^ zZftlkuu0zBsV6d(nt6ou>w&Kp$znBbe;bqOvut_{@JI->(d5l!FStRijJQ^H?8x3V zQNB7HYz9TPy0rd6SxY{=11Om67LP@q^H!5 z+_Fz5k47v~YqIi@BgHUU@8obr33{>&LsB+=+Xpt#P4qpY<^AKI@Ol?|YCr-%=`m>7(USB{$(xPg_};dX+XLs#_>30IK zw@FkfuOK`E@eCC&a0uBjEx6(GjNe4Dc`!>Ka0fV zpu#*a(G0uUOWjKN#5gY&cf6@&6xq_e^^A%z+CsOKf}FgXis20f5EPzb!;^QF!y=|X z$PuVWehu5*ro71!d;PW|6Q07~5F$N3*V5P_g@- zovETUi~9gjyb${-Ofj$A2ez#vZH;+)fGzZE%gOrG%am44n!CaY%;QS z1Sk)+tJvQhoSqE!#)e^?67|VB!Ul<%k?j{O!(`%iZFN;)OT!OVtet+vBI`1aS3w(T ze=ZwB@2_^so$6Zgaj4@BJ{TDpIXKLQ@D`MRJ-$B8SOCma&|Maqb6Yd5%nAa*coVnI ztRPbI2pP>OM2O&m{!S!4mwPcdhB{%t+Hx*lFsj2^Psc?zkr`8!0|$8qCvq*$K3Gct z?yE?Tj4n^mVnLyFpJjzwkHhp+Ch`lX0{9Ald$6CK2S56U6m{h8?;q{#jKn|PjI*tV z{zC|6+OFti+bFAQiK{govrm1KRrMG5o~;!rqMGiHvro#`$yv7|0_Rz;CB^Vj3-eL))sO41%1}m# z#s5l$A7=WBIT`+}Mdzf9<`u>+vKDzhc){L_6K@(PLgu$2ZCWL}{*tW+7+V;3zXmQ$v}!DIf8T*%H-|Ws6{gw}2X;}(-eE4k{CWQK z`G>aGY8QSDnHx(qMIbst8?@gYW@$~u(TWEv80aREy_L*)XO&N9RmMYZct`|n16mWI zZE%_`f50iI&5t)6ZDFwY?}x7A?_RulgIspv)wMRoQ;x3p`gM^*+lsQy1R)NnE2YRrNjokYBQB~DWnT+CW%)A0DLJ77qs* z1+*&b>qU}jB(UIJS^fVNIPo z*8Knm&|6RpQW9t%1Hw=ei$v5Dd99*Oyel5R%pJvr&1}{{4{na6Am;~8u3T%1%MU4>$vm0~N zJw^QLrUunBmKq_Csd!SvLhu&5@M%MQJ5gxxV&|P(-RINBmJjY3_tJF3)%!c&q@H)J znx9^i9sXC*+iQNj-XpQFU-rAIHWUNY-m{LupmZFW`lDw&@|UmpBGiYQj|z)C3B(o? z8#|?Sx!b?xNoU#NCuj5%CIa znP)w@Q<2UC`2N%6=lxUaq|n$(f!DmG8Xzf9kgP8f0trL|LI#P)cNFDH%ql4@+e|Q$ zY8T^0SxiPvAy=d=a3P|rmm-e|>0UyjD4^C#+Yj;67T(hvYtf^z0Ylfr9#F}jeRH(W zGP$??M|%q2XAt#lrAD9z)~ZyE3zn3x=V`_%^#Hme0ymwI+)4CW5Z9y_#X!<$GoCDZ z`6t^p)I{0{P@uiDgyuGeKR?M0J02Lf`MHz{Ck2@?|5wz*&Hb965L5m^H6c_Dh^oaC z_A5xWKBMTqv(O|T(?dPll)e9mQQp)dKjHRs67sX7a3ezCa8k{=Uzb5sDj6o-`UVC| zGqbZfOK^F&M&k5^@w|@I2>CMot=vctv0Q99{kV!mxF$1M{d639|EiWOJYrVEH@`)? z7A~+1)nfCGxbUTa@HV_*BrhmqjOws7n7)AvdtsBJT44=}SKVN%7WREAJzsS?dcM^5 zbUdAfGN}tQFDZ-N4f<=bmol;jQRULavoj04zrG1^fue{kcXo<*IpzEP2pXAF*X1X_ zNrX9oS$C&I?nCIz5buDXqIoM^pn3oJ)N7nv4?M5QB8h9mw;iqD4nwt+Smff)+cUV( zyhN!DIn(q|NEUHVU{Z#)7A&-f zB9XWFgzaBwgfx6?W+PBOllo+Mi&9glRoYb>)#Hu+A$_H%*E|jSf`fzqtZ}s*36V*6 zclq~1Kot={nT$m#95;y@J;X~Z8Q2g|>T~GqVo??mKJw6cu1GGbJT2S7hE&MB`6&Tr z?&|6!h97SQ5&7qKvh)oaE{AIR0{iStxDRco<7sY?EQ`X5pWO*{{3fF@xrV2@>bMB_-$qoQ?3QU)FT^`a(>VH zMDvoz@z2JO%>3e}UuOf)A?H9By+YLLVZUjWvUuNF_M^qw#brFs(eVX--`{_(-E?%d z$We>W1?K*5^g!VES;!)ow=gY=%2MP-%6za(L-U z+nX7?XbLFKCOLcv5u$=b+hmF_C?tgY)w_uuIXY0QC7m4x$!5)uuu?rs(=U|qLOoFq zi)VT_HHedFHI4cpAkdeadYf87*lz)fEBK5?)H48pXo3Yk)m2qZfF)g{)&bfE?qu+5 z201?5z5j43m0rJoEyqmw?L4TbHf}n!tGwL)V;1}8Z-;J~Cmdj(jT~mTrB^(QZl5k$ zCu&$ZB!{3pP@ih2{Nu2`=T1u+KOrG4z+&_#2t4i6)DlJs&GaxhhkPH&r%ku2ATJm) zQL|jzdZH>Y;r<;#z24?wK}X8&_E7h_YlbN;?lnH%E;P*m$UbqfBl_=1sVJq?jNg^# zfIBfCy=Z(!d+F-un$@p~Na0Yp0w_Ok>|-kc)%0fM{+ej~-`}I1!iF@W6igg7GBJR6l2xg1K0k* zuhnIdXMA&W)8Cq>$Q#e0mqslVHU69Ds(s^96)?fy$IL$|N{v@-MegD{H1zr&kM1Pi z4_{nR&^OU3pLdk$B88r7=TSv>+gCg%ZK3X99U)s8H$V*^@vf_^tVG!+w$=LVgUc5V zhMgk&EHGB}(ZZT*%#k3-P;jXgG)2QZgzjAiHVW4Ej#?((?%#wCY7jcg60uOqR2B~t zmwYG189Sp$i@({-Y{B*MO2Dl3?>*(EYzxa?3IrXBXIGqN?)Wfgf~=fzAr!s^hSDeW z{m$K#Kt7ZF>vHq{jyndUE>70kX$XOja^p1U@oA3K`iXOx0bK;C-sLKYu6Uf~ix{<; z;54@$nTY}PtTAd>0JZn%DoCa~b2)!}=LL#oK-b!5$VcSsNr_X$>A?X-W;;9*dLOFt zqT~;C@(aU1SKn%oJOV3Hk(U`^dp+i-ib)(jRn^t!Gj3mGY<6WZJY)6y=)puKNGoKuVgZD{YLwVg{9#z+A057nMxs z#uwI`HXyZMYrd*34$T|kmK;JNu|sKAOr$!Vg(QWjsmsE?^EDDxO;m32)(S`CK^ap zK%zt4*G}!WLXTqJt781&9;aAB|A|tPblU?um^$2xt!{KLc2oR-Yf4$%|I#7_Y`f5< z@W-llL*{Q(Stkc!2Ws`Dv!tB_V;VJtdwXg0s|_pps7u`KJ0BBug0o5c`s;xX>E7`j zQ>58$pQ_*VuuR;|ZP3E7$z5;4e0AG|L0VqIKO~Y*K!`@vlQJEq2O$(0{%oD+;_NI; z-d~=lC5Q^;od4L0XI#5r2`n18{%IB~Q(j@PIZ2!qPY*u-N)Hc3JV=Z@blLYkZ*IFW zxnx;A;39=%(!Yi=BmCvFYX%}yDfhh+WEyYGMCF-=0EUO+rZ|Xv&jKJw;)!Z5HzF&x z|2!vDBywe={43itG_54sTsf$PllaT#oE@dMTT_vRq0_O@`%mF+->o}X7HmEA8n?5} znua<$0vIUALz=`^wEC6v_7~dg(Bhx>}6m)prRjec^(}(m&dy0neDI6knsxPHSUQP8^gN#4L1LzEH4mdD7;`5>T zGh4Ay-*&x)b`(A|mA90@UV`s4m5ySGV!R|OLmm{sjzB{M-~M!^qH4dF9Txzu#10V= zuJ9s3&uE}B0>W&UYkTKXVc|!ZU;{Tdw@T~YtcXF1b@ABDJ5a6ziA!Fb^g8e>^U(5F z03jC8tRGv>z7ne}Y|&yGx}is!_aYAOYch5Rz7Q;~dApaIniJ@WBmB#&tE(@1bRfU_ zrc|5?wf6WKy%EFQfip{3` zRez(Wb?Bg2pnzDtgL~EmnmZ=1j#qMkO>&yU-67}&de{U#MG?UcUgPGhRTqJhIom7= zo%#8uM8+ZpZ;-m_V0EH~1H5bIr*ynK!hw505yj->R0j0mY%E)`^ut~u7F_uMD;L^P zOjox3b;^|DD#isTQjrT`zHBJhkeAGw%8)4X{VDDj(uVin7k|6=9$ve17@IIA7Y)`l zR5|W$`YQa=2M>l2&gh72=;+xtLUOktS~rwlXG>qb1~*Uz2uVL6@Iz}Z?`4MU)z>0 z21yog@a!lB%9POhiF3{lZWaHlfM>YTtWLP9LWhCG#0A(yWtrcmNz(53g`IH%@u*fQ zKhcc|$Cl-0-p!#0^QGn$Eb6sB+FhAB`+urmYDfFx+XYt4B^WgxO!ZYIX9Qy^2tcio z6Uu~>_d!!1ONeP3St@dFb7mF3@yWiJy} zZ~=CtRzz6=1Lgvl)r2v+^X_f;#*#Za02ov`v_)C$`i#(c`DcoqQt1WWOsUTh6O#cU zHa;*N5FKM>8!TCDs1ga0%U$`=q=JE2joWTOw8np!q~B=6GGvT9+f4#%;>R%e2(e1)#05;UT zTk)57$M=?nmH0O^c{y=U;p`ifH z9XBOJeUz?8%FvUx!eQ51*t4`#nhrocTA7;WJG;7ev(G@H@|u~yU|ky+miz7O)EcSP zzSlIp6HP@a1B|%DG9H1I#f=hV*5=cLMeq@7A6ou1tl9WAc*o6?=~fZRKa5b%4U=z6 z;XtlUmokQw!)hK$leY%Kx;NJoneN{YRFaCca3GOp_?ufjkyDL|thoos>vBr~b;>)( zaeYv% z(yIPdy52rM=dQwL0fnoxXjZDnWZ3|de5rJQFzcd`xI^k#)Las|C|(q&mRCV7%K`U> zWFfJq+H-&h1CR2SjDwYN7d*T^cqbCQQRGx)AIvm15DfP0G)BW342*^w-*ZEl-lB+} z#J_knm)wu0$o^E!G2;-XsTSIQ?BB-rpLA<$>rTQFg*r}vbr=g#4>#!urNDE>*S&iU z#_va-qWq(o@Qw_ck8p9;AUb$Q&j`z1@ZHeraly^FZzOu2s;%Ey zo#+ENL+d{y8?)x9^5do92Sj{$=~k|;?(XiIusopnc3q;N2+%aJw+4WF zeB=himLY3q=j^e+hA} zlxV(+idNM6hfBNiYXEE()yE+P(UY~|BHz$(=WN1xXD*BN0?g zZy!as)>1xdPHlbai)Y$OJ>n%@wEz1TsFZqdyZ0u!%@X8e8bhndgF&*U^>C{Y>iz_$ z;w{Yy^V2CJ4%W}J9k{0T#l9agT~Vhu_@QKN@18w_f&JKqfE!bf`fe%xt1!}3Y-ZY#iz6rWO_5@favyEj zg{KTDm1Ifa}Aovdkf^g^Z(Oo))d&pT6FHLpd~}8Acz!wM#Tjihv{7|w+(+AdFg**W@mK#^Y(f=Vt_& zF#aU1lhuN$Qi>v$06yi=5Q(O8ivQGJXu$Dumb9p7yUntvCZfZZJRE;lNRJvgr*BvMwVyhwV4a|t1Vh9&?-XRo>QwnDPL z5@yIlb|Y{`3I}9qsg#6UCed}iAbSg?e#~t)sdR;sfyQ#ecwSMReQm<7@A^o3cv3M^ zWhOG%undC^3US$FM>wUc8mm~>nclri;&>b$3J3!iO9Ms(dv(7_IdPO?VH%$K_2^!I z-A8790F)de3WqD7CF9tOB2pj?UQIUSH!)uiLoG_NkGwnF9cH_2noda?B0koQQ#5yv zc-b|&x;Z;jX8(&^Nf-tMaz{Yyv)lRfImfBg8y!oLQ69_JGm-VhOIZd1GQyX`uxR4P za4`as@1hO9jpb!Fm&XB?$GepJ=I`h#Y_-{7d|(v;uoG6gEX??29qWJOLJ>L{1O)Fk zx-FF$MT38X%+HQS8q4<>AJ=lY7_-WjF_(_7ZohEP9RvHhzbp%Cvhh*r)xNcnedcb`~m%jXX*H%cgMnzAWg73KhV>ME! z;7e&mi+mI#D9)ZhO8_iPqRGgZh-o!Bi&vT#tRU{Z!&cXG&KyuXef5j4d*5Tw>w0~9 z(&w`qg6p+uuBFqM#eX*;bS7i``i6$(Bs+T>PFh4{sc@0E`+h`W$`^kDrGd}6&N}ed zuiaI#U9nY@MavYJma*>C7$x$NH{miHCNT!`H$71Z@;~WO$16qCLwU{yE*xS)pL`W1 zl`UL(N)A7&xD(L1A!B-Au6pWGBxB853yF(}+erR9vzISlvC0$3#*JbVN0&hL>PG*= zc2ZJNsCn3pk63r_{`qSZ^R)Nu8;Uy~F1{+%n1PgMNkAZxijtBM`xV}fcSoUXzDs#w zK`Bzf7P_tn7$*;HAmJ!fy#pJ0}`r){;?MldHvdEj{|0b>L&v(HZs2#J= z;gTmPVXuO;R948pOpgZ}@qJwl8nQ7AD`$V$by!}YbYn~2Z1)JiF`g~nQjhRiYQlin z&u7Q2Z~hIWRf}nw0~CDv{pB1YooYSQh|})I9)Yv=uZKMzz2E@AMbZyPYCK^NKf;54Zm~D;?K#osaA&^)br7n*RH@CPUn!X@ zLhYu6JBDYA0$HKaewMUm?RS5EmGf`z999MxN#qy4=n?}i)SSr+c!j!0;1Sy=#VwKd zzOC+huc`Wn<8P{UM$bK|N6@vSiBuR^>~D zP-tB6TR+*TAQM<6tqzhH`>pKmPmMa)uv;*h#GpY+DPn4Eu;dDmadfcCC0YAEonbr0 zZ;5PNyzYe0PTgN%pY-%NBvUuM!^4a73oC?Sm!eyTDG3BjSa)+;T4EE3f%F+Jw>i#p z?~kT!IezfOe)N1<*;S6!27?p9!rvZd!O7gx#zz13Jm&@rK(>=Cn7C^Ch9hlXi^_Z# zy|P`IbMgW#&vod@fcCGUhkP|V3Pc``Z4vzx0es~|kfr{Y18xen~{$e#jEa3;SRf}Z5*Vvaq;=b5Gu{trTt zJLmxKOw_3JL3opXb==W zF=#|vHJ13U)mF5JBGk?nG{?p!$OgbRHkg(lDs+Fy>r~4nCOnbuVq_rcp%BpCPbYyB zn>-lE|C|op+DsMj23`deN$>Lo@mg?2*&b?c;>ADIT#^w5yy>Lrr?bo__gDQ6CCXwe z7(WQj%5qv8&t97-7jvSP`d@`;T}TK)Tmlb;?A&Alm}s(9R#HB-qPH2mP3d?^6zUzL z*3YS-4={+6Y8U`fGUN3U)^PEsyRMEWG%!$f!nw8Fj44f!wJ8#~-$2RQ^fQhte@O|J z4Yp)p#7=p{@8?$pt)Ts+Pq*K7SsF9us2zQj5pGm&(G6;Hf%?ESuX`5qao<8C%2<@i zHbp|~S`2Nxd;SLc`EAMRnDB~Ako}KjazLudl@fyx)rd@HbA!q zcdGb*$opz?I5NO(4FovrOfT|E_th!ycwYDCDPSmR-CbEYO9eq|``Wr*E*zhCWkxuK zMP+rSZr<0>j2oI2SYO_!02#jdn_aNMHZczgV1TkPU1y1U7UK*tMDhvsbU* zaahy}@>^RmR(~;di+h1z3#vi5qFEZ4esoF}q0sw(AuRK@xN;Qt6!u79|Yo`0za4H&6KL5}IQ2-#d;Ago{kbarpE(w=WG} z=knAR;=H$adp)$rjV;A41rL`v&XzRjutTA+KAFNO zJ3Xz>vuL$O+6VtUbI@JHnbrG-Diufpl4lM z{h#MFT{h8!6TyJ&PE%W3yDd5C6RsK8L7ED)CP$VQ%pcn)hT=7_cT?P*8Oa|Ksys@; zPW?BDPq9NxnV{saCyOlI62hl9OAcv0~_BGX!Vwz zl3(0h{M;}KFF-63hgpy0gun(F0D&=7yERZ;>xN31w@{)-k`T`>(Kg*ZLlA4zL-4#` za{Ws}+&Uip!Nf-1zBV9~)lP2>0h+}5;gFS2m+Jb+o3rG4$UIS;P!OPMTGE{D#xT_` zgn8_$^*5mV2LfV-3u{0_m%9S+`Uy$eTtf&7;&|KUk@RuIo4*MpsxS)_dP5v}`Np!9 zM(>HB%(^Z#maMi?+qGI_^Il5krxJN;QdL1M2GOVI{cAzt1%lt@-)xjN%CK21zr@?>f_C5YofW*H35df% zKVjX}1f`qs9R&pU2z`4M>WBBWmUTM&TW=e=(%9S`MqKBXIe*PU)#&JG;`!@I!0*fO z@o`p%`fb03WT)xA2c0M6=5u?B+c;KqZtYJa9Sld(ueN_70kptx82BX%UGnXVNyg@5 z(v(g2rsjHb;{e8%GOYr-VY@?8;`?~azTqkHU3fKl=CW;|m;+rH5T$1WphBTUP{sZ4 zh;sE><@cY*2hKSsRE*KzXRdMR;I}-{qhJ$+vBv(Zo=$AnnmH_$GZt5J5eCL)W?hDS z5^s~Q|g@Nq5)1@FWjM(Fr2x z!Hxd}=Dx+^1`sWEP8f63AhW3!Vs!4%{tg*F!$MySf%8D3E4;{74l9ywe052w46ppj8X#XQB2^$8gNQ7z2~3k@z3~o;*-?>Mg?D#B+lV}Y@!8B5*eE+h z9k@i(BG3*Ag@ojMfbwSpQ9JCOgLshI`L{}?@;ls^P$WRBJhiM#Rq3gHrR1E8uCjmg zFWJrY;9nCey1jj$sQ0G|k0vQQv7XgX&`8V?vK#Ki~pv-^$=D&Qhe&F z`mEpfb^A?I-IuFtjZFqL$@!zpUOa4Kf1o%su2oyB%}Zo;@&HU^h8uGsz4r?4sh1VZ ztf2?1he{I>fFmLQlrK%*0{rd*@qY#$rinw7peWKa%OCUb({(@oZD)5WvS+_UDrTf_ zO)VrR1@8Hp-lyxnd=-bru*1h>^~%+y$NTAKYqD7x@700mV`=T_Of*TFr(?8=a=ZHz zdNIcNUfolDT&G6K+ceq-0GbsCdO=ludxOQy^@{khl9EhjsVJmp3+(4rS}= z{iDnT+EZuyj<+8aV$t#2pNNBGUC>AsK5VlnQpN6F5qA5AcD(b4ZkS1coeav2gN(=$)~ zOdcX6<+o@Vxy$(Kpzn3S%NBJii|nPJfVUD-1-s%1x^EE%e>=g011s5 zvF2c@HPW=-5~4{^aoS7)2Zp8}?MeRz#HySIFPZ^~#*}mGy3gwy$}+~cZ{McS0D3n| zGq>Xc@<4dJGZ5~-MZehl2gU>c88FMNCab7F-2WNGRj{i|fVNt6glPsqc#@?cVpIkO zo5bobUvMkaK+HJ1kHZfVn!Z8iQ40E4n;Q&`A4yrhLxGMghJ?hWu{%U4+o)#Xza}Ae zE~1eVdS=1}%Fx$jF~|;DzJ)Sj?e&kp`I@VHgd8fK)x>fIVDf{jl;qDpj?a6figQ-~ z37bW!&DBkf_$>==qdtB~T{zW5&e{AD@!`YwkIE@Ax@F?Pjlw!B@@v_{3l{?oUtl4e zGzuILVXR;VdJGE&+&w;>$!Gb9v(P*BAjkYw1)xjB*3w;qDDn^H5T*?53W4W+2jcGWuih7p75Sj5?e zB^52m6dkvW^m1`&Kh8XV4|dFwA%9M}Zs`k@ob$S3zI=UzprE1UvfK4dOHHL$Dw+EO zV~z!g14B-qvM<%}oRd zAQ@g;F#t5OqoAbh#5S$ZwbeA!PzxoT?Z_c|96bIzr*1?VRhGyO0!S+fK$Q0$zR#^{ z+KmmOoiFcTGjDvf7rG&=ML(l(;zovOO6nS$CYLtNvvbwTTB(X*seRW0Ec-{Os2F?a zrh$Wa(eHT*C4w6tM?Oj&v!&$ah%YzA1l=bvrlo%RLrUs=@)SNnJM|df`2=*5)`ttN z1ViobaviMa9au`+Y^*G}iffE}?ylCl0tz@_Nh!uP|UvdN`v9xbN`Jl*8_H8#8Q3Y8O ze|fCRAJ^wOcC}DiefeZ(TWYC@`Cm2}Q@tbFzmM=+wcn(*u&`+?FK$xOOhCDP;p=C9 z;Ox@#bKd_;{rT$B*-pRMc-BeXQQC)rtQx;M6;&0X-2G!fL>7DQRsU})A~G^k(}oY( zX1R@{RD%2l1doQ&$?RVRwOMO;WvddaYgHuAB1Y2z8J@ z+eg4G(1SAl9g~xod}%{@8DPkF7p%e6(yTEPL*hpccKMSg#-B&&RC2ju_RLAWb%|*w z*b#|jE0b5Roh3vHh+n`#&8!+U7I2BSS7m+&;s^12No_e>@T1C3j&o;~f>vJW=eW$&lMz0~U)O zPb=!f=>R_R3Q}rTk!^R#RJaCN!&6(<%sms~gT z&4jZE2uh|C@ePgJ(r|E@w5%Uo-Cs{!yZr+t^O*>gx!ea3g`aV@nJnuHH%4%eeS#$= z^je(*0H_pq<@lOE)$1dtCqG{iVQMZQppvnCq5g7L2GHj8W#Vyz)iF%kF(V;l{_ab< z#7)NL!-FQz(Yd#npgtsM3+OEe(R2wW4o4Zf0W&{2t94oChgL&FMhStvKVB5eV9&^X zg+xzl3bY?zWl^X52BwY8{S$iMk!Ca#59enm*K~*2?r|$9xrbexKG=;RCY0^nY59Hr zvznTWkm97GFcGsh$z_g;iG|K>!~*39dHW2}N#E*xz>cobb5?QPG-nQP^nDlawCt(f z%Knd+_Kv=v;10PF`jG5>bmsuzQH@5o2LQU_q9Lms^n}k+xj^H5ik9XgJzf#f1qfV< z^=mhP`k}_P>yZ}Mb{!a*)0q25j7$}FB*wGqBfqdj;k@+HD5mL&#j$bm@$sp?aF#RX zKL?28!BwyMg)o}_d(zDX980>~@KM?8?zPM@=2G<-Cx3vu9L;$Mdu}0({WmC7dr4{E zi)MPtT#EnMv(*CRiHF#u5UtPb1n;^}NJ4jp6*4CgEHHg}Sv278*PB*^Y9jWKvx1U8 z9)DAl)ztS0DG9Qx8rPrAKB;L%iGsoCGyMD?`;I9ZQaEsDXO30I$A}Y$_y#{RkyB6( z@1H@GV$u2df6811(@V~U+XXsqejodh)qA#?W$x3^Cc*!6@gwjPZXcP3u2dWaGxFRR z6@l=c7PoK+{K~h{FJu87S(#J5S;^(xH}70bwMPo1`uwlaOWU*nHKSj*`45^@I5^;q zwK9(1#{@+FEEMA=Kp)UjB-Q1^9}Oj1a|Y3Ef6F+3gB}y(PYbA>nb%Aqq%uuQgs4`N zKm=U*aCx44n)D9wIqnuCfB!-fH`hv$I0@wTPri5pwEYW>X#eEE^mpt8k7if~ei(do zXlIf3(&(kVfv=4f-x_$Jo@=o#C!u|}WM`g6R^<4!B#>5|hiPUi&hXeepg3<$RGQ0 z+=O_aBTQ6XAW^^HMohiwUQgk*r5LqdaN8!ge&i?>fxeXSuxVdN2821X3H2U6xwqyZ z1t5-cfW#@r#a{bcf(&36IvOY9PZj`BqA*j9W`$W(uLz_lL>4gmqb=ZhD!JZ7)vs-G zs03jV44WfVkDj#3%t*-p+iMDD~^u zIHfb)^nO`1)Tl#bpqaywn;9gQyCKFKEL&>PE>(P|@A2~=8*pHFDD~+wQ8Vm$%a+?A zG_+@WdAT!@Rh3v_I;T1uyI*{3CUHLnVBPEO*omS>-ux3T2~WBm-p!;IM1;f(ByL}6wq^-|I1N>HI@HAN2Q<`r=1n>(M2w7IF#`) zd&=4xzOZaG))EvJHrKdd>Tq`H!`LkP#z*x7+_L_)zZsP7{P(NrXNs7=$~dbSUbuiQ z#`_aP>_m?Vtzp!%q6{KdtjGy+EqU=L^qa!Nski<}qJOS}L>N??hTYe@#RP!MCt5&n zdzqA$A_Vp4n!RZ#A$F&Q!sJY zR(5pUjsQzP25wU=Oq>qBB4u)W3YjCH*XQ2$Aq9diSl8?`#%wpC@V8hMusJ&o2&!Ce zxH!@SmB4-KSfC>eq$2~@16?&Qq;Ql?z8#DF?M-2BZf@FHFGVhyffL=~xg~$YiyYDH zpsqeE1kEBmLyk*E_O!Cj`4QOI_^IV0JaWKNyGrC?AvyU2fDYC}rekF3f27sy9PRJ_ z;7m+h$+W6LZ4Qm>otKjZ!CFD?NqlDFnZ?CaV05~S4_XjXwtpGf%FwmC@(m4qitUV) z6)9!#|FQJl@l^ls`-g*!gJaK-nZ1R~I!gA)&fXaj$zElLl5vdiK9cOL?3ERcy)z@n z%Ibvd@AG_qKY#Uyht7Gup5wZ&`?{`M>N+7!J|mU|4wpb$-`?Ey2tNM)4hz&_*i5{{ z0S(P-sLC0>(N9PCr6{t!r-OuhAfBu3Z{+8Q2pGOVAp+D^nR$G*!~adgO0B2D2dzUO z?);TDF?-biPDSirBrG8E%Re}S4QL*r|JOLDAGGBjK01L>DdyKH<>g0+I^gpmPrL6m zdQZx+);}$zxAT_n?znFgieLQS-)#Rbwc0Mok|O1vCO9bY#n9$qc5NxDJu*~c{3rqK zzfbY^OgmDL4vc9abgaDJ*p(utU{vhQhta_Cj6_5&Gx{3YflQ#fzhl92c{PzxC<)GZ zI$-z})fY#tsR1FKxvkZbVqnVar$9d@*&Y+?`I!2I$^0(8;Rk>%)P!({f;i9mx-ya8 z`*6(|#($akY@_Jb!!}vGTjZ=0=)}k9*urs|5!$_2$6`&26pwgVtK+?c{FJ5&yMs@Xl`?D!EKHs0P+`v`qVq8ECI1jnF zk1rAe`*bP?uk*@zpRBrQjS$UE(JOW&e--qn9d=c4a4_7+y~K*Dmghj@aO9i|J>x2UB2?d^98mFeA#C zo}r#urPIMU6uod`1w4*-$Wd+6nJ5W_w(Bp9txzQwGFeZKt*t_3U*HJT|z8Ds^`xF8sq? z46Cc^E20)ZPmbB3)rWKhqTff!;$^Q+(oZ@R&!b`~IG>U* zKI6%cG-%xIT)979fXM@2R$Wa^?JkrC%sk8C!QjCcBftkA{xd$l27X8X9g0JUf}1EG zDEma0!2O|%*T5Ib1wXaT0(R^M=NNlb`1Dx-P7C242|VZ0-M_UDJn)(SF^h|S|CAV~ zH6Hj@DE$^E^SF7k9{0B|=3J0`PO4XUATywJTAoIQqBi&PL84qnY7={0f+l!1Uua2Q zohrg$MJv306opF!7!&kFZqeNC(KJu<6QHWLkk*&h<)y*c8EhLup05S(5T`aR?)tUg zUVMs+0cJ8T+?xV~e%YY4c1PaQ!5k#^2@nY))%D;8?6@@qry0)53gbnculY7A;fLmD zX0KW5y!sqdHVsWoOk{X(ls=y`e4crlN8=kngq=WND05_Gl=W&k2oyHKDXgcX$LyP* z7rwlXqk$s-6Rhw02`Z+-b?oLQsqs@nNrpc#4F7l_A#Cj$@B2G8K~P9$Bu1J%wD0>Q zL^=yWorr%?IJ{#h?O8vc_gezY7?^*G{G=(Vn|`YyCtSBzQJi=zd4X+Y{-d)nYxbRI z!{{$_ds7{+|1(rCo{2maz7rh0U{@n|vcJ46N=^GZHE)Ix1Vr>28DdG{ifon-M>1Op zF4QIarWP4A5*gop|M>}eLx*cvoM264uH6tmjcZ%gUv zgX$^q8bZDQ;Lng{lmhB@+DeQ?XVCyf=h_+#n(q*Y?!&LVG>dzO{yG{P=CU|G5x6UY0#cfIP&0V` zSaS9&z=X7j&&|w&-lL>;95J}qNdITvW9P_sNy^yfU7621J@rr5rP?CKK5|l8Jvr|x z{<4x}DztmZ7RDq?T;5KIdflKv(;@bT3}+wN1?p<-HU${D@FByq>an<<(g!0BwfChL zTtaT$n{=x(CE|Ps277*wC4tZt6#T@xy1Fw)O9%Nr-ipK* zq%wod-v3WeJ{(r~m_Gc;xO~EzLK4jw1^&{9jtoTCfE_3-kZjWl-xZ{o(uvLkKJ=fuww4HB&6N0Jd6<4 z(4NMVPZ>Q{k}7|9S))A;SJi`n?d<#)b5b#F?_#Y)TR}(RA4R+retF&vD)G0fh%Ly# zY8l5PQl#0dQ_HH??`=3IM$b~;KBPd@5 zsM%!^UwYK^Ve^U&M(nZM75%FR)Yx6U?S1jM6fb(D#!e!Ds@_a`^#0o0&)3DUt)ucG zkw@(Ji)R*BC|LD72D4k1HV}c4Z&8Dcxc_o~x^PB<9iIMc)wG)-^5ihpykK%_Zb)yF z4X!g ztBcn|gLW;5|I8Au8Cb_b?ET6m3O`BADbd7Evg`-8&f+QIti zO2f>KVuM~3h;NdZzboG~|EjabaeaNGUrJSf-KgxGh1^M*-x1HtO<}NCMTdj&VmYO{;xmpwZhz>BJRyKy>!2U@%Bc7(JLB0( zyrBm@n*HCuSsxiS8Ut~0neomm0*7H&!_EsaBs)Yuc&SNXs-$jd2YdRXxw(1hajJ(H zV@q{)gKR#`p3u`jNYUw500Why#y&m4t)w!@7ry8V6t{}>=`w0cAg*KqK2BPpD zeUqWZ>X}b9trVR24UxROJis?z2FXd$vtoZb6k3}Fs5*ja5%J;ktEL}c(3Kj8RJOLZ zDyK2Oz#$r+2fe3bfB)L${^L22gGUk1(#Fg$S{az14T2<1p056HK3A(f@8XS0MO1c- z?rak?n$nM*?PwI6L+I?;p9u=`%TW;%CCC0Hxb*@~2=K$qXJ98KIzJCfWHQ$hhZBvB z&;b_-Ww%E-Cn+ebfX9x(!1C*kape^hvY*q%qeawqW94VVZIiTZqA@Gb<*6)G%FtzR}$f%QZJT<7(b$9Ah_iTrz_sntW1zZ3W|zhC2iV>QPNV6U=!S{ zLXc(c5mg5Ya&l>30PIrAJIc$;n_3Ij#(~enx@CM>8HU0av#^srVpQ8XI+mqVtC|jp zun}8N;mO(FlbMi|_%XhFoJOFP%J}Oerv~(k?QkoxDHAbKm0Z;S(uPNJJmKOK+)02e$rrpw>hiBB zBj*Ok-qiY7OVcE#^sIUu`{!E22KXVjg@rFU!;fmrZ$S&R{yK`?8M15;KxI+}wp!_O zz!}Z8cI!(^2cNXtcAdt$ia>p)rBB68y1fngh$Fqy>>wknsG}qOZ{eGvy!9l{4@;V6=lXZf| zzM1rzgcZ8g_I3Pe6bb9fb5i@zF;a+(SbOkMmb9{qn;TgiSQ)2OxRn8cfE(_vxRD+< z+m@(JvxAapGjTp2;VaT!w9y!SKSo59eh&Cm>Ltc9#F~+ECh9w3&MUJ|20IZ94qd36p0v)@lbGODv)c2>w->mIfM9ZJ+Hw-P81lJ?QI6K|UiulZq4vO6*jrF|$8|$@d+eTos z{DS2MDi>%zT}q}5!K0Lg6X%HX3-59j#(#>V^uLGq)gR45;818T@%HeE5x$Nc$@kG8 zmEjhuQbB|=V@-cjQZ|0P>NIB3Pl-|C5NoJtXvj`0VQin~!_euK&>^m&m^+Z~H@t!X zyP?9t=uX>Da_>18xEf7Yap^+Pe~Ykc8YL%9-7x8goN5Yvujx&y;2%gO_mBJ)S`)O} zKvAqmrAXW%UqBP5BaU?zo6&05I1)@)Zv?aGv6{t}cOWg#*9ZK_^hszgmGB(L=Lzv3 zaHss3a59)L-liUUk%Kfb(U6{M?9F#Bj31oPDJ1Iip95I!vEJjY)C?G9FCt{@r|-pZ zr)=iHVOwP9N?J=xi{zHE8@+DTo-L-}<6|a_(#tYILDB0_4tPUKi3ILa+4Tzw!XV;Q zW3>zhZgfP5m!6w_ts{@HIS7)-;bz#lD^C=!sabhDpH-9}FD`-_WWBxGG$FV+G+kUa z)HiP`0`v@D3eD?ZYTzjN@_G)mPGzz&hLCCPW?ATixYGilY=agKW-Nc&!2F?7D-KPV zYN{OD4Xwh}>kpVJ1q9ED!%c==M2T6)KoQKDW%x!3D(f+VUTA){+}2q*+Xjq*zNaNd zcrKS!4s&{<)LMweU*>qUHlGGOY^|E{!s*5K@vvR63xbXjhyu|H6C&+y5R*Se$9qVq zC^7fn-yeU@X0M?N2%a_?{EHbq*T<;9T ziY#Z;kzMhPgc=@Q-H-0unVqC&q^F$%+U60{*Y^oFV*Z6KC_9LGhULxo#9tT@QEYjY zg{X#qRJ?&);qF9x7Oi$Yx%Yj@`(6k%O1e$9)! znynv}12#UK{u~?AL@NDjh9^jbZ}Lx{*+NY0eJi+8BP$O|;kN(LzbMf1eT$@2-Ra8O zG+-6UzONzI8O)A)B#NP-K|fTj4gDG)74WM)Gr)RbI|bmZrG%&Avj$d^9K9-f>?m>db(c zTuD0SbS0pErkSn-6?`ja8y9`#Dqs|JX=l~CdFUSunuC(Hq4fHWr&qDlq9A_I_FWbS zH@h9+ST$3P3JWR2KZl;(RBp3>dP$TwGJ~s~oEdv{vR+9krF^BKYD?@nhY|TITZN zJ?*aWC9f$u4jM*oKpYGotGP&xp8@d75Fq^9(vWq9Qml z7U|DgV50=Z6w!LIjcJD-D=22$<;RBSiR%ZZ@D4PTT$*}1UPV5dbc7U?(Nfir*M!XJ zOyW6-Qh8)O3|JNl!z@#c5kthrRP6274tt~?65vOSr3spRWv17ZLHh80j?w3ohpp(K zmww065S7|z9#g8BL`2Pl^ z5@uVe%B{DK&a*2*_4M_tZkmq_5Jty6NVAn{T->uyke*&nbZ~TRjS6pnE_r2J_wJ9I@T+&Erfl3qdU z1{5s$8Z9YT1$N_s2OWckpIupr4))OtGW|rL6mlh0AjEi$^XJ4SiI0K0NEO1zgxp6j zWksxwo&~@?cv0wph9Wg84}!6J6kCBfHD(?Lo!g8-b#vVSN5v zvfJje51kx&`f$x&XvoWuinq(f{Wwtf5m|d~{P?&@#O5NI6&!OP7MvE*_pfj^z;KKN zC4?`)(D2Kd1o)qOvv;zlwhhf=~4>?(<|e35m2i0>*e5B$y?tsDo_p zj;QK1!3mXG_p-K$RNG(R+`>kvtoR1FZiz#}P^o9F(r(ii|1M6kT%6LX>J+#}nda*s zKYj?KjKB^c+W%Xn%IM)%01!ocRyV zEY|MC&pbAJEhZ#hR0!){)^7ydUs{on1%WJLr&EOm@0(PL(D1$miEJvPzZ*Ppi1duK zR8|s#SAC!Ef`V<{9b8=Bhenj0>*vPhzB+G0#`Gc5ka6luF?(+^cJ|Vpio$#d0ZOCe zk5Or$w2xe(p}V*kLSJy#Lvml>v7Wi{akC1CSEzV@2`HD_%TRW2>qOdr3YN!GKtA_q ztrb&ySmxzj4^3B803%hQA|uQ=HQmYjW^-(Snpr@A2`EzCDTyf4Gt!SC|8@iw6}Lc3 zVS-a!)zlxif)iL{y7XQ2x0xVreE-g}2KU=S&d++o^Z`c)<~N*#&#_@2M1T%x{>TQ_ zvcb#tk;soH($dfH$AUi5(}oQau<@eI5<5W@uI)td@Y>-U4YEAsczk?2q1Ooh?fXeo z*mg;*D%#RaQUh^!cr&{k5ZmD!x~7%BO;IrNi{w`@g3l#8=`E z*Y5KQX5lHTL9h9KXh_5$MJ2pE%R*y1S~612qvsk~cCI$F`Qz$MYB5(r0DI)sae-jVT1~QdxF)FWsigw#4fy@^^&s zpaj1nU3TBBlW>4$rW4gb$;l^J6=MyxlMtCLQfAXhUp zHlz;(4&~|T>G>a?*zL@3E~`yd{i~Zoi-4y*Hes$F8lZqJlI@h|0B>Wel=hJOgW^mot^Y{&bTnrMTP3LcOgQOM!;i*kVu44 z<4EL~F`Q=d(RI`6s;BX`JcG&f8Dp#;T1w_Nlx2EjBg|3U{43$B&OsU{172nh)C9y2 z{pFdw#^693*!R(nj_(&P)Hp^%n?5=)LFWieVW@8^Queaou-MSh)Oa4dX|1WD$xcr6 z>H}`I%BIGufOa6KLog0664ul`}wE(ubWcJk4Snwzl$O}`jDIb zat%nG&w6+Vs|xXZjFQ*OZ{~S?Y*S>Stniga^iLKW{)^M0!Jc3ak0wVgp0(n$n3+I&G$8O9w&Q?pXPU!q^K#BRaKT#J>E4YWL^`hJdE6_ zqlRb|rhoj|Vb$J#pC+K9)T;17+-9?+C%p>h%i>jUkANVE&TGNTX$#2uWe;%@sC;5~ zb;E3mTloaq6g-sGs^#nLj=l}reHN8)aMP^9GvNaeNCTFaPf_ka^(y(zc>%xpWX5* zLl)z|g~*f{TLZmWq||OPE^SWjFAbt5iq|iU@=fF^ANnFBK5uF{ku*^nn?jBZfOI&k zO0hVak;i)KR)2!u!i-3c=5j=Rm*W(;p(jr{@&_LN&%#(x+77Z<_Bg|7?{PF0N1@1* zOWXXIROZJj$TSV8Pnl@qJgwfF&%xBPgT<_3`<< zY1HjCH9ijKNjIH|&dJnZ3icS_4Tq)OP8(Dk*!33C544~30TQ35D=z6lLf8UQXt{YEgh*GNrW)sT&?ZOWt z?@+ZsD(R=Igg^UbE(^qM%F^_5IE2dw&M9DXu#yEKBo;Fb^+QmFRB8$sUzy9ee+-9cL4*?Ov^K_|`+rDB^ z%ovd8k=i~QJy}Gd>MGLIOnFJCerk_DWR z(dC$tp`lL#$e^fFSL<=j%sh{Yj^2x!p5d>on^xxOx}Ezy@+I}ScGo`!$g>POoj+@! z#|Rof($XMGHig;dpcBuUl9KK6OeUswCMM=Hh4AFqZUWmYDl$N2 zIk@}C@83A6VxiBr;-U@+c)z@#_qzpW9xXL!_RnH~5{S{Ni>Zm=ag*Wy|Q96uzWJ@LFRN~V^R<1eCrIk@S}I!F*4rq`FLb)7b^7AZt=-=Qf8PmWj!gC&xWbNTLcwSd+|;G zEm+|X_2Mq7@>Z+7{l}n<;NMnYpAY005)(Tw{QQz?U-~{FCBQENamslUes{&+KBS3x z&t!xsh!w!2F`LN5SkL-cGKr$nN;8myKASu~Mk(tmS z6H-Xrgcxnb`6DY_d~^Vz#JNpIfZvz#M*+KMBj(jq>r0s2W9h*0G29d0=rryd=OXqc zwgv_2yzQf%nHy-m&#Ur1(p0gh94Hd^=XR;npQ``-iAmQW+J0OFSnh??cVjX+>Bbu2 zxJ^;rGl-@Y9s~I^#fH|S;P@_kauwfSr`|xvB zDHk#JTV1n*h!JMv9ky4EQLqA^_|R=W3_W+nSl#d@Q>;|#Q)g5N9jF6-tR3R2SikmV z_L<|R>+U$MY z(dZ&Nuh96QBh{TS=xAnml^s{DPJ39~{>{A`e+zSSTlGr9p4@fM+LFzWT>>)H_Rr+v zz@_tGR*ZQwnp+D7tC_wg;V`D8Cd7;k2=Kk||2V^;;Sv=d`0NS$$1Rt zr&DI$z+^B5G`5z&HaCNZ6JOPowc+X0B}!Cf;<*blZfOUk{EAHl;D`~Pz{cF(#32rO zhVdUP`-u)EH6erPMxT0~v)-||zFDuv`Eu98bqgQGlo_*oJmUph6T!3Xa+;r*o%Pq} zW@W)iCtZzqT(xcrx=o9DQJaU6BQjP4Z41xhHoH4IIvRloWhs@e8i*0VB&=VGD~hpeh$71Ik+NN zGIyJ}z?!CfgyTBrTZSpdeA2(&;KK?wsfm06sv-C7~U+xJpGcg6s$(pI16Y7;mv~!`5 zsRN|5Q(!0$pJ48!EfzDB>6g8`4s&>GtFZ6aoSc|<2krH*FM8_C*leHln?l7S;M;h6 z21NDCB&f=yd8u3{RMS>!7$lXj!AD~~&wz>-?Elt8)J1btCHh*$!-T_hL3RM&b^KE zk_(W6T?P|fZu-F2 z(o$DP|GWPxy|&glf09_s#|N;@1BY0m)W8fT(vVId?NBq16jR2rQwBLedi?C3RQUCj z+Sql}x1Y@5(_hqXc+MLSA_Vy?bMQkDe_w;YJ)&!0$&a;~j?S*wF{Y2nn?Oc9$2Df^< zeC;YG%@gns3g!LZ!3sPUWDjTG;%`OGf#6wrqR`i$gYrpM z*nyyT2Y0F8U2K$=mSzE{#`WuC$EuuS(;5s;d9TK;*ngFMI@oaK@t2sUwGC)=o60`O z`JL}(1q>Xfhrg{h&s%)?&nr3B&&d?F=EF$#o$XHSF!h+SxKA+t|N1Ce(>d%jWXgjqu*| zbRQ~AU;bq3Tp80Q@5=R|mgVw>lJ$jUi|F>DzsS7%*CN`n*CA$ZA-PCp-42&00w%>z z2VvEVB7{UlL<%PBm33^dr`vNnw~qfqqm`csR6q6f4DE#IEQ`%ZWMiP+!6Lnlr^~oO zOYhP%%vqI!8~TPXSOp6o{67A$wlfOEci%NG^KQQ+w9TdG28X=5Hh4dkqxv%>f~Y;E zN)V`+rv{@}0{}3MbXIDvmr+Cd&Ye2|*)4g*f8=w@o;VeOSoH1J5&JjytEVpU!B3~xQa&cZPUcgdSr{(R7AMUFDh2yzGC?xbfK-uIg zv;j!o^c^q%Z`$WZ8O!1s61LR;>o0|!MJFpy|2|rx9dwf4QO5s9g(31j{~!m_?1)m& z1XdYnzJmc{64*N`v{^en#Kimwh5c)XzmLH5DxU@&1z|`RN1)iTQDl<@NX4S0xh?YW z?EDVbQhJ8ujQWLWc&6ik46J^Bw+)`$?CjtmB|c`P_5>Nn7QZ&}LU)|)-@w5^mBW5s zqE+R7R;iT!{d?IFu5HKL%Q&2$9jui6UtLC}fFrN`jo9$hL74x>eKvxROd26#w^(+f zqp2pO^7Iijf^~JCc-WV#2e5U$5(A+x6@6=mPn-%V9>#FXWS)jOaZKE;6!?-s9O&ic zwfs+L03vq$ZMPl%3YCRat_#3ow7_GHnc3NuigE z#CdBsIW;NEpG7AD?YMqWyF^v`ZFv28Ipj+S%7J$E&)xUyrB}4!bfQ}v|4Kn|uk5Md z4#X65hUVA&bXA!cA8?kY5Wm3WCl=2sDo0(8(iTtSEYlbs!MXAOuVb+ihtH85xBbe;zBF z+=D_u_{#_W-Bcm<)n{X5oYn1sOWuIg@@_vc-wAs5WI_gn-fFmA-W8+N6|s{c`A3Re z#NLAAFRg(IWYpds06U6Ux1PnP3@B36do>*WejUwe(AzJStTPqWzl<<<6 zZbeR?d1yW|ewfTgZdN|MF-d+Xn0dfIYkb>)rgCQfQ=CG4zyyxv>zT}NT)4MepPiCFbTY-1yFqHzc@DN+kSKya0V_@mpYDp$d*~}l6pCwIPC%+C|B>2 zX{gOY^xh0;Hr=PW$r6XyLr$LQ7RUD31$YL3vbp7lv&<2o!V(kG@nwtat{&bzkM<}b z4^=l4(zdF6smm%^-RoA-hW>OKCcgZ*m>wj?8{wziB%6i+r{HNe$Leh1XLEq{$NU+3 z_D?9O^w&(I2gSns)}Gs<;uhnC(|wA^b(7f?nI+u6^5P@ILr9*WE9SvWx*mcTKKxvF z&%Ix5uKv=F(}stTOL`~dDK?@nauJrnfM{`2Q|ny+2K`BKeK-N?FYEB+N;e^ido8hrLi@}>X|9-pJmQZkr z-zO+|2Bv2jJPCFFaunCg(k#%&TWxwpOJnLH_hH!4J!_qYq6Lwac{0q47Xf#iv^#}` zU&xu;FAMHi`EOw4%7*dLj^IEap5m-Fm=z{ zqjHT(jRUBS*uZTWfz*8v;~dKD{k>%XP4l(CR?tM=u((nNKnSS_8uk2AVH)8Lb%#sD zvNVW|mI`Pcxs9PQ{QiS`WyfYI^bjBIsnbvg}jwjRh@TLV=mZw?P7IQb-vr3zOHSW z?AgG65R0Js`R-jaD@EUn^Y_t{U!F6(PDS8$HU9seq0EqT$3k(0IhX|SxNKt@M5GMm zFU`3j!M; z^7u>AeS=8Q*BiR9B;xdWtmHz@>H?-pB@|38|C5nCY0TP0I(a5}IOa!_1qCilOm;d! z1{pr$iS1`UqNd?xo-*HmR2$`klO@oM-mr;9hTS=#%VN&W7lswC_Qqx=yP&YK(i0o# zb9Ioor%lE1Cj0UVI$-B&@V=XH5}g&>Mq;BI*`xowF@L3;q%8&r7ZTUrPrl)TVXg_p z=4FvYwV`!r1WnD;sS+-ewRwJwb@xJahN;BcQh$(h%q$yf$Ra%?YJ28Ix(~*X(mDE) zu3=A;Zs&K2z472AS4Fbpi26sM{%QK4jWr`WhQ-{Q2YV>fbfgvNC$wX_L?_T@GxCb> zc)Jb&I{XwJ9UZGKD+6KRmRyvV)rXUvO(;+^OzEKO}ZjW7buJ|pb9<)#VqWY%*!<$-Jzi#ydE%@dLA-0S(KLHBc*u*b#-<1#5aXLlzn>1oS?OnvE7E=z}{>T_ggo8UMOn24cnI? z41Qp4`E=s<3A>&~)KidK94|$1>ktS3Q;@h)&*hSqDx`nysO=P#4dSBHxcJkIKUnKt zmA!_cgSWhafy^^Xw)jFCmt^5-kZ2fYMhXm9;j`$Ai7Vkd`)1>Ls2iFKhc2F;UK6Yw z%RS*FUU|nQ9~Rs}JUt@m84La6mZq0sZpP5B)sr_gtCp;AMQ&lWIV4IifpWq|0IjqEMR zOB_HjN4_=v-?GyHmvb6^XeVO->9N03OjXBt6==zonfaYBb_WYTU}k#Ytom$!4b<<| z>MxzZeq@I1->{*^T4V()jnE!_eEb_i(^ZZC@k=X|9CdC{G=pWH7~YI&z9R8|9hWp@ z6EX4Auh|;8|NK)?k+QCex^?25{P{-fBRm)Km>|PO$|C+FH!Y7b$(X$%Fzw|FV4IW{ zt8g8GL@{M)OmzrNmYF2hBu>uTUIrV3xsSmuUcu>##V#7|d$t&Rd-^F!&s?H07DwYO zRJpU`P39UYMe6$s&6`L^#(ndWKj6}LaSU`2St$gmJlX<28i|6LhO*q_If^)t=z4i`;q7-PB83Nt zygt_VlJL=@x==gbihtf5LDPaySL&;HWp8m>3y}`xwzSAYn6L{3KcumHOp_COq0gSt zI328VTSNX4Cmcgw&?}5E_g1a^B2hy_3(xkt{WU^=VI9f+_H;Weu;ld%Z zHVGjdn1}YPmDVP}Of6)TiJQ5a%VAueG<>;hn9~HJTiaC4Ee8q)XUbf8QS+egaOb4> z|B!{dVEE^(1HAQ4h|A_tqU1{kBM;{h;B#5GXL<7$&>BpJN>%DFF)&JstDE6Z;g>E= z=s}>>rzo$iSJE$8Ts!=ymg2J(?6wob^!YTdpRA zWr=Jen48z}wQVh6^p26no7mr1gXpZ;|&YR)=QYG~GJ6Zz6S5tNOi|ARz~|-KOxt zrk4)yc)NiQEDDv_Y`hoI@XkU)sDgSD|A$vfD`K zUAc?D>7#l6Q=E_^JTi0I2n2V@s`8GP?zU!RDkp47rVO_vQ@0Ln`HEzV&MAWkbTBq= zWQq3rZLI2FX*Aj{4Th>8gh|&)+|R^d(eBg-h&Q|cZfaV30yhj5FX_UlNo4m$2W^pu zSnQRCx#hOc-_5bNihLU%yMDffjYPhM{O^3b2X?2hRxOBv4+)^pRwyYgFMl;Xw$i9k zmtJmlIXRXoNEPKR5#<_6*0CNtT?wLjv&R4MN=o{WB@RvxAfSX9qRRCuqy=iyx>VR# zNLy9~r#(*8IUehW`NgSNz*J~Eq%RlG*12U6hKJA>5fS@~q(jLuuW-AgiZy8C!%Ga* z3td1g?*vD%J1}NsUtDqZk9u8(UNdP=cGhU|s+z}m{&Vpc_jNITDoXv|cZZ~=Lbnf+ zETI28GV{D(eXLFs$^Z#g!g~SoLc_#IcAo<)Prm14G09{xY?!vAqjba*D=x#yNvW^3 zIb18$;m%a|iAjm}Xigq}nQkIM&zO}y#W=yW*(}mF4XKqqy~V!rZp{K2!1H&}9Nhu- zJRqPos;Hu(A~p8gkdk#**>GiXv2>;hRR@vJAilYPFf(w3HmlSo#x1R!oZtN3@v{MZK`GTGfv-avsOjMDxS2_I4y?;p41NdH+{xMsAif>NHD!!vH`I0t)rPcig` zq*HDvOs}MQY(H5p*Ve)JuOkE__<)WE3*{i~`HIq~O6aNP@V@31Y&X~@c2z9({^03n z#%>M^X#y8+b}aROdhPTXz_(LiW9*1Mr*vC6;0`HC%WNm$Sn}C9-tmLjwG6Vshc3r0pFY>hZ;?gz*EcF*EL^!`aq`PNntaYN|E z%vm2SC-#NI4&VIR>`*h*DBb(e# zcO=7@J91KZOk;JxaF0t*`N0uHK}?Ku*>{c=DWrY;UDvEn?2>xOA?FKaqTEpxbUbI5 zn0ZS1G9Y#GZoFgBC(CE57_G2I>2pienw}lImva_1V*YhY)%cYjod!SrE|#13Ylmy} z?4h?vV5aKdKCI;p99EEkb%nfVja%rxm`C=pU{3norOP?jQt;(LHcEHCij#La5#%>u z#Fu*))YmY#i8wWOuJN%BY$EJHn{XJz+GU-*yt9=JsR^7k3? z7?+UY9l;Iv)N&2+N)cau1R3eBYZW}|jQSi6r0&xs!fbJ&y((oDf5U|O8SFMkG;R`& z@xX7svAqbI!zHA0O50Q}FXMH|`gYCw$Aw?_c8ga27C58UGv&tJwZE5IE$m#$?zJ2@ z{*Rx-;nI6M^W-QWFyBmB+JXn=r3k{Plp!|2d!i(_SOvH`KkeGXAH90T&?+Ybx=+2R z$s&k*;^Bdq8g%wrioR#&mn4dL%v@R0!foNT*0?#t&?W(f0ePwiV?lp0*gj=CA_w~8 z@G0!UY*}Wzwnopddte=bKJ1=@$6%>(x%`lLuRUwBcNfOkO|jZVUKUFh*V5vYTQg{i zjev@}taNa%hrLB6_vy2V6it}b!;OA|xr>B(+#cE*wWgo5ARg(-UtHa8FaeT6YZkF0Z^zRl>_Y#N+Zs`P(0jc;$HXQv+ zZd1^?F#hXEMx&+uAApw6GdC6KOOHnE%-p?XThj1$W(R&XfOraigPgnbtP~94rfdKH zN@AB?%2_sQuX(q{wXty@YnC2AeLx>haG~&NtyW3i@Wg36&}!8{-}hCplphtpZrLL( z`bZp7(b(aG_@Z4;3SKx={q;89jv}yD7G#?NqkH-CC$;zf!Vppij=70$C%<;DTFH#q z*^>J_;Fb(Kk0mKpc0|{D8Qp1)R52&_kZqKLFHFUiDOaAue8w#hcB1-iPTDo9lRjO% z#~D&pg(Y?<(b!3l<-yCy0YXpSnuCmjY#{`Tl>V#k^8``)!o53M*k^WeTo{l-?UHZj z&Y`%EJ*9{z*huR7`Q4^46w*Q6_4B3xb$~Elh))jjNTpTl8Grk7p0D_H?KId5OtOrI zXobSFY)V$7H#HHF)m_Tu;JC~%>YYv^FL|CaPmiyLkSr(D(-|{}6Xa*l_y~yl8y17l za6gh#GE17?I*eyp2mWB-DDjVq5H)DYebu-G(xXKt^?V#%Ttrj1+ilBa2RmNcGqUV~ zX}69Nue9gWMQBDTFwu-Zx(2)J&UpL)e@u~-w0ASF)m+kT=t3H@NK@M%WGOA(E}B9E zY>BSRT*UWE^nk_E4pn{i-+z4ntJ}FlL7gyj5q7+%pj1)X!hXFW_wqsM$?v!|9&ckR zdq*kBlwCd5|KML$I`xh}jN=pj!1r(^YX|=rI4%?xj>5O)M~0BWQ;6+?q_Wp#^^N;1 zj(eJ?GaENfFK^Aga}efo{TQ^ji4AT26E7_}o)fUwx{-@Tn?3&ZX>$03VFlMQRmbP2 zWDtMOWM$C|AJ>gBy@dD+mhiy=|1Cv<>SEfs$sL$qzv5H`#B(3L{?-WV~U4H0-; zpPu17IxjY~C6%3(J#qt{haDjcR*R^9z+0lOyNgo0+vR0dWKbN67b-jC<5L?QUBZMn zJ2s{-(GlWv$_6Tsvf1HxjZgO24qGN36vl7k)g5cnm0a&Aj0E}U^?d~XF1VuK@0VOO zLS1yd<%T?Ei0+Ob^;ilSY3pAUkowK7=HTQyZsx?cSgN3by~XCBgy~NXe~4lrBz+aA`D$X$#^856;V!%PdaX0uMHa%U? zc9)vb=fdB5xt#sy4{kaTtZP-mxqo}Dke_ajoAU_O6wJFX-7?qr^@+&Km^E9u`WDKM zmXKrZBMH1*v{MbGRz18>{Z|xH4OLf`47pgM;NjEn-57n~W_a`F&Ho<7t7#T4f8Oq_ z4zooErh}@fctGTIKnH4k^F`=%szzek1$N6fu>Qt^vW{g+^!A9rU#2BChxy=h#gm-a zbdCCw@1KLR>W+6HpO{_GosI=RrS}5^;WJpR+rUJpxek<#D{n;E?(svOHDRwx{Odcz zitDa7j(39u3mzc;(k^|DyO%Fa->mWUY5a%Ag=4W7{#~4Q=!EgP-x&h|9zygM0-W0T zw1{8RdwYBMT(Uv^4$lieUyY=S(w@e}A+Gcg@zXzwhJLjYY?j_+Z1Cod`vNHw*^w;1 z`kXT}Mae_zoPtE|99{n(QSTj3_51&iKOLl$bdYR13K`jZRtF8+IY>6ync3^8`fWR4k1FxUdi5yBC=QZ?>b(e_xI<&+>X;V9^?MFKgJcsIFtOVek6Qi68?VjgY)vD z?`>qql>87DXP^D&^>s9Q_t9WSaiR>Dxgk!4uRg+_H@@>r0!6FOY7u4E{^nU1SBD3k z?#~r8<$IYc^P?C<^9ZI179Tgs+pRW6>yp*GH|zW|d>=5*H?hNJ&VDExHdhD=^`2UI zC^U3TGkXoKJeO`!Hlm{viM6c`Xuou+rf7SypoE8U6KP{JZ;WeU`6@~OAnEQLH;*%2 zIuc2vg1%3X{r+dmRmElJgX8GW&tpp@g$ZK+2&RUWzM!!fwmqq)uwPU}wX@4HXGycTk)rUKP4- zq>o?H{{2+j;fYmOFFF(i7qWP`%s`Z+K`2N72}-1Ozx8rlj>3B{CZlI=Onnq05g2VZ~f)t zrjN7ps>B<7A*+IQHVr1^b|!%aL)dpwKAB3WTq>*-ch_DidQ`PPG&UVQiKjzo!u|<% z16))?x%bpl&9$h0A?f$;awXlg;+6y_C{_2pT*Y9{Z-47TYMX7-g1F(pa5UVIPm4gVa+{DOG|#GDXeyOD!_CM0c!94&|G-4 z1=ZS7Zt`6#=bQ1hp+5?6NJL*wV<@-JJ#CDnm$k?V3|dxKk3e}eJ)LfadFA&1SvJ4V z!m2QfLnl~F%9E^=K30=QS?+XI+{8<#9P>41T;8+aqxe4M<)A-IYc4+xOwReCG1GXT zo+zfzJ6ZlyB{>e$Q0x&+^O$y60Aq|&(OZ87)50f?f3@&soYiiWht_dHQ+z&C<1A9h zv5}GLfB_yphML>q8}Q2>cBaxc2Tr(Zn5-6-(bh`aaj?197VV%vN=fBp)SM<8J; zzcIRX!p5Fz?Vbt_iob-a zIOgYp*zO;z2cC_zu39wtXsliJ3=@-~u(kc%+bgsgX_dtHMMGXK5AYAgi5__^R-WK) z3=R)_^=756J(KPf!X9d&*aZtX5c1m(;wI>KHm6AIb**PqQ|H<9-!D!3nkcJ6;caIh_1CE zZ`*izc^Urg?FXmH#YfX8Ap`|sW!9i$s`fb*6-WOj*{6y^zw1x#khcVCAH4ULJH7z+ zPA|g+g|U=B?xc_DosSL?cnBAHA8@eqe%OADC8B5p9X~8+_?0z?kLRb)38~pKnWkb+ zqp0ra-x3XlS87}(2|WX*B;nTwr{8(5OwhLcGr-@Y#bo(b5#q`v7Cwol?o7}}SHYzG zZYrb`*j_9Ob_RYDPP88(%NLCwj}pmg03%||+;0U09A)R31<0f%_%6;1bMnmJmPu7G z$^LjpWx6DSuul$^KeV4oX{MqQ^!Oq5SS0a|wu8H+#V24691g?p^{_82A6ZWQ$Qdeq z`bQq&w)AFKWKdtokd`m@z8!bmQcV*5Bpj)b%sB6@>KrgV3lGRdgp|6@l;rCesp)BAGu8_97jBdDeWh)FH5&VuF@U7gnXWA=$hEW9 zC!TMKiQ$#VqANOJNOi;}SxX^ZaKApNARJeh&Z0BZ&^nM@HspP{jbnfdSk)z7f~#As z|7`A79yI>tM0Z>F0auLh<-MwNX*~mP?k9WBZE{N97t+Q@K3?5-=rS+K;Zl=5igMn7 zV|E|TMCBj9xLo5P@!;Iqwb#=7j*^QBFTPxRyue)HsgD+~FPxNloEyZXJK1!fujXQg z{Yf6XyykCE(73Tz+q}6Gx#!rRYbG(j491l{8m@LLJmyL6KEF1Xpz$@HgoSP1z|kqP zx%l^_lf8mVOdeap&&To86Jx%N+2UnC5=Co-spmP|W)qe+OovKLr`xdbzT3@Y9PmD} zpX>i>boh5-Yqig+3(!XhpdA|cl$NDZTTHj8FDf|v<)RKl{^UK+&o{s7hoF`e6PtK* z|K0u_cK*S0v>2V1o(5i#k&zLpSt^Iu?Q_16d*2_q3l0sBRw5iwqI>@J2I|ANCSPBd z#HFTrjtNU}87^-JqdduG$~Y7|fhB)$No$q0ugFjy)>1LHFjU_cYq4(KqpN{LzV?rxi~ zO8qyRRq`{(D2amudjoDK9v<2#Xu_eB;UQhwPS3;UFJl7k8)c}5Vx@=Y3~9sueiHxM zU(Nj;fJvH}-I$jR9k~!G25RN=HX%=fCO&!;IA`@e;Z#NNx-bx?Q`WEN?Q_MCE#{*L ziZo%4>@~kL$yYFKRI@1!LGN8>Tgq?g_Pt)*%?XS>hUbi2PQ%1H4F8tL|y(hva6c(in4~)_BQaOJsKN$GSuKMdfs<7>IhUDaw1N!Q~L+YChMeR=}KMs~G zFO5sMc{yG7F+ev+{CvC(^H1qRWQAbQDIv0iiP8whcvE*Cg-NW)*0kz-{#JZa!+!cvyL1AXhJ!&+yr?8OuYQ$V~wlWn(#gqdHW$OU3=CeGv>>0;FsI{FfV!Q;~_iQNZ& zh}iUFS+VYAEos5ay?z|28j1$p!_8p)dDLxjMf2*yB6j9D-V1l$(O?!CMHijJRw*Qy zq$=I#*+_9LaIdqCxEvVY+ILWAe7P8{4Kf({+Djycep)5T!qpy2@)yqt+I*wJ?m0`bfeewyr5ZgGoZEj3`$(wO)#S$ zxOQ-DX2AB98y*OVHy@d)=1nr%x5S`DV}K$}e4lT)y^ynJ}7i>qzkr zqZ+6MUuoYl%aGd{=NbVfdvn*S7S1TM8mh7q_yoC;=|5CI*!(=c+nPP3(o4bPd-&~b zMYPA9wT6CG%WUuONtgTe@S4Zrn}kV1ehY}Ap@0ck=FtO!}*KTQQiypF8k2oC2wS`Rr0n0(4Zl~7^5v7{HQ)P#$g4-g--xR}l3Uf1Nu4U!D#ojE zm4i0y2tC%p8}dtq<%E)My-abaIiX)wLpi%SL(zmcR)&9QN&>OB_6m|zlT|}3jf0m_ zjpbhaNtJaD6^>5j<#S8(tsSm!W2GC~3D6q17<|#6>ULtM-2V z1qoaJql$$VauJ!o8^Wv&CoF|Usasdbcnlm?Qv=vhe9}j|Qkd1aK@NBsuuJIj828a1 zXIhL8OAVk1iLKjquOv&D-(Odh4R1;e*PoP$qZ z+V{Gsop8}dxQ0GP7+xY8{x9E9aFt9Eq6a_g5Cob1S-ayEUpSf4F~6F&;6`LMj8reiOlXDsQ7KtQ-&f`Bc zhERoVlVBdL%twgXf;RFjvpCEJz#R0jb&POrOV1Y|(s0+`=To_(SpK}QKUotO&LB*u zc^2Y)y20CJDQd9!%{_&ZoE-OoO6o&FmBMw~;N8s?sTvgTV2P41;gO?ez!5oaXp63} zdl4%xt-S4uJ?veJwAcQ+d7P+?$Rhu~at(nXq`Vv(J2~SrxE5WLr1OVy&;NMltq(@W zE;}zu_N%&FiEOHdmqoBz?}s6imwmFdCh0&8wo%O77dgU@j*fQqX1e(#_BBzfFBgj* zi>n}g^-4?vT5s2x3?xiPoxF%*B9Rgj((-sBA={i3*Dp1kCMEB`*cDIixAT{AOO?wL zrF*Hy-IqH&Z4(5cy-&9dh_Y^e9(VY9UJ8|-M9KgDI{t3qFIKm3cOP5dMJqk;w0N_( zIo#>JS2#I`=U51wBCtdIf1#3YZ+PJXWNHu%T)X+&?D{Uxnmwd7<0bT)8W0jmPi>m4 zM0{dPe0S_vuPQZfx&hi{7q$eM2>(>&OWS=2mBw6y%$Gk+PELkU)*bQjZVCIF&Mh@D zv#T9haMWt4{>bV7Lva z`wa=dUvkiUXLq)4hSW?v$@VlR7lCw|6jjWY54_`3jQVyP6t~07O2V2gv{f8@TZ-GI z{Y{T9VkjU6z70l|Ff4(U2&Yi+A_0M9$lpG1K3CDn*3sWYCN=FT{shoiZ2l(|##HF9 zq4Q53l{w(|{zLcS*&pY{ABbVI>TZS6cJ*ULCk^dvAGPrV^w(huI-f!nZ&rU#*a4gf2 z*WG}V&|h_0nA>2i-S9JKTv$rnS)B}ef;e~3ia3|flXO`a3PB>3LC9UDR1lgmf(Zc; z0(hCT=Y*YvCrFnQukwiBWPU?<5_o-Ww7hsn2VrMHS2vhh4)3)QCH*d+rb+(-Q~7K2 zq{LsU(K%*a`Aa={Y}$iC?DbrVVA24pi3hGd_(7TxhCn1m8t~JLH||0y)HBnv<)f0Z z!l~|u9w)dz?EJA#N%mtdv9i+S-hNy$_edHelO-Z$_cfId!NXFud>h4tAe}HpPQtX7 zWyk&cHz+UlSRRwJbxBDVaWNN|rKv_x<}fY3a@|$VS$XfQVyAD-z5c|&D-5}F(b&!R zHYQzj6&52|8))QSGa>)ehf{U?cgDvj|5A=4)RQFyey6&B?-`}itpm*-^tinZT!%Z6 zL_xfM{eyM>@++P-QK&ZKqm%ZXrdr192*OXl3vhT}m$KaI2}!jIN-+_j!jK;-+g9N6 z`df<+auM=iU|WMRR`zq+z|@op#mx5~-u}mPvmv+5C&)6^j}G|oVDz4yUCTlxg$**; zn7%D;+YsNnxp#O}_YAW`v{H1mmp34gFVEGC)W46aJt0yr$JH9!7t9VsD*a)w>jy@on& zQ+D0^@V53Ri>~ek+iNs?PrSUmdT&SB1O)}rVz3YeczZ5TENipQ>aPR&x3tfn{dm{w zDNz<%rrM|mlc9q!1533y{cm}*KAm>a`Dsifc?5LTLeTXgXuwqk@+D|Q^8!23j9s-M zQNl64XrsWTDRRv|UZyB{-Ush%IbYUYCtn=LKpYk9h&qttgzn z#}&^oIG4waZ=>bjKXAVPPUq;nd=BBGd3E_=Gay~??f8BgjLMu@2u)5_OWk`|$VnJ1 zI(G2i+EH!J7>9KxeDIN`pW~KBBe>-_Cz(d(0*YOY}%sIylgU(S<($_?0~= z@W|n^kDo{)P8gWHCq$>4Wj8)DQODHcKMk#}bXh6r_N-P^RjJM#BzssPni5(UnV4li z0)7q0%RyKo+4AjM&~K6RFob<$W~eTpSxvlqP2!HnHvdUiwoJuK9cilQsr~B@R_6NH z`f0c8mKn+~W<({xa292YIgq}EVR<;N>WrBwk3x{xN7|M*s6mR-%D=sZW3Sb?de1f4 z6qc0qeooL$t}t+Dct=pZx^evDI<;+`d();<($8xKHwp2r7G*ry;_j@Ilk$32^Z5t7 zYhrW}t7lwbLxC^hh%9?zNCH>y+}~beKDs-106Rq6NF4lf3v$lL<<4*Xu1oaDA8gTO)T9!&}P9|J7ljH|I+cs_Cg6Y$vh4&uzvOA9(P zFfS|r!v~1~w)}v*b3pfpoiXUS|7=8Bz=6W!_VK3A({CSar5Ky0Ll!Go>9#3&6)QpnU7V> zoYo8^`SULBTx0mc%F4~V*Iv^o6%lUGM=*12j%l z#IBOgqf`)f?Ca!L+#_N9xRl@zc$w5bmfSn5KAKksd()>{`}_KXvioXzcXANrsmBMR z1^Y-J=j)SzM2c4ReXa6P%$H9E7k+sV-)~7ksN@MTP4Br zpn{BkZ(9xQ-ieVL2Sm*D%t2~y4y=F)IzbSEriq@hAgGSks7n2~qF85I(fw$DJMzNN z-*^3gTK#9zGMV2SOHXJAh7tnJEZsNlRj^n>RuM?!!23`V5|Sl5!K&Z)4nO>vu&h9h zkvS}pXcc!9egz317#;F+&MsyoYV&4k+#Z}Xd8 z$s|!Q%)jG7pb60&8v)E={^5OBv+NsB_1~vNC^E7Y3`yR78hFEJd$3z?oB#5(mc~$g{Iq4AU0>fqKAD+a-lh>gzhB$Z1dDg*Qm?8bOk?gotd;Z=7ngSV)VlH0 zzN^hHC>NORN+#8N$~HdTDRB^8Mndm=g!y@-Li%d?(9~nJ04m6EXr7|jDZ2HJIQi4% zvC3NWE?20u`6n-JSRF;m4N|QnwV;Kqe1Gl`~5^85Ok{@hS;3MS8l-hUOD* zwk5#Ao5GBQG6JMu8+2B4%SVp%gN=E0rhEfi{!W7se^&0xWA=n6oX4NMzpfKL!ZGDW zq@`F_m5x8{9=~tf_GKx1H6@T>CkTx(j}OBoIW}H`>ma6!>Q{4B3&AjBd^%zEgi-L= z^-22Ay~?!!E{Kl@Ku}t|F{HIox1Bb>eS2GgccX?#lK^WK!gva)wUbLe+WV9ls8MQZ zSvr|-*m&;R!5?0L?7N8V@jBO!7hX~^$K|(GiXLtwIwtAGD3sy;EiDsF(t=Mw7vlK_`H80mwPSLYWC!<0*&+)Li$2M$q8s@u`GDhuDU>VdO|WtI=%nGvXvXpKYdE3y9BSX z1Fbo#jISShDlIO?kzgQdlpBSycEI+9pzT)zv4KWYqh=RD#!X%>T-42;E~9d1Kx~GV zTLU#HHWQQm$YumHsW>$&ULS;*GcW7Yv8!SbwM9B5E;dsR>~C+EoE;B6JZrXu7KQQP zYASLYPC_sF>=RFFeIux2?f1Qy)P4(Py#svPL!91Ysvw`(%V^;D>T9xVYgLsM7Oq6F zn?J?!RHTZ(#9*p_ESy-b1PPAtt;v9S>WZfNPGBm%89nfKihKEfT_l`sD5qQP51{ZIma*+=fA*Xm?L?cHtQb$^aUyo9`w?*PMSVK{KdNakl;hNWD;LaL_9t6BE;h~QN8Xs1mnCJlPRdn zRU4%jzL*6Va2b_?4PQUMP1u$Xp&NSZY&PR;wp`1=h@iA%jsa2NZwaRf?+>d7SN@mg zV8nNn4c*+O4DsXLlO#`Q-!e4?YS@Im0KRPHcVBYNW7Vj+%M46*`M&l6RXX`7^f{%} zA5566eGVk$Y|3~*>=+NXE%3%PrwMCV%7nbnXnSim0WD5$U|oH_H2iP!46^9MJBp9!?VBKxB*LX+8w0QUy%~)K z$)_Qc`n;N&YxO~z{PeQX;-B+nJie1OyrZnjM^(+c0@jYUiLAGJZhsYa zv6;9aqWxG*`$BZ3YDE0Zl-1W+Sb2lXpI1)xRW+ow@l{J@%GT&j#e{;o$6N!~~Am!BDTSvHlU^mM%eL7feR2|E0UGca*q{^*g( zdes-qb+`$QR5?$7e5%kO05j?`G`nj*-vcUv)7}=_0t-RmLN>!<_{rBJo=LRYqB*<( z%BU|{ePGCYB+Ln*+!vyc?9MxHBIRna9dgC^=_4YQ$Pzm&_TGb&H1fCN;tRWjDP4A;tR@g{(nu*CERs^Mb?X#;Kh^#-Le!#}arBC?)bd@ZMGJ#;AE}O_{6%I@j z%MET~{&?o?5*AlG9JT)bYGIGHRDORaSUmT&RlV?F9AqT?eENpPvp)-<8AY{VS?=yT z9qP}UY|Yq6b1n`;U3cRI#igj?V$!t5A3!D_5WUF4>ikwmu39C$O*+e-7~Tq_+yM)w zXossQxXJW{1cQ>O<%Ed|iy-MvfnZ<7Dk~uKF2w}7I}qE2^)^<+7N6#x5b{i@@i$br z&S@EVZY?~KvyY8wx7xX}_B9WVL4P>IVJ-An8@1Z1MdR}H-8Bo)yX;ZN)Ei>xu==8s zRk!}znEKLfIVuLo{$PK@kk^M}(1&N0p9f?~mxaMAjU=q>R#g ztDSoM4u*sp33}aU8-LvR({Dnm64=hjnq3kqes7q!QEOeTSU;^XP-)`o`X& zF#HBFAh~k~S!_vU1M)h2(stdTUz)kJjL=%D2s4eUXku3YZX>0QfEi{94>mKpWHitVC|NM z0=>mSml|I<7>)gaW8P(YKz308k1gd9R=}XK>C4cocB_?0Bw}7j1UW}6U;r?965N;J z0d1<)a63M$zBf|wHx_yb2@xWjPD+w)G}f>7TD9tk6K1)RK}M${sW${nAY>^;G&D=~ zr?(o__Kv-O9C77$?>@XSysh|GXS}KFMs>LaL|5(b<>T!+M`#W6j@|sk$pjb>Sx&?} zIEpinf$PFI+9uBAFlvj4TlRI5R1H?X{RwCyTC3X zxO1UoLZY=Eue#sc*Dz|s!8F(O2o-Am&6-kTCglFRo8+GAJ2Jw*|4^@v((dE${&~7z z#&hY3vvY~1x{$wPuq>C)M-7Z`fqkIgAo>Agkx!H`Epv!mF)p_(OSDoK4Xx(1&$%^>z9h}OkjN3@ zh$|AM!(llrl36k>V1v>Mg>C!ufs6=yQJ+xTmA~rsFtUH!wQY38IZ&%WeUET!6Eqy$ zO}d!SXqx~J|5C<6G?xgGgQX>M%Jzy^fTs!?*bKaA_I~N61t;DH-88?vu)x3r1rU>7 zR_f5%Z}QC#dKij1rC=^>Anh(&sbxS~jJ}7wdKy1y$J$3WzlLrKZvLuAf2!|_S||hQ zSV2y?UAl!Y-!vNUa6Z^SC_s?pA#Buly}pjQ(qe%52ys)`x)R}KX6v4C?^yw}gsI2i zhW)>~J^w)GCr<<<^&zbaPvA`QZjxf!)^cu%1q~){*VWR7ePw$23~SU%G|JkQG(NLp zCVsRSiVokR`ClXkXS~@iUM=m!rjX7^L3DB@yg3l_%!J)>$5p>WuO83fOdf@28t>hs$% zMJL>qm7!O@OJBveotQuC)`FBU5fO+h=+1c}r`reURl}h(5ZClMZ#cyBO-nqPJftlO z90ZuuOpYUi*Wf@35|PC_Fx|gN1N~Ab75nYy`TE6|d%Qn)x3>OTJe}7BAuptK=hpT& z7w(4o`wC+fDiC3@TDOP%8)$prJpU)fpvLW&0`{rK22C%tZa;RbN{c52yfjc?6DhPV zgRxl7*M1F(z9Se3KDqU)>>6A#4;UyBZl|>q$iWd>xX+!3C8TVaMS)*+5*EwGKy>7B zD~B{Q!FR8TNFB~TQ-cgGR}wiSS~vFmU9YB66D#!u)|%QinQGo^@u0fLd_d^#efZDS z`SUP2!b{CN5}R@#@(uZd7)l9F$!#sHq$2J#{J))o$LKtRq23!y&46u2H{gc1hi{tr zU&)p|p&R*ia4f>0OXE=R7&ZKq1`}Yh{CPdCv_-D$Qg4{8*#7hO9ANAD{HGO5k`&wl zNxFJFeQdE{O1fQG5iNOvsR@oDX}2zqZnY40`g$9waSBZHI!1K2|1u z4-9#N)czZWNhoiMp5&$&K~;;K6AI0ByUBMjOS*d-*(xu{i6>8lh&ka|=sj(V+MEUY zQ|Vao3x9=G!8t;sq2Kh2-msMWQNC%d;J;+!f|`l(-UV`a&CAnGF6#DdO=m&Vyai<( z8(%?B`$eHb67eCF*F&?*h4O_;>Lapo^{bi=`z2U4sg0*DE^A@reiqn6tJgN!S-#E0 zumL@qYx1o&8syt3E;FRe{Lz*D7QQMOkYRXB#({Vfn0j!cV7fBt!+s_H)2;3W+wi2&Tj^s z->zF=XF0rx2~+(=Fwt4{nctu!@aka#XyxYRJr*kQnUs2}s;x6x;>yUmi~o6uutnxh z??LnJqx85@tr0*WUlc73_Pq80ii&76hp;NynQ3SEhfIVbP%SGDu_XuvB9+hL`*I^sk$cW~f{A;X|tnSW_9*z6p@6KTd23~x!Gi+G0{$PBSz(`q2)&g zom~S5O)wQ^_UA#SF{u!JgfdCWqoIg^pf6-*VoU+gY`(hiFD^Rip=ED1ml~ozfsTNh zi)OJzUkI_u9+`D!YR`x85;+&KkEy*N27!k>dISnIp?I~8*M7*ig^`N)!N}SEKeIT3 zAFv}Am9FYsSh4{!d*#jO@zzzhd2h`;?5CM$=P$0~7~X>+-kg_eo;=IFa=z4pgXV|W z3@>7~=pawV9##05D5x!`;>t5u*!GCa&jhV#jsBGXh}>EUy=sP>d;#6f5UWi5B4 zDZd#$8rm`?U(wG9n&?MuuMHJF3P^)%!e@qQTfQd@8c~!k%6aeK@#2i~ue!UKbGavD!eJL- z{A=Uqm=6dJsO_%IQXMpl^B_fQjkSvL6@}B2`Jmo-dNTS03Hqc7Jd7xjwB4)JQqGI? z!O|(+*1N)4D+M>`qT=qqYY~bwMhV{B^q}R2iUM#ppKT)p0_>o7>uI`X;)RipBc3Q?Oa>ol08X{RgUV*bm(DYap-IeBq zYPs)QW7}ekPMp7tI5F?K4wh#cW{ZnYoU6XSFmj`ju$!V+^vHqn23WX<^XbBGcZ{`5 zocyx^W$^R{5_YOOUv~peB+iH$Wxe}ub~i%d*I6Z!k6iv5bJ}c2=Vm`xX5|hfY&w>; z!7+nwM~@D^B`p1{Aj|M&qx}UWk&<}o$g9LmB{!mkaaEdp#ng4K?SxMy`nmS%^*C{HP>YMH2QSBMnK zAT;P?1pRd=tYjhLVatQFU`KQxnXB87{Xs#~iQUISY|*A>Xb0{CW`)v-rTh?GP=-GP zYDR8XcIFz$ZlB>G7{|!%GE{4K&k5^w0E>D(rWQ{Kp|NpJ7o!}dWOahn+@*_9gESbex}(_*0V!#L ziWk0%c4LRw=%<4;d1Dv#@%<`perdyE zs3mX;S@#m__(YO^#P!ZUAmZ0^N`p7CIK}4S1X{rTSl_m!qzU``^4^NdUXeO+ zfHNl=H#)5E&N7_=E3x=%r&ND2YkU5S9Zy%d z>ZR#FcAggVYZyMOLHeO_6w>0MtG;M9$m+&O>_?GE@7)y@0XngG{GSbKYKdg-J+t-& zEIc`;pYy@H?_c}*Bc;0N5!ds9CyZ{Zy+U@UjsyFNfPMoJ0DJarBr;dLfG8S> zs!B<50Uf+P&gY+dqMG$|JG;Gu12@x=exaRXU8?q5rF@M9hz{R{>{Uid?>loLKeqZf z!ch)ThUtrQjl4Sg`jT#>sR6jN;Nn7wo&(XkEiO?{)A$okEq~EcA#fv))ZCMVDJ*Pb?RzXmUJ`>IG9j>MdvW$Z%(HV4bprb)mB?@ke zu>f&6i3+&EmM0XaH|Rt=ScUUhUt=9nok;c2qztWIMRmt0lw zUu17SwgJ2XqRLp$sP)PDS!+CXYaAz3P^6nHvp@>M53ef>RKQ~GntUeu)0zU%_Qs+# zjtyNYbap&kY#Rmz^sc!lDkv{vJ+uwojk{CVmQM03X*EtFVbL3hsjj3a$F8$Q9~q#+ zHiSz4$Gg`y7RD+aRxPdp)qcH0gkm-KkIg)Svi{$v0AC@`|pd}?5Phjg^`r9R9u7O0x zoGN!47?1x*-vMpvw_!sb7}xX5iDfTIxw7&~yOk1m}$xhR?U{(GI|QV#!AO@kX)H)(=} z9W)zilaCVBeqZ9e8&z#lN6|bsfozxwV`&HB$Aw9L(5G?mDvgEYY(@_16q?(T{NDm8 zaVf_KrlXD@{N^@6Baw_|@JnG<|LNVMaRzy}TOx^a6pg?=crdi6)!mA$q!#uqFH(AX zG7@Tqgq2)3gf z>A`L#lIr9>PMaRfX?Cw8#q5tyeh%d`wpodUGJbkU!{|)+xVt?F*@9yOCzZ9>LL@lw zmMxCQ>#s`oYsyL(ffQrG{d*WNVR<9r?bsAjJta zkLjLk{EqD-0t(d8DGYM!YU@@+&E1(*|5J&`&N=Z#r~zIPo6#6!`BQQ6dTPgNiIcT}8Rr&1b%9dKAr*~d zBb`rP#NJo4t+vl9zhCyV(d`uzZnCa!^ZpbxsrzTHQtU>?AWvoBF@S9h4h}X1m6BN7 z&lTMOPPCEdN}OEUO9zd*t==W;ocIXUrz__{iRvVdrRHu?z}z{(N( zNG#@fZM#%BWUTK6pTX#2z*o_qJ&meaG8@OQez!y`icD?l8+#18Z6Aqh$j|tIy?~M< z7HRvYymVyXH)1Oa{yJ@48rY|CZeG-VC`p(Tis<7}B?1nCOQ*O)dS>7^J0TtP3eT=Q zGc+|l0sm}a(ai1lV~}>!=BIbSw7uhGr*$}Sp8V)hS-MlpG(3b6PAX!C4Q+g*-zw5vsrsZ$ov!X8Zma ztE@^3*cqKIIk8!uW$8Bb$@29d=(Ws&M2R9&;c|Mfbfu^F+C*%pZuRR9qvh6HTJtsL z-SC!4{r&WwTO7fOVe{KAu>tR(UeO8mEl;5_ubTvKnH|-cAV`H1W_u1a;Qxe#u9UMg zzQG8^{;<+>d6?T*-RkRacKqG95!#%&%n6nf5PWcz>SVKG`-JS<*!2eq-_l6w8ImCu zahTTqcW1MSU!*im&N4yza1B%fJ=p%YISB(8N&azX(Ok!C~n#`rRZC z@C^Jhn9J>?5^3}S7-s6m{TM~QH32#C?X3zjGi?CQe)lYhLufx0dA=`jqMDy4JGHDN zI@2e*n%k>fzJXo8*>^Q6R5MO`>A$moMQjUzno&0;eVX>cy>7hPb#}CV)%5JNX?{zJ4XPJ4qXyZW)hcKdfwCISv4{Gg)ge)v z6Okl;s&uf@{F*)gfq|!LJEyl(O5c)j6N=N$z9a+{vmNgW_Ji(2?(|F3rV1kC*BY}Y zj=Y?xE?oEr-yC?se%Z%jZWA1=yEX;r9j^bTf{2F<3K~OTg*r30FPzL@iSTIR=0INN z)chjsL6))K+HUYS@sEEX^PLAsG1*f^C7-d)T(k1hAAX_hOVp_&d3guqTv*T3r0lmm zbYm7}4+KiB^0-Teqtz&ttj=xauY3~x(yjR0#C49Ei^gXp?tbNk^WCq3(L5h19FJX; zf!B~BmNI<9U{?+r^V}Wd9{$EMY{o-GX-vBQ-M(2(T9Yr`kZ6D zzriZnw;-sYP`^hgh@(eoQYnifc95p*wBzoe#`>V$`*1G$XlP6oDwOu9lY+g zni`#8E>O?7aI!?g4=kE)M`+A}2*qP9j&{ylNHM?v>`_}Z^5#yV{5O23e7midt-&}9 zt~7U|va%uzp4v51i8>}_`;jBD>5hDPA_|*`*2&M{%pqg8tge96c^IQtc}XM^Zt-q$ei*yofmnbsegU(U@bpQe4VYxZ0jFvmXhMT;U5Nv{r z06N{%R;fkg#L~Fq1x;79T&t}hxHkEJKJ*fEWZJsisti=AXS_{Uo*?~w%Rnxf8y3qVq2Wz}RG=UQa|+H?L8?&{RHA&2 zzOX!Yk4}O)H1{M*L9!J*cD4 z!u$T70*1M>B`8CXFqIw*yKU3-lByFbWaX!C@bY!&FO=Hu9W#uYOH?IxN+(_t4QUqRE9LR$mlvh;5D!%_-OBbXI#BLV&*7H5qCaMPSv(r6FBC93?WxZtH zmP+u9ll|)$X$3VGq=iMK5c=0bgU@dt6IKWr-aHLc3efD$Bby1o6Pf_MT`#e->~Vj~ z!;9G6z=A5(x1Usijl4yq*m1CsSV9`$NT&P^@AM~#LP}8OkO+xXFE~gM$ZK_BQ3irS z0#iBjrzK1$KDT=%HfNZH?E znG`p07^Cy2y~kg{ePf=2-G6Q1tWqa9%=Uav3~ep+$YJ%z4-N$3!l}W*+Jg4UfjhJE zX=+^YJX)LxYQXOzIK@w7Ho*Y&+}D9%NLTJL>pCcDdt7;7~$-1JW@8P z9z$B=)2CTfa+MccGb(^ya2jFi z>@P|6Gr22r{)h_>CLx9TY`4y-Jm;EgW5URYktcu;{BMsg=lcs6mX{}cm8zRJ>#(P4 zJORCE83b=;l9YZAdH*j3riMg~{$~8|B4bgg<#{gJ745-1DEb*Q-#4=T{r#I{ms~S= zj=`;;Zuz~rnCQQw z<+)$)etTgw3cXl50SC%*3SdDW6&Ud*^m#6i*k>a`DaPnwMOliSXaXr()WcI}LcFb+ zyD%-H6$ZzAnm;8T^fRf~5Wg#>5crBy69dGiqOlUBkw~P5gR@w`*b&WB&7}WtivYkF z0g&EHR2xn*^4V^#J$>eO}^H{{LSZ&U}p|Gt2aBe|;p!1kU6%)eZL3 z2Log4^-3B$GCHp|egac`t`i+W$7y*82s3O+HUmy{uI#@wI*G>GuAEwR;`U0V%Z}Mm ziA3kl`TrLAS0j`5oD#{Y%dh>I2L2rU(DM&INa1-YO#DmH-2qPg|Nngkr~%T0q&OGc&NlXeY@Pf63Hkp=)O!a~{l@+O z?{ln>oxMV(C?YFch*Az~0tQhx6=`>od~v=1>-3VZ0aZ`pE(s!72`~u1ul?Uw2c)IX z|88T%Mv-sdLR}Up0?Ge0vMY%N-E``||NnnKwPym$hNe4R0ytb}nW$O5PyO#>2cn<< zXtYPDEKkAyk*!mUV7*E2rOW>BYsGN9iz^>9gBTdkM3P^{1+M>h%U?-u$>_wArMK^V z|95p+?ceD_bV$hr@^ARx7ZF@^L$?fzi_!D{-R=1Qd7H_fGxHLbTu#e${=ds*17?(o z&{?dg-_Pm;XZ`@Kn3#qCd(7?tCEOAIliHGTeo0OEec=DDIN+ww!b;qsmR%C~=Niw1 za?;5DF5n{DF0OQbzA_FCTrhSv-W3`f?(sl-Ky)~`m^7m9a7Amy*dgtHaXD64lK1S-bg2PMb+e2ANe&Z?8YOY=WtD6Y_$gy`dt? zA=8=a9hfqsjilB;OywSZnF+d48y5@=MsJkGuCcvnAJ|cryo9+$^&4(++C~tWmpiL0xtj+CUnv_k4OUz2O{;VSkC@`_F$c?Y+a)tg zCYyEzhS@zIrwVK2*QVz|S$jU^arPVE$BM`_tVFjudK|Z2Ck_b%$#R(28`6KD)(M z{_l{p#PX20ivC{VXI@t=iS6)S?YW+ru%EvnES#aN+mFl-HY z==Rn?4&;ddJJ0ZdP|$5)E9!q=SNAMnS}=b_5Yn~T&v+i_XMx5St|Z)%Ajx4mpejlEAf|?tDgwdXfvi}5Kpdi~v+c_Sej!T`v`}gh zIR#4oO33VGr3vU%EE9Jm_WO4bPdjj5Y<>=VJQ|j_se^_%SO&X1~{GJSFQia&tow_KHki80% zgm2nWoiM96+G}5i?-%EvG!Iz(F(F(`-Qp3wa;3jXYLErz7h`da2CvYc5(?E(E*au7 z&*Mr$R7qVol(NcAi-cVJ^wB5+-^vDQ7IYOB?!-_ zuhH!k+Tt%H=7BWZi^^YX=UJbFn4S08tvmTv5xFiCwqLE$5>#~v2j{P*YnXqIa#3%# zZc)jJ4arThsM&L%3>zQqOr6!aOLkFH2L)x70)=qPqHp-M3f+-@_}&+*H0?=%PM+R7 zUW{Gog^oa)<=4{RTM79m*w_$2shHDG#wFht<6w!FaCpE&4w=l%4+ZW9w^-6p9k(EVg;N zuRlfF+xk32WBidGqeQi2XkP~^>4>8CLP0h(eRcSkmgL*0_gFW|N7?~$&uWddRnxj1 znqU{;_g1g&{QGwjo^K`E(ZGc%EZm<}rOVONs{1)1A$tLUHvSpU(%il9CMAEPSyOA8 z(x&!9d&6Dypk<9y|1NHa0V$nk!Gu$z+2o04Tgo&zuJ)1GqZoNUKMCBX{se5e8Z(ZQ zqJ|NhE_PCru-q+qU#m{Iso7gc6@@ZY>+3LGv$J;i5c`$3j72kp3n3GpD?HfiAl;m` zH&_KEqp6^g7A%PZyhyWrrq%i>a){vGmMGoIXaifJ?=K>BOZr zP|S~K3^+;pT~a4^QGy-DzsRu%UeL;-!cjVv7xe^OMoxzmA7r?@9I(ZNoYow^zjzyl zL4htylbK^mjmwM9(!a*mmxU$24Sk_Eb_tMskP_>|ZS#5mWB|Z2_bNgkTBbr4Hl6jA zdGmo{TMwwLuJ#JyHT1Y~{-3;ROCrMzzfEOCn2w>guFYHnS(SVKU+VY`^A6s^^3&#e(_}<%X$Y7%l0e&o?P`t1yL>Be`c`+G6KcWItZxjf7a-F zb(Q;mhq8R8KFKM5HCcS-(E3@|wZ+x1U(|!;GLA#XN;d#DAuqq_hA|$!a`GBkw;mai z`e}7ofk8Yfhe+-?u9TLdD72;-2w%jAk5ti2pDM;t=8rmk$RR(5%F0rj%|%5SVtB{i zS%f#8g944W|IcDAlAqbH<+rbGUj4YImz*( zX$Y^s3Fjtd}nJs1p@L|8Laj}Yk!2lFb% z_o%!so12#!;87}w(ugkh?Sp9yF(u97Plfl+;$1i4zhTD5qKJUG-*260Zyn{Z0g0GFASm(>+~s69>L|CwMqlWCt;RfaAgDWVF?MV0YS`X+pS;c?6$n`9HcP61zQU)t}W%Ks4<6drD z|H!uA`PsirA1lU5&t^ZsChf|5ix-c15!r&oYgW4lfJ6FEORu+Hx^~K+1FYo6h`CJg zyuPfM6wh1;$+!B9^)UdnK#T&0_Vnf-~8P# z_mMlavJy}Sw5jXN2f_RkAunJBKii=xCH%%XSb}8k&FQTiW<=3Rk@_XB=~VcEtx%#Y z?}t2M_n`*gy$?Kf+)k}h!Jn@rO<%yKHHdxoDLK6^xGB6FN`a=uGTXw-cz7>%&#~?v z1J6rvIrLgfY%3Y~hrYQ^_dg;hK-E|e^i~g1JX-~m-(;^EdzWxr0U>>Tgj)2Tn{cg}BHxkEQE2JS;wTX1VSx6pA#Cd*G%cC$^Q))U^uJy6%kXIBL3Ko8lJ9Bk=ve zF<-Ubwe&85Aq$|58z>p$e#CXo-lf^wAc>*1*t7Rgi7z`>hsW0@Ia{<3g*)qH3FCDjR+ zg`CTu0m;NbL0Ep4KU{9?!mYeFK0s2wYjA8hvqID4$3{sMQu&<;Wk}ZDefg|vlVJ&JovJ(!B3+zg`~ z)X!xv4Iw-8D??Jji@-P+t51GqHPK_|G;YpH^ zY7KhegaUUo1L#eX5nZ3l%lwxLWx_I;-irCf13qE?34|%s6nJA&)nS(nw(mmX5&4-i zCAdK)1jnm;K97!@0|#Tx-R=djB8fVctlf^cI%cyeHaX+@41s8k%KmU%$oY^^fwW&- zb^TUJx)B3j9k<6WwGXsU=&_OX^}9Lsm>GL_R}a-@uIC}QqB)H7da^|Hr*c3YyGjfW zYKo?CmO-$}e5Q zrXHtYKY_gw&mIu18NFiztQg<&yw}LMR%($%NwH13`2E6w;BafMs8_b9T10`5)ST^k zLVnj^`Oq5_+WSMj)7Y&s!&ogpdJt9ejs-?=A8s;c)-R{pUtG(o5-R=fgY{kb5wm#A zLw^~}e7!&z8#E4eK~JU~WfZ>YSYs@~Fw5W6e$nUmkcE5^S#hPgvB^BlgxV`tuA~@G zUdjeEs^!)7eqa~C&f!Dq955QP?q!z>tLb=^4YuDb~M(*Z`xJ}bt zou#x14i5g!TbVU}pe1|o%vAf+^X$c*>M<|>Yx zsqNO+uOX&rHSdzAP*tr#JQ?jG!N7RJiLh=jEV?~OFJlire#y1W~@R{f9YkYw@WWO~~Afy zjoo*7&o#`F{)B;*N*h@1C0$6Bc~(SL9(Lo@%<;^>O2S z#Kn&sb1yRvaOC5rGgnJAE%w!vFyKEjG2T0H6mfL0L7kD_A%TC2tP6o5FZ1XnB6kWuv?-1trKmWMa#cw63FZJy*!S?`Llh;UG#Yc8uB0bX zofRGb5_;zcF9XlLnep*G9VTZIcQljp)64480;o5{FadS8(vs``oKWx8XH1_Vr4sU5 z_e8Umgjp%%j(hTk9?xJD`2TNQ+t76P2YJuMu62z@f8oS^2w`21HQyr%v`q?teQlWe z-~(0Y3hy@F_5}K$6=-tr;r&o~w=V}P2`lR`PTO+&Cs#X^6*EUY0s8Mrfk6!n`zaEc zWl9Ky!ZRKzOodD zdng?(t1dHlfnWnHKX`MHti>JSSI2JTZtnXG;7;uRrkI}XZnhny*sQh{_tDeqb1mUKWKg7>RDU*7({IL^zsBL6k2Bz&gf?U_fdCcW$_4H%I52I6aTA}Gi{f! z@nltRWOT&u)Em^B9ptW&=JAvP-=J|=ZdpWkGsI&Gy93Sxvx_(8c%cR={$RqkhR-H+ zba+E;qY<*Rz6AC<&YjUwPZRJ$9eAFhmZPcx`JQ>LVYmX>VD30Q`Qey>0Uy1xK;?%~ zRsm#J2BUCr_)E&_>bKb%%%h%A(P;E7rmt6dv6EyenRJ^(YZEd2OFB`I^80j2Y4JL` zOD{m<%DaE?e&Sf7TZxN@syF}rKy z1-oP%UBkD(t%2qj-lUDrawl{S%3|-XNz$BSojF5zWd|RUA|sZJW3)WrNcIm>v~DEK zl;;wKhmX~YjxfNds={3@yT6fRBdMeI(?7PvAXLLpse#Cr^LFEb&ilN16?){kfieC5 z9OFol;P{8cd;yHf7I3Gjs zcnoXpnwTzG_tXY5Qpv%A6(Ejrs`T?62skZDzB9jEGQn;1y9MXg23A6okVUunC1xEX z&=fnZsUQVgdIh%1uguSPg_;lx8`A zQzS@!{4ySBc{Yq(wXz(fP@#-C50DZm^uOvZgKE+cpURkDvM|qgMzU47_5t<8>*Vt`f?yM2S zTE*}|oB2?vU%>1s7>DLZ#*va|U9PA5Wu8YZRj|9JTpkxB9MV`Zi4R>H-v;$*C^Vso zyzE^N5@yw;e*3~3@lYH-a=*8Q4f+$^1v6e}$uZFiDt3ioSt(>LmhoZS&U#6qZ;Gl# zf~ZRr=sI{@)k&6YRy~&#@-R~f@>`mYIwiK(pA@;W`lWWtDn%^>0*#VF=<$ne+{RQi?2yo=1Ov8J}E%KVv~1@Z~2EU)7d^}*`3sIhzV+4%#} z9~8KCD2&F<^9sx#7FJ`heNA{mOyV8CoGD%=Sl4wIO$0EqbouIN>1RjT`6_iq8zH}#u8qr^ zhZA;x|Coc8SL%SEF1T05vJx2H=xs`Ui%}lJPAJ*(!%vn#o={aiD;g71mq%1TwD;IX zY>1iGZ>-sO3e*E}^jo>4ho`F<#C1wi&;pTpai@@!=ehuMZU(0WK|2S4lcw@RLG?>(|r)f-56aC|? zN!#ohkoRLRy7c+-Pky}*!o0?FJkTKY>hkurOIoUypmy+=cc?i3Ocy*&bbw^?;m^HU zZUX`h^MNa{cx*VETQ6)ML~z|6KnZdY;q5ntv+3&iZQxn((}_tEA>nq>H_bDx)T*jS zclZ9CoWhQ~*TM}NZE8eTCQ=wbaP2uNQ{ zHiw%iK~EQ#+CH`eddTzXiwYUIz>^}6mqqEwTA0Yzb;wsp0hN$kbo@tor!*PR|Jc#J znI#8`B|kP&E|x<_`}3{jofxh8!Qls-Plj{ zDoWGk7-=T$TBG`~*sq%EZ!#;7Js)EEtY+b|>*F2UbV1;^F6J zT0m~svso{NEM|~W8^(!)#3oWENX5ICOn(GGEtGWPMXFzEFSLZWy&@*aYb@mHpCfcI zL+t1%9gxwG3dx*SJ`$9BJn^*#_<)!&qqN8?X!co)j7j4;5g_<1oK^J!Ek6UgDoe3(Su4>-v|&ZfFKe;P5c8likFYO zcsYisC0s7t6=#3pkqM~+F3F$~L8Q$O8+dWE8F&*IO^6$dUbVqc!!>~fR7?H|INHpc z1k(4(1QZTNZs^sKnR84$O-wV)e2GWNB0slb=aeY(b243J~V-Uo- zFWe8ku9d~)w#Sp0(7zW2zh*-?#7D&x;uX{`31_Q*pFA7~5t0^V-vw5f|AIb_Po@GyCKl&-IBroNL_UW{xQ!HDznaOS<82US%*Yp9zqbFEMt}8{ zD64r6anZdtzV=nz!_(L_N*f?qDMF~(fH5Kz0u%xS7`dQ2>Iul=g3#s)xIM;*=F>WI zS}eFjcql3Nyu#Bs7}5-m-0v;F7x|ISfc;>jmeHs!wG>{{@tup0Bk|sdtV~D~$fDY0 z4P-cAv&Yr8yAo9zD55yo(6DjxBPzaSS)k#YmmP8$qOK{Os_a;p6Ht_SIvtrQA;_Px^SGY4;`?A1HRh#aS%5H` zyZvyYq$wzq{eWl6YVXgU*j81aOv+;ZvpW~IKY$@B8TXH`Yr(S%gf$W%$^kl}*8M@H zVozhSGbglIs@!}CfqLETw78>C)iEn*NJx@+fMPya1E{d_+IK=oGnLVi2cCL);_xQF ze}khbAIRQ?)4G;2L}RLv8Z%W`Q!z+U1@R1raAJVYVwT^8uRW9=ax}Ki27#m0jXjB= z={&}Ft=rPdp#kSS4eI>`0ejupHU15%glB2JQ4Slm72(>)`_!UM`6Xc9nP67%ieri6 z#T^Z*ncY)zcuz~g=N7IM`eu)?s|Wi{%{^?QTXMjZ&xqjwRD2ImFRri$7}2g!Jd* z?tY%&u$G>G;x+`6$}=nzf%r{(y#X5kXHnb?L;p1U>&aS-6iNE?aba@u>Y05K-_e>s zri)iP?T*Lvp-KDi_AK3;1K#_<}_(x8HlHlxok?e^5Zf04ugY99O|oljTuY?(W?tD zz`~zQYqfn2eMeb|uxr)0ovBm&6N-7Iduf)Ql8iw}9UgMW7(3r1%EMV&Ut)4O5&i%zYd*?}^uO|r9Y`_%%;*s$+mZ?M^SHfar5kD){i=09 zq492hHs#f={Pa%;x&evRxA1 zTI%BZ+F{py)!oW_>#{Uftsf_74YFV#h$jhBdm^QueG92K!$GgiGd!ybCdnziW@V7i z9c2?Xdj)F({Zulo@&t=)a(D9hQ1LEsWSIJuJ$?P-OaUNl(v*$l#vkOwHi#|F2|_Va;iD;X`<^?Q zz*keo#FcI+HWEgm{8F?`xy=3(^C2phv-|&Al>5ubz9WM2?T*=vh!rQA7ylwbqag|;!{+Xb9hx@L zVzU(xAFb#uGP)hY{%E;qjnISwv;S@HCQW0IHS{ys&5z-;=o+U4w?_;~f0b3t*t+&w_^p9Beh0?%brMJL-9HQyv%{ z)ZQ)Qm-?Jgj46uNiMk;E_LeRZ6_v5-LTYL@2)Ga6iQWYU+JkOSK3)tTktbIT5T#1SgALH9fWuDCo~zyoV?S8E?j{Lzs5{!zzk8yMZ440Bue;})W^V*uX7(Y`b)*KX z=XQW^OlF4icq>>0aol>=e9O6FgBxR%UzA}a9Z0LyL@-Ds+Vs5uZN3f5i+yCvfXIHL zD$U}Dna5}X&oG8QC<4q(sTt#K?N96>w}jxE8;tBO5M~s#Ml&og-T|F4s7!)jr~DcW zJE96wDXf!9pWgmg-ELLrQECjC1y){3!;t5}1Oue1`5^1=4L@B!9&jwAz}tmhP0UCgi$kyx_Z( zz=Sio_XWLw#Q6w`RDoL$xn_-!j2Qmhy!^#Z!}_^!c>A$uXrAo`vf_aJ0xJy4V_S@01XE)WIAaZYLt_Uf=$yG(2^C?5 zTF(Ag{qL^C!9LsB71Ydt$08l!%0OQc9DrlJYefxtHyhJGkT3uxyG*)_O%rV!#rgYP4%ni~;H`ze=yt|EJruS|bCK9pqG zp~+Q%Y9A>^=6`l6syrNwA6*AyOd7> z-298b0vAfDGXqGARfQ|1ab4?W|DMEX6-haB<0^a}8Rg#Pn&7YVc-?Gdm{>gxh)V>% z-BLjE9%roNJjuQG=HQdwXsMJ_=#Uab^UEry#|Dm!2X0Os&}C5FFfFgjCZABhxik;6 z0xgJYb8z5{?p(9<_!_6>i#KpyI8(oJQR2_(pI`IU#(WF2fQ!NFp+t%~65pW`Kst7t z3EYKcS);gs52_CsL%%q^%aO{ylSSdCHe-<3G@jNS(ow14qMmUo?viGXuP&IYWmY5g zx>s}lH7bFK8SI|l_D9sc5|?ZWz=JFMBL9f`{IK6bqR4g=3z*7dju960bFZcp$GRCF zH1uZLQxLppdj2|q6cHcttiyVBd;W>t`Aji{TjHlU~N&4ZVx|6s=I2B z7W2B6bNwd}q!(^Wo>S55Y%pAw_1imIzWN_GS2KUH{okdoXtW-W7RyFNTUGTDgVJ2AlzEO_1rAKf26ZafY143T=L zUNaglP1(3SGy|9#{4BWU%nxi^q8-~3_rTI5E($sp(g^*bOT9YY~6Kr9Az&gdZ?8H0HpnIrHs~b<)Mj4MFJSTXCHxsc0$# zrUKvr4G^}M?~;Bgl@$P45cy9hh3gWG?kEptY*Xe)Ih9jn#`}Y&YE`74N^A_hThFoP zB+jIiA`{q*PFSYK&o>GJDBGytta6|j#i@|{VQK`~(6$NJx_F}TTx5YoiCaA0`RwK! z6W?i_Cy#+TT8ypFbTy$+QoNRi@;Yuq;1jO@_}d`4d4sZwjJ7+h>;?WwX+NVc!g^H^G&9eVyjq!4F7<{e|u*=h|aaL4Y;QNT*%eU8i&K)S>(%tmO^-b()zakxji^c3%RJXb8;2Fh3ex=*2-{UY9?nxT6_6}%|-S7aGQ3@DD=Gyj`T$W+7 zNhu{>H?e1blG}u6^bff0^2#wk`4V;`tm7X08ZT-Qmt=ZK!+6GCGKIO2 z5hSK9TTj-J&MFi!UD193YWP;iDQF@io#R`k%M?{1F&V4m_NKlgT)#Vn;a)N&NAASs7PW zSheC0_B~Wzr+GWb&!Pz_IjM`3IfBnaxYXk`b&dz-qnS|3XC9B|R(%`(RfCWL`z_fX zOaErWk5>%tpf{o_NmjUxNtnh)M&T0oqDKsy|Ir;L>EC3zy*n{K4=^MY|2->-&0hJs zN=nz#O1HZ1SfeysX<+YG+wQ9J;brQP5$P$ODRH_9MsXB2df)^pDNoi-CQ9{2?U)Fe zKv3mv<=>|2O}@PmEz)Zh(QX;W@$VX0mHYw{dxpp|O2 z8b{UBz1^J#UW`5;J6$VtUNl~bL>bNYa^ z>a0H37=LDQnys->(EGX-St5s7FbAv}It(^6B&^WVFtvIg-I)1Au1GNpZo}Wln&s^J zp6X)boyo@qSj|@Z=yY9FR6-qJDG>B~A_o%EJ`=sbEZ`LhIP=1#9nsquNql3l|AunY zqM44^@8j*Mc;99EWZsJLwYx$qYwi4(dNj8~ir?MT8bP$Gn4lx9WQaQ8^+k^LOoN?F zS4(wC_ldw7FV{Xz%B(N>D9>CQk_(o=_j{C>00U$EyrIbX%t6oD`)S|$jdxWT-5)*K zx=CV^V7f~K;kh_RNDjF@1W#vf|CMXC z#x(Tjeeeipid>b!#l@d#i)2U}5k4XoclDCv@&R1*coS=qGcd?aW94B$ zMYt86(C(O}OSS;C;{rm>5=HqWtmupS!4GgNQ!gS<$i>$4yY%4`2b`MwiC_BMx>Id< zcSfQ>I6raYvv(~iWA9Z#-7A@O>q=ML^g+M}!`w|LNRMA}OAVx^$6sX@4?dYor&R?I z+6!~l7RI=0z`e2n-+|2V`O{ZYSNdpu^qm?Z{wt@<5_qakv!RokstZnPHQT}5skDqh zgwxMUz%0_szdUrp&>|H%8Oe`{?;G`r0XEW8PztA|)q7zz{kGZc{HnU9^76kPkc4NH zcYOZz6wp_g>D5CYuPZ_CmURc0HXd3h|Ki>tbM2@sqRK8nOd982Nz%sYzM;q_l>|<{ zy0d#PbL9o^c7md?acG-pC*HY`z`ukOP% zU@mT)#&xe84nDaM!gvm1`Ev>D56b+F7PEaa5d@mv(0$^ex9ElUR&gSy#yD*vD744H zq}Tg)$vKLEY}U=2u_8|BkT8cfA6eSZ-U}o@QEVv{nHOJH=GJ`Y#U>Se+D<>iy2pmr z43#BBzMsg{J(p#!o#J}Yjwx|;PhMsXpc0F5rq*&Jqs^z|Lo!OScp<{G2M>>*BywK& z@~C^I#q^5@4L?%OXHg)pjjJvBw`4yBt4?=PSnwRvP&F~PxK3nJ=2Y%cp$?+_M*qIP zAF2tW)J$%{GuUtn$)_SBSB>X1PknO_H$1m>L!sUBAVSmeK=In{ zwT6p)<89jE{I=s*ZLJcJG2EW4LEMdA`l+geKqWqMIanm?bxW(0$o8-+MaPz)iJlTK zGba}KxBkmAZeZwg;EZl)!M3G!yp;09Nz1Fuznw2enpE=T)j~SBQ((y4ldNsv&r7p- z&)AQFA@6}n6n9hg!DK@YUkBn@=xsdH={1Hg5ExOzS98t4zD_l(XI^^E+PEzf7}rwV zB^9Qc-`6&Ba6gUxg^+l*a>1FgXfZSsJVO&^W~>A5zuyzvueSZ?%>hBqx?9={yS$ih zhRe=StdL^63d_L|7y}YN;}VlQ|M2j`EB4x(#-uCU3FLujnI`A=8X4g3K64YKkc)q$ z$cJKgzmG7kC3#}>pu(&jWHqKZ)4JMB{qr)?hA0VLVd%{O^<-9b`Mc z!c5n+j?ffjBOc?J z!8_q=o8QFu=jGbc+S*#f3mwOsnYt!&PXfPIHQAA!NqC8@vY??IDtP+R#-Q!OP;x!#&w+=7yu$et76-SJuC;7Ur4L^EGtFUsnHlK2;s$ z>>pMVO*feoWW|w!1gH~YMYQ1a$h=90A-25;Y&67jY% z56M=d*qUk+=n@>Khs=<>Vc_V-ay z{9vW_QOX736V@83%A&c#uuC$TXR*)Y0?u%OLm?A&2+V@xy-`!ewG7{xUK|Lb*{#3x z#C~idu`0-@ScH2<7sq=%b{RjFGBSo>P=;L;cDv!nMZej=^sEtu8ynk)L(Vlr?r#m3 z)moSz#Vpo7A({Zc>lDI8SV@kb6uAEh)^@__~ zUfNg=6hi#sQu!351~(nH?~Lb9c+q8YBaCS6r2PYojnQfGvF@7p+Ebs47D@;`Ju;pA zvJ=mSF|0STCKFRsnWJ7^!IU<0U=2lEKGkt*hz`cl>cI`PcS~6|)9;fzvd<`#9!&&} zL2bEl?GT}6F|e6#GOk@w`bO|G0vj5uEu{1^`_|{-Tzts4w3DP2&*+qdC)3|T53{HU z?mDsCSP)k}Ep23=k_r{QTvYT{*ADAxl?o+0RxS?x(d}GQvVEQr)9@|#H>IDEB=O`k zb4kB?2b(dDNtEq*HYioEX?imfPdW>8zy;Z?1uhSB+tm0>@W`+b#)~9d&8;!Io0(^E zW#=H&7@%S?)iKA(%HEauI@!LxAOlmdpiv#po6WwR*CN-0ulXv&0)dgC0m6FPBaus7 z0w7RMs-z>Dh!=^7jBA~fS*LO&#V%}7e`D?&*ku0Y4E8$uqjVid>L=n!CzKL*oO7XOw4$XN@TZ>8 zLy0$R^*=*QGBQUj78A$v&#a58+h<211$?eB1NntN{RR53;Kn5hDZb3+o_-`TwGM_Q zf+Qh$@Nayq`X>rjAhL8vziH^t6FLQboVW^8&^z4QXy)-Ka)Y3zn2!Mnr?%&@4-hDx zWY2WH$QCz9i@#2TipXFLdehA455`_9MF=;g7U?TE)59<*QtbJ$&=;=Y5xu@rM{(2QQ~TXK+M1>v z%`X&X(7;b|;V<7J{TAMtpgNQ%bB)d>kq5Ug|0@SWjN74;w(`}B`<2~#c|-11d6?|t zsPCxO#rJH(2dzJ?3_2(1djPr~I0eV?Kjp3)aTv#+WPG>$W6tqWBj61jy=qC+L-!p`y4*xMro71Iy9DQ!mgh#NgMi}a?X`85#O(U75;^%z zT=92E{9Ii7j$Dj(!ihDA@EW5?x|$}LnK5Y2iF4#c2}es&;zb~45ATw|7(}sr#R~n5 zm}ee=&p{Qd2_)JA6P7l3i-_ntkLUM;rtm z@ZqixHNU4bi9Y?Uh}Yrqz?4m#EgX4)L7AWp5Ck1%cSvzu+fJP&t=~Z5nnSNG)kKnm z8jbzZ&e0`yfkSDro5dBf{Z<2khvy(H+20<0SJ!OqA2??d&5D+=*p^?PZd5umng&j- z1M{b}7%4-J-rWORR&$ME^4NQR|%mmHOu?v-_kGPvQfVo4(=T zpS)^c#x{JISh+;?J5u+fHyAn6N9fHYgk?KK+8gppvIakY3^dS=av2h%*Dom9iGSj4>-+boo8Tp zY~X7jl)3neH7YfUE>ga={1-1S+%rXI1@sj1F(iY?l)gafn(IJhuQkJhUMf4wfgO9t zi7-ADUs+XU#e#rQ2M4o3$%;?lp3 zSqFBoseTb@5s#^ys81CItKV*)V+BT#a<};}BZ{@E?Xj)V_)pjk%x2v~F^n2;b8^<1 z){r4m+HThLxL?LxhxA9fbDSCSo>l_b;QGIj&%3Ui`gZ0JG8rV`ES_ArQNIk%Mevq#T%?vgNo%U9u@r|DRm5^Wzu&X2;rD%}6Z!xp=H zNozM$BDvRlhZn<39*kQYs*2_<++?jAf|>!v(R{{1PZ3Bd#f5G(uL@emFHpI)kCewTFncc8Uj!GD>!)s|0}6g!om*1b#~V}VN? zn%)~=b9RRIuZhxs4nhrSZ>mV1UL}Q=nDJaB%a??npe#*mF2#Grq1hisB|!l$bOCby z&dpjqQWl8TaZC@*xgkV-cBSpO+iWrVGjIiRC{)QF$HYfe$mav&iV1#f9}v{P&`Ev} ze_XJG;!0mxdPmlpxi52M)8`QVbWQG#jQ%|83~BuJyRObjLAp*0(Q06=YWeG-o!w5s zY#?|K;y0ipqtOw9Vx9YVlKxlYm$lyUN-GkNt-jZPC|tEM^`LoAv~nJ)P(7}po>paZ zv%CkXR)OA4G46-*^?*$qHgvnYg!QAyioG^=?edVAP)}ZvpeGaa-0+C{*E7Z5$&yn#3-%a z9R%22lk05TxesLrBzxfynInz%zdkk6q_$6QZgCF^+m2ERrY`)1nm-(HKeLD%QEWc* z^7Lb#B?o=L_3)xYvfIItpKj;ry~DU03odG;`Dvm)h-lYN?((9P1riEB;Po};zFjTa z0H_Gdd;q;>bxm<=-I;eNlu@u_c?U|4j9# zFkM)?(lc?$e=%y9R=xNO!)#mTax)FNncVjG#F!0jqGZ-!U(%W21g?IXiI|&(SUi(* zON8{gJ(u^#;hCdqBkFxhzRZOJ%weln581O1w9^d7N1sa#*=7H^7E9sl{C?G$ne>9W zc8B1v^k5-pgt*7=f11C!yXqYpDuyL~nUXd;94r}0f#AhpR~QQf>#AA8u_)h^#Z6<& zeXsd)uv^Yq9RCGDMi`pfKRuc?mdq5Gq@$W%^y21EyAklbRcHd3)IID;uKFtjS9eNJ zX#XtO{|m>kvZ1eSJbSnJz*79)R{mSh|DhxK%n`1KMv#{sC?L5tY3!sd69t-goQS!H zkU-)4-W5zMVBk&um7W0o?F6wcJE#+rUw+&NRv1hzahrVFgNmN~4)SkkatB+|Iqde- z;|9vZNFaPZ@g?DU6jUW_k>W*MX<(NZ$1BPe0V_43^~{md|7c ziITNPG(~J{FuvLnTx(+-jzQf+|MT!bsL(ZKyR9|CJ>In8)d7F^I1s=c z+f{4^il#J6VDxPJR5O?Zqt*ov$g1XGEFM?A*==&{C%aq9SZw;_I^Vd>eeFM9*B`6| z3P#pfPytul5MYZSKaDAbY0u&O8VG_kA3u}x6-M~%5XD0rnh1$tQu^+Y-C__0v)ibmU#5bZq8jj1DZGA^rbSpKU2mi)TPX$Y2jj=T zyH1P)Cw~7`L$7S}dz$>AF$N$v=b*cga~p;Y5gnk89;7>0R)*heCRNwt6D*HJOO0DG}8L z!s7Hck59!r8DBQcAiEfnZm+VBv8?FPE39^H&5uUrccp_rK4otQc`zNg_ykIGOd{HM z9fdOb6|YM#Zf?yBq9^J5DWF))A-qJ;f)BR;f{Zox{i5pYAQM+fLIpbuh&mh3ALDE9 z`TEx>(NdmU6H<4$rKxLp{Bt;o`SHWIXJbE5QdwJApK0gx@j?2xar1Xto{?AQzOOD! zX{)Qn^+Wyj%3>hqhfpe4*T}QASB)t?CMtUkD4%FrAw9|6pWfM+?ceTvIT{0Y`x$Tv#2i{%HjY>svFd6-S?R`~LRB!n19%ATj1c4Fh7`g?d zq?AT-Xpoj}22>c3P(nZ&esoEP#3&6zH;AA}NJ>lpw`ZN3b8+s@f z2`5RxO5c@%u}Yd;MxmfYdW#^)pCJh#;<{%m%gbXcS;3LFV|<{5vGmiv$n%Vk?;!Nf zc&lMrgup)h;(Ga!17;4;ihILvr2vet5~Bk((**dtF84O(iOjS^m>rkS2!PJ}(Pdzb zWRDWg^^6^m^l6P3f3h(9O!I5YIe!U9%pBnKz2|)!+EmYdnKbYuG~-j{5-<%4 zLArvqnq}S4K!5*p6M(PVuFlNRa7657UDBW(4XR))o%rIyL!Oh-&Y1qHoFi4k0vTW~ zej=r*QI!z(cyIvYinW0Xr{PguB#OOI0}^7t$nFb^iCA)dm8Pa3&e_=5aPGD%0f`Lo zGcqD3Skfi?sBv9BhGg~iRREZ{Zga1=dq^gNmGe714Qwoa%-vXwlY~zF!lr1<09l+5n zTg8*(c`b*Gxe`wc|F@YVmeAFDz-DGE8dcn9a+b3R=5Q6C-=W-b^K5-TiO^}Efig`@ z#Dj1NZ3IX1poGvwjctP{Y$lCE9AMkPi<`-h@>${7pi}y;<_qU8?fqu^GiZKk<}?t9 zudRh3Pr>oW_YV4JyWi#RJ3zH^%|BTwnj;vPl0cQPEMivcflHAQOG!esg^=*i;l-|U zAMo_xG|cufbI2JQQUXj{1;#^JO9~E!?jL<0h0`I|){(-5exY|Iv4K#eSS^DnTxvfO zgqw3W1bvtI62gESN#zP2u8W&Ub4~BmdOj;?ZM0YSVB-E^n}dM?8uyr=j>6tAv%lLd$~ zU9WD4&F=}GmwR1TXeeVLmGX4B=;wSTK2@&SIY8rtNX)g$#MUH7Gh?=*UuHfrD(^_5 zt=?(-+#k{b@C0h0)mEux3RU?~p zmGh-pw^4({B@UX)F)N)##60>VIj~#BASXTTtFJRu$vCU5Dt4kyenW|7csV6ur+{bW z@MuY4g3kv$Z-0&1YzN+0gQee$2tM+#En!3qhAs1;!)OU@uNl4 z-#%@$rYng7KDDyrEfp=;&S$#|iZ0?;Dh{`-#Qa8`&-hQz&hA@0t;=n8HcZ^PVtG)I z2~ZQ?-=?Q7b7#r?PmrWkW&YbY*~s(*iJ=JwE>pGQe+;hbQG;f7Q`@z8A!o)vK|;>VfzZ5{CjCJAmQae|PsAQ5tyDSr=4&Xt z*IrkzY4W8$6;7?hUnK4P_V9x{hBMf{wR`&*~@hei+)S- zV3rpkFK-Rw=Q)&WTH&sl+<_xbJ6#k_#`ind+()2_z|do^S0@jN$J2skq{B#AY1G2o z>7LWgq|p)L7BanrerT!w{C)MJ4Na_FU`ZTqrvDYba4-Kj%kJoL!jIv&Iaf_8VIaag zUcUzu&byQsslHH;dA~$tsN4)mr|55OP>pj+Ln&UsAZRu?C=>}Oe1|^^PQqZ2WVp8O zr3Ita>P33ZJ*R#dND|OGBL+2F#nQaT~a$Ex1V((3s+xnfnXlsIK+hAXyUts3z zj;pXWivIK@VW@QVw$h}YufHjLd;ZLWd^B0m+r6|Qp;GnZ)O-N9P&j8_ll*c2*+>SS44XKd^gt)NZQbCB=W z-wsi79UjL(6wsh9Glnr8Cz>De>Nls0;_QF32VOwf?U8Hnx+oV@e`vN_^; zDR5NYcc@<`WjvTji;s`rhXYKgy0!L}{- z^jg_b<*?&MOm!3eDN0Jo%6l3cKSj9umZc%AAZw59um%Y(kyfZ*I#|LhSP`F@03gk- zDkec?2qx4kOzD?AMb5qjFy9W~7pl@3eB0WRLdN%w3uF4!d?e*aSGNMp7*pqfU!LIc zOiEuNs{g3JTaB~Oyx}E4W#KrP?$m=S7_fx&ZE40TAf9S+j5VO{s}a8qU|CIXRFn1Z zlU(bOB}0n)Ry&M*w9jwJZjTo6$ae&2F~qd!y_(*{9n~O{mWZ0%a?8_m0l4^vKKHF*#y>B3Z|WYgL3>)SVNRKW^M|HNCS(-$ zZE0*O&$fsmOt{3I>9|F2U#b7))8Ayb%q(}>To5$&ixCGp%87F=hD3W48D1689gaXe zo4MDdoOfm|R4%!aPu95y?dt4KzZO5s@Y2hhrh@0++xbJ*Iq)xpt+ZXG_EuI@rTc*Z zDQ9ug->cUT`!4f8l;jK|{wDa)?dv4C6uI*1eMM+B7ZFQLU_doSyjWhPZz7+0$?u~v1dSmF7@pXl;8fegIn2w>?(Wm=_C5D!-FS81kqg>GcEp2SS&@aw1jTNkOyTWn0VTZNQ5UnHiG)PNd!5yy7 zJTi`CrE|xO4Jdx(gxq=qehl!G(ZC8LnjL~q>X(z{4v(s+5uzYgr}mBVPXeGMt$VQn zCc2fX=a^yV0Y!^&j2Ra6d@1SvRL*{^`=1xC${A8nF=o3T)G982e|k!mqWOmmP-y5y z>So!dYYjDB9k|@4S}(Mp8X7(%+Xzf+VEd%`#aXmdX z%$4##O!-~&;(2J{7%5ci1yEP4?ARK!*%Onn_WBXo2noppDrIkUh<(PzllKm7ViFRZ z^paCt`6+=n@*rA5Z=Pt-?npU%Y)p+oZ0&V~HtrkPl07Vv4NOnSF(Ge%A1{9UmgS^T z)Je~^P6f9|_9-8_t7=zBC{a6uS%YxkBOyVB>$e(R35y8{bzvGpe1&)cJxJ^84odqm zbRm@fhf1=XVR~nS+vkW^t0pNE_5=VJDQZ*f6WH3RM>*b|*1UXxg=|xXD2yHZ0r(i1 zC#UQAiw?g}2A6$(i4;yo>%0XXgkgP~nemOX)}{|KG@Oa7lR#oBHW|u0Kmi;d_f)+q z){h33Q%^G4QpCV;t4?Lug=D#LvsjPY)`L>N%)r)PO)Vj#OJ<-u@Mj#ie(dB#3{s%G z$4mD)%dX_;QllP`E~!|huu0$CYv_L2Fx4cpJGD&GV z4MZ-%aQR*^5PX_kEZyuPwgbu4{1=wwH!h4v@VRFvrspNIU#%{9m0tL-dV#tq z^Mv`8|D1Adc}1>>5k%w;4#wagMT;5qi<#!8=l^Qjj1r#~|JU4h*$!jiSR%=+^ZuBm zUNS$N$DMi!Ysew8GPp2dU|$H2F5%gRa252Xas@afn4jtJ82_)MMCnxTPjV*E#THPM?pxHYQ?r&j4dpMAnX$(ykS1(dq9Ng@mAH9sy$DkZ13 z?J(%&KKy0U1)``C1Nu!64VLp5}axR%eyS&-$ z>UF?E7gOWzIY_tr5-!BchSz7MDb;((KR-9 z*fn5rQ*-1ik8mbzv2ryQ`@Btw$4KfbBQm6{AM55Mf&Oo?cJB&x(FMGo$oAnyuB(DI zZ`cUHJ|zuM87c$g%6EjJosPOKf4&m+OO{P%g$3ugG3~u6eRcjg#-{;g^<2X~=1bw{ zvcaaBgvl*`*Q+~M7Lg5>p6gt$r$)V7Fz#mKcyFf&m&w6jo+PvyMS644i)(OvhTvt3 zd~C>$P#QZnJXK?Pe0iZ&g^qA1yRUCuyD2oZHY-ABE?IA4mKt`H>*jLr1KYga5~8B2 zEKmiK$+R?DtXJmJ9h6`=tz@|K#QqsNxwpMTOcy~3uc)nkInVGzyXofkwcZ$or-{_g zFqcTNjI{KqXfZ9_amaC%m%Rpm*>zb-&ZQT@`3WOc$*YLtrbLG0y&qu-V6^hEA&eA_ z_c-N;HvZDM z1n>5kdtG-IcbqVB)(ukqvu5}o58y0S_zC4P%V!xGgpcdh#$Ylhi(A32qgI9&6P3li zD&hRr-Wy5biM*O7)c!{>(|eb-3<}@YgDHwSbYHFIRCkl&SpR*lpsAX}IByZA?5-r` zf+a&71_EsQX*3} z&HU=6v)fY6?);hDco4y5>~hqgfW3)?@No=ufX5(OJVUE9COzB_CI2V_)k#t{C)E@h zu5W8ALL69!3uKMm5i6v8Q8zM`hfjxtb~7{PVukduzXbcQy0+mAlry!B_w`~%xAe1M z{vH3nD|z<6amh{$){n!vHyjx&1@(6v(XAjdv4746}zP! z`WE9Cp&lG*Z!53dcv9QmWXRt?WON_=<;5Fyc+e#ZWSin`B-ZM&c)NlLWM>-Rj^&(h z;Sfw$yKI^){UV#fBVL~D{P$@DIHvifcCQ0ar7sn%+1cKe70A#E6HXhhl1UL6z@$+e zi!lbsXi!&juKtTPMr+-u1hinECnQR-|3s;@esiWL#db{h%rw~MY^|K;u@U#P#4m*$ zFF)n+6~kURmGyv-I#nh=>Lq8PNtxHx+Mb{uJ0QJ)@fN0(SCGg|tA$YVV+{zABxs!}wSm4VSlqm;H`k-~15A zw@(C%NEKSB4GM2>ttd0HRaMf+d(2B=5gB*TqMSn307W5>{fFBPd=+UKi8dS3Mbm?^ zflTR~@CgGzf1C~#`?F)sZ|r#St&?aK6#iwkwfr%{Z>8v4IJt_$D0!8Jf#Gs{uB~Sh zHVnoQCv>}q42@lJ$_=@GL&hlj?*BYrTMhu(|@JJ*(!^xAK+#5 zh}<&3-mv$!+>uhc>0fSaOtTJHrYI3UHsCs~NpR~TURCFCw%)kt#e_kTQBhIb?|pr9 zZK}F7l=~T2uQr#>-4!KykVYa_c7UgRXdnSnQ}QPt3~H3Dm;3!dUgf)@EWR${J_J|a z`K8dNmc)3_`K}a+2E(DSV%p!|ckPB1K}<_wqzXJ7XD329T5l!IHrLB3z*hUSXDnnQ zbzFrbfWfw_IA^Ahl8cLr3=ck*QGGrH9mWUH2_384+~XXot(;SyJ8*Q(W;(rJytRv) z|N05w`8Qz#F)_C;?w+Ns?D&d7hcIZxW}fcb1FZC`?-J>r$$eR0AY=SdQXaz@7tD2$ zi39g(a1JcC0)x-IJ)enMsl0Zy;_=m`klzcydNhDxu^W!GUD|;J;4}yw|5PUDGnt|}LQ1-{l-)}jeu#dhLUps_})FiH6|4Em^wPvNJ{u6;qM7tZH9cG&Nh9Se_ z^8?GzeLet(m2Pj9R>3;_uT+=f?i;;&*}3JWm7iiideQC~ACJH!v%AN4xhTO+Fq(VJ ze25x)pMPWX1*;}(_QBhji#BYRMcoz)5xm!WJPqKW2uHM2TNc(THS{|XjGB@_vT2#b z%I!{Mn1pzjH7^+@ogoEyQj4&tDBD7^$gE(Q&$}p}*O$~E&vmDUqSj zpPdB*4I<;DR&vPYSf|9;Q6UyNdS zk8Lb6(Xno!3Hvtq7XwnGEPH}aQ+keE?H%{~=dRaSgppGuL2b{EfIMhO?J%^(DBWYw zuKq3Qyr*qE{k>0|KiTN1fNA9g4YSOj2$B<>g0X<=r66>8!nZC$l;6;9iaaI2N7{w^ z2b)64iJEd|#yj>R!YF_B#W3aI{hXTCh202d6#}ST1DKJGVWHg_Nn6@zR;HObLDDXW5D4jC}L%L zng4C=;x>`&h|@#ZIv|#-4zQBWuAX`G^#wGWs8z!(N9>|~iTCX5k7oGsx%o=h4y)sg zy8X>p4LDTl#oPSVn0<#lHFk=RhqopFG7J64!@fl{5QCPlq;H%JofGpse4VG*N-4<`3ejipYWb%t|pov_K zdc}fk?lAy=K*Wlp==njhlOKe2G-;R>jx{xQJVr||R7L;FpLIPO#=Gse6E;yn*QU0I zzoA@w1BSwDf#1b(WnoG9gB>KCpOYA#v;LIhN}QgZLnOYkqkd7PUFtY91Nt@mwO7pj z3`9!KM!hKaqEj^?jC=n@h$U9dE5}TZO9Sps6w*=U8=k$Tn#%<1m0xui*!eUf4wi>T421oW-c?;Wmq^?Ctc>WKSo8ZF z9OS@Ud^oZ-aPH{Ce1l&sxu&shQKPT3Wwh(Y`TuZ89-5L=cYX zuqI`0%uSh&2W{lLr$H1l4tv_L@(5Wb#p1g zuA;ocQNW_D;aDeJ(Z%BX%Uo+5#0`DI0!sm#5 z2KDISnR(NVy~0ON?BwZyGU71bn6>j8Xs~?mw zWJoGkkZt-fL~)DhG^?KW);o@o0^c5(Zir!FJxV{kDM@UZ&KTKpE%ANy)kA_4cpgEb zaCYcww#184-=ihg+n|2%nV`wn&Y@0p6l^+N1z)WlEJF@??_{;>7zj zx`jne6+78PbON#kKRF8q$he?E-Iu!{iH!b-Zsb;^-hr1zv(vL)tRm$zY}?GDQyL0f zF=YhSn1P`&!tcLfO8t|EkEhe3XQmXbl<};b_WG@L#Uw=IaCGbkJT2o-;y5pve+e@K zSirxuD*C6WQ|yyXu=bJ&cowtSSZPli{n%;#xrd7tk_r-59=E%xU4J%Bteo!PO}S41 z&;}nrpWS4mIQhYo7i_U|=gi$X%tIZPNcdFWaW!J`@-l;e_=7L;pM%VI1@{;C>cCzT z+j4$lJ8`=F_wXOOqM#_otYS_aQ{W!2n2WAJvoz%^w)ZV1E8JDn^)C`M*ahwl*bjEv zTZ~>FeX0}w4}y=fIlOLmJlQIYV<7haFP=ztEcktdBaivqIi+VKNY+gMTF$ZOLY?Aw zR!%~KH{^K~263ej`Dk5BaYw!&30p(iTg}JULUdde&e8s9l48$-4J+7ES;rh?2U#Nc zTkynYKz4pbx0XE$Q_$3uhM$heuj?E+zHuSde|Ki})Z&DJA7g!wn%Dl`ebY;vkl7K3 z1Hm)A=7xsdhgKb^zvE}3kh#|Cdl{WWcEb1Q0}Av`?JEutVxMAqnHtUlBxl+cGr416Xo6=3-kId6<43#{&YGQ`wy(b*uTlA=dqhak&sj zH;d&|Z0+i;xxfiWfKnZ5^@bIKCU-i$K|lh`*@d3;c83_dh2G$iPU#Bq;gwBFOVR-$ zlczVCLIVoCnG+z$;dFxM^Wo~$Gqj^wsC>gsfy!AlEU&Y~Dm*rbwfyMB9@rQoRzVba zi2yOkW9@MhIXSu7fG9^H7Poi&w1L35?{XK#YT?m5X@RC$=-tE@mb^vOZZ>NHD;AX) zeSP$Qw-!a0#*An>*_E!a@nxS3-m_jLc?7>83}0=Xrr{O`R$WM0fySDcGX?y8 z1&WX%AiGIb7hS>B7#6TaO=lghtNrMX^-|d2%_F~v3pvoc=c&JfM;%`#7Uc2;VwrP40pQBu*M6gQ!uCX> zUa9sm_HUz=E}5g&qnib;mCa~e;7;8A`tMt>)^Eku+{7>{M&Naj66hoA(op;F_XvRb zy!4fAhx&)_`|I7@YLHvj6f(dnZjouz~fcSci4(H*bK%xEqz?Qg?@Hr-A*suN*meBR`^v}nn&(OEH1 zjQ%}Ul8EK`kw88NxohR@VvH1&C0jR>c}_KvDAU$f45|(J1>hN-+|mqEt;Q;1A=yBO z?+P@d*uSHhajTL$4+Mo&Vq8}Bxw>`>>7MKmf%d2m6cvjOZGn6Lve?jgsqo%Dv>B6~ z010^b++cih)UMu=yGvsH(#Rl4!fKYT04%uhtNg61FY6v)hQJJ zwshWMJ9I)qGjLN!jX2sCPj8<6@@f_6CLVbRgqd=9%>wTFF+C}>=A-;2-4Pmp40w_U zU0GKPOEC%t5OkNnpKpg-3qzq22#C7PMvfVlhV60`jD23%{p{VEH8D00jux*#uJhtf z)&<+n_p`GOZ&cR*6EviLrkPPl3P1_}235wrbkN5__AQiE<(!8C$3t3k@|NR4USQk( z2}No;_cLVGWCfFW5oF-38~*_f9%I4zL-#UA)sPHnz9kt&@v=Z8dUVmI76L8!FdPup48I3hMdPvZdu zaBL(YDv=D;2P-W;zb8jw2PFfiMFQ_Tzx`|3W5d~g=H7H@i7!E|xzYo4t16tGAK~Y9 z5dZp2gyytZ$pvG$S(=%A5rq{1z*QW?MiRmw0)N-(#qRTPgQwsgdQIq#VN7wSIiNHA z=Ia}xw{0u*sIWWAnIVlTVleQ~SWg?dRe{i@#ZZXQZ{T7!KX-=L|2E+V+vqhBF7yEp zqrAQoZX|y01WQ{IDM3+o3f^&genVcNchNT)$o2az2OI@vAb;WBthswFTi;K9pc8rI zL0CtGcC;7^-hmUL)3QE%`0!Y9hYXl!QflqeKy#X}G{4O{ZI*xU_^XuCpSF@iT2OB( z$!WjkaINcpb8l%zFfl{ydqG25zUQBmu#JHLvErbdFCi=`(qpGLE3B^AjItM8)3ALH<&*70+tCs=KYZuB5Qq#&)QnQMZTl0C17Z$1S-kFwb}@Gc7aim z9dMp@llah70h0zUHV~BcViV{-3%#=^w;;80jKjje`&e3?v+7XarP8vXMB}f}czL_5 zv;8>5akUeT&eqwkG#a~dNHGb=6Nv^L09#6s46@3e>#75D{eVMy>hvZTRy;NWj+S%u zhDagzgcy$8VQr0hr>{XP_CTvO)6=>@vr^Ecj0;3jJqN8xr_J z&2p|y^k$&R2FlHRn(+Y$|9F-O_0)H(Ocj`M_&yI68-QHf2fs#jVHxY~3 zP_p{u^sato$A4%i&G+%69~!?PS1cC6S7SytQ%z`BAB@GioNL2`9|BG%Dz7&K$4+-X zk`#||W4l>(md~7LkJm+=6_%8gEXPp^JV`m)0YPp;z|*+ZG3 z2MDvDHGKBaRa1vf5v$JSlOvegW2!xN2dXHW_M441cd*jCNKkYiYus+}Fa2|zf`fn@ z2|6~}Q@O18q`Ez0d`W@Hl^O%}y#i3>%5t#t-PS|6{HXGABx?X)+OYcEbpGCtA`omK zEhLZQy)A)-R(0rOv?NOI%T8@}pq2zy1 zaD}yT1@4vY*ET0tSJ@@KZEveva!i+)8FZBnw8g<1@3|SpwFiG0R87lMd5R%v|P`% zEwHBfw^0+db)Pr4D;x}9R39*fEQLYQaqqeuhyT|pQn1W&%q(nno%-_&)X-&y0b4k0hBHFu&lnUO+oz%CXY zwmigB^|z6np#3VDk>N~7hE^YnGcv(VKH`svkpxx%A?Dbm>;gnz#yiVD-m#$#Sm{O| z;{;%K%amcZM<#G44(*Qy2s8aD-GZ7W!UD(VJ0)V{Nh1zv`L8pa*(yKB;78`Z+H!mN z$SxRlp$@w|MpakKH}jdzS|d9zAjq+_)rvE$KY;9rGJi?7-{O?6g_cZHs0{KM+1i7) z@iguZEVLKahxfC?!4Wh`Ka7R%wwZfE?jh?n{x%LnuJiLwS>7(c4<7LPw5Nt2U_5}R zXTTiCI6cDsa_tq^$Vp5}05EES*OwscNqc6-3hwG-h5RyN(7alzYLpMOY+%6VWF1O32$ zbkWrE=)D$4@>)6H4i6}jL@vc;k2l&`!p#7TIj`!joFl6L>&Vb38r4JdIpho~Jb)C1 zXdc2q4ZEDD?39TDrVUQKIRA3m`K5(GW@M&h9BfaS*(!_>y~g;rf_vc$o@B`i;OX-I z9OUQt0H2^NGRWjs6e_7b^dgrNM}`VkXf`Sq7^~s9YF7_l~ zi`DrVP{HsVT2MrLtjA6{mPyGZ*mliSLSPyWLCdpM9x*8kLIR+sV{hz7d@bfmr6<6n zmwTCONSiEx^&ujVn@X-GqUrK7Xe=e}3}Pqej{C{I>#k8`LX!u7c0IzkC~TXXDeHfi z*;#NrsgQ$gAu62dX7qn~O=j0XpC+me1zWY@Yy}52hYt`Ot5VPp$iNs5XOW4glp0{8 z!S?H~!Q@0x{XzXDxhSPBrLsCa*Qlpv9%980;ZcG2{aXcJ%lGtPrjhuX8B_If;50P6-gWrV$IMJH z9rs-r4~l|mcq6pi4(V^be=J+MN9(zKX{W|iRi>Dc#>+nTpG+DgirKQz^1+Sr3!Efc z#RW_Nlin%sTX{H8XXQ$f}tzG8?xb8)QY00lAnOr%qbmvc5Ys16Y{TaUU~VeaA1T zvikm5ITPKEi#9cKZnQUX;W>qUZQ7&aTeZ+SxexE6QL3rwpg;>dgyoLxQZC!QCe#4JCoGKvKnJU(j63cy}rGqJnaH1BOd*vj0IzyldY&lqY+*$6i6+FfAqx6NvPo4XRGLiVU(H?iZye+Q8m)u5Mex76&oG5Fe^PJiMor zBF&n4HIn85Ki=3LEiaSkIKz-@WJQ88PE-xX}v zGKMtVtTy;N4?Jib8@*mKIl|ICu)Og^4h%K+tT*I3A1?c8v zoWqYvxf**3BaMS1jMd^9Pc>eP>EZrBY30aXY%P|T$9$XP`xO9Bv?(ZzF>xud!0{29 zu$&B{;++}A?5*`DuF%iTIbZFqEMvYErYuf{=g;;kn`H#PnhnwfA%n29NLKv+|NDRc z1<1W`<=^^n{ocTI6afJ6&`0IDkDaxTy|k^DJ@^k0h6{`G!zK8I#0`Yt(xMX5q9VL- oxHKHzkfnI?e>~vkVfVrz@c;fm@0W`w-~)iV%2Va)$5xU51J&twF#rGn literal 0 HcmV?d00001 diff --git a/cassandra/versions/1.0.0/Chart.yaml b/cassandra/versions/1.0.0/Chart.yaml new file mode 100644 index 00000000..e69de29b diff --git a/cassandra/versions/1.0.0/README.md b/cassandra/versions/1.0.0/README.md new file mode 100644 index 00000000..e69de29b diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml new file mode 100644 index 00000000..e69de29b From 092f9519bb4ec85b99edb24bb409c4014204a8bb Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 21 Apr 2026 11:22:02 -0700 Subject: [PATCH 02/58] replicas successfully connecting --- cassandra/versions/1.0.0/Chart.yaml | 12 +++ .../versions/1.0.0/templates/_helpers.tpl | 9 ++ .../versions/1.0.0/templates/identity.yaml | 4 + .../versions/1.0.0/templates/policy.yaml | 12 +++ .../1.0.0/templates/secret-config.yaml | 33 +++++++ .../versions/1.0.0/templates/secret-init.yaml | 63 ++++++++++++++ .../versions/1.0.0/templates/volumeset.yaml | 12 +++ .../1.0.0/templates/workload-cassandra.yaml | 85 +++++++++++++++++++ cassandra/versions/1.0.0/values.yaml | 21 +++++ 9 files changed, 251 insertions(+) create mode 100644 cassandra/versions/1.0.0/templates/_helpers.tpl create mode 100644 cassandra/versions/1.0.0/templates/identity.yaml create mode 100644 cassandra/versions/1.0.0/templates/policy.yaml create mode 100644 cassandra/versions/1.0.0/templates/secret-config.yaml create mode 100644 cassandra/versions/1.0.0/templates/secret-init.yaml create mode 100644 cassandra/versions/1.0.0/templates/volumeset.yaml create mode 100644 cassandra/versions/1.0.0/templates/workload-cassandra.yaml diff --git a/cassandra/versions/1.0.0/Chart.yaml b/cassandra/versions/1.0.0/Chart.yaml index e69de29b..645755d5 100644 --- a/cassandra/versions/1.0.0/Chart.yaml +++ b/cassandra/versions/1.0.0/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: cassandra +description: Cassandra cluster for Control Plane +type: application +version: 1.0.0 +appVersion: "5.0" + +annotations: + created: "2026-04-21" + lastModified: "2026-04-21" + category: "database" + createsGvc: false diff --git a/cassandra/versions/1.0.0/templates/_helpers.tpl b/cassandra/versions/1.0.0/templates/_helpers.tpl new file mode 100644 index 00000000..907a9e46 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/_helpers.tpl @@ -0,0 +1,9 @@ +{{- define "cassandra.name" -}} +{{- default "cassandra" .Values.global.cpln.workloadName | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "cassandra.tags" -}} +{{- if .Values.global.cpln.tags }} +{{- toYaml .Values.global.cpln.tags }} +{{- end }} +{{- end }} diff --git a/cassandra/versions/1.0.0/templates/identity.yaml b/cassandra/versions/1.0.0/templates/identity.yaml new file mode 100644 index 00000000..12e99946 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/identity.yaml @@ -0,0 +1,4 @@ +kind: identity +name: {{ include "cassandra.name" . }} +description: {{ include "cassandra.name" . }} identity +gvc: {{ .Values.global.cpln.gvc }} diff --git a/cassandra/versions/1.0.0/templates/policy.yaml b/cassandra/versions/1.0.0/templates/policy.yaml new file mode 100644 index 00000000..ef291ec6 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/policy.yaml @@ -0,0 +1,12 @@ +kind: policy +name: {{ include "cassandra.name" . }} +origin: default +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "cassandra.name" . }}-init + - //secret/{{ include "cassandra.name" . }}-config diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml new file mode 100644 index 00000000..be952d11 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -0,0 +1,33 @@ +kind: secret +name: {{ include "cassandra.name" . }}-config +type: opaque +data: + encoding: plain + payload: | + cluster_name: '{{ .Values.cassandra.clusterName }}' + + # Networking — replaced at pod startup by the init script + listen_address: LISTEN_ADDRESS_PLACEHOLDER + broadcast_address: LISTEN_ADDRESS_PLACEHOLDER + rpc_address: 0.0.0.0 + broadcast_rpc_address: LISTEN_ADDRESS_PLACEHOLDER + + # Seeds — replaced at pod startup by the init script + seed_provider: + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + - seeds: "SEEDS_PLACEHOLDER" + + # Required directives (Cassandra 5.x has no built-in fallback for these) + partitioner: org.apache.cassandra.dht.Murmur3Partitioner + num_tokens: 16 + commitlog_sync: periodic + commitlog_sync_period: 10000ms + + # Storage + data_file_directories: + - /var/lib/cassandra/data + commitlog_directory: /var/lib/cassandra/commitlog + saved_caches_directory: /var/lib/cassandra/saved_caches + + endpoint_snitch: GossipingPropertyFileSnitch diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml new file mode 100644 index 00000000..8231543f --- /dev/null +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -0,0 +1,63 @@ +kind: secret +name: {{ include "cassandra.name" . }}-init +type: opaque +data: + encoding: plain + payload: | + #!/bin/bash + set -euo pipefail + + # Derive own FQDN from /etc/hosts. + # Control Plane inserts a line like: + # 10.x.x.x cassandra-0.cassandra..svc.cluster.local cassandra-0 + MY_FQDN=$(grep -E "^[0-9]" /etc/hosts | grep "${HOSTNAME}" | awk '{print $2}') + + if [ -z "${MY_FQDN}" ]; then + echo "ERROR: Could not derive FQDN for ${HOSTNAME} from /etc/hosts" + cat /etc/hosts + exit 1 + fi + + # Parse components from the FQDN. + NAMESPACE_HASH=$(echo "${MY_FQDN}" | cut -d'.' -f3) + SERVICE=$(echo "${MY_FQDN}" | cut -d'.' -f2) + REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') + + echo "HOSTNAME: ${HOSTNAME}" + echo "MY_FQDN: ${MY_FQDN}" + echo "NAMESPACE_HASH: ${NAMESPACE_HASH}" + echo "SERVICE: ${SERVICE}" + echo "REPLICA_INDEX: ${REPLICA_INDEX}" + + # Build seed list from replicas 0 and 1 only (avoids "only seed" warning and + # keeps seed count stable regardless of cluster size). + SEED_COUNT=2 + SEEDS="" + for i in $(seq 0 $((SEED_COUNT - 1))); do + SEED_FQDN="${SERVICE}-${i}.${SERVICE}.${NAMESPACE_HASH}.svc.cluster.local" + SEEDS="${SEEDS}${SEED_FQDN}," + done + SEEDS="${SEEDS%,}" + + echo "SEEDS: ${SEEDS}" + + # Cassandra data dir ownership (cassandra runs as uid 999 in the official image) + mkdir -p /var/lib/cassandra + chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true + rm -rf /var/lib/cassandra/lost+found 2>/dev/null || true + + # Copy the mounted config template and replace placeholders with runtime values. + # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) + cp /config-template/cassandra-template.yaml /etc/cassandra/cassandra.yaml + sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml + sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml + + echo "cassandra.yaml written. Starting Cassandra..." + # Drop from root to the cassandra user (uid 999) using gosu, which is bundled + # in the official Cassandra image. Cassandra refuses to start as root without -R. + if [ "$(id -u)" = "0" ]; then + chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true + exec gosu cassandra cassandra -f + else + exec cassandra -f + fi diff --git a/cassandra/versions/1.0.0/templates/volumeset.yaml b/cassandra/versions/1.0.0/templates/volumeset.yaml new file mode 100644 index 00000000..b48ee13d --- /dev/null +++ b/cassandra/versions/1.0.0/templates/volumeset.yaml @@ -0,0 +1,12 @@ +kind: volumeset +name: {{ include "cassandra.name" . }}-data +description: {{ include "cassandra.name" . }} data +gvc: {{ .Values.global.cpln.gvc }} +spec: + initialCapacity: {{ .Values.cassandra.volumes.data.initialCapacity }} + performanceClass: {{ .Values.cassandra.volumes.data.performanceClass }} + fileSystemType: {{ .Values.cassandra.volumes.data.fileSystemType }} + autoscaling: + maxCapacity: {{ .Values.cassandra.volumes.data.autoscaling.maxCapacity }} + minFreePercentage: {{ .Values.cassandra.volumes.data.autoscaling.minFreePercentage }} + scalingFactor: {{ .Values.cassandra.volumes.data.autoscaling.scalingFactor }} diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml new file mode 100644 index 00000000..cbc054fa --- /dev/null +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -0,0 +1,85 @@ +kind: workload +name: {{ include "cassandra.name" . }} +description: Cassandra cluster +gvc: {{ .Values.global.cpln.gvc }} +spec: + type: stateful + containers: + - name: cassandra + command: /bin/bash + args: + - '-c' + - >- + cp /scripts/cassandra-init.sh /tmp/cassandra-init.sh && + chmod +x /tmp/cassandra-init.sh && + /tmp/cassandra-init.sh + env: + - name: MAX_HEAP_SIZE + value: {{ .Values.cassandra.jvmHeapSize | quote }} + image: {{ .Values.cassandra.image }} + cpu: '{{ .Values.cassandra.cpu }}' + memory: {{ .Values.cassandra.memory }} + inheritEnv: false + ports: + - number: 9042 + protocol: tcp + - number: 7000 + protocol: tcp + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + tcpSocket: + port: 9042 + timeoutSeconds: 10 + readinessProbe: + failureThreshold: 20 + initialDelaySeconds: 60 + periodSeconds: 15 + successThreshold: 3 + tcpSocket: + port: 9042 + timeoutSeconds: 5 + volumes: + - path: /var/lib/cassandra + recoveryPolicy: retain + uri: 'cpln://volumeset/{{ include "cassandra.name" . }}-data' + - path: /config-template/cassandra-template.yaml + recoveryPolicy: retain + uri: 'cpln://secret/{{ include "cassandra.name" . }}-config' + - path: /scripts/cassandra-init.sh + recoveryPolicy: retain + uri: 'cpln://secret/{{ include "cassandra.name" . }}-init' + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: {{ .Values.cassandra.replicas }} + metric: disabled + minScale: {{ .Values.cassandra.replicas }} + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + suspend: {{ .Values.cassandra.suspend }} + timeoutSeconds: 30 + firewallConfig: + external: + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: same-gvc + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + identityLink: //identity/{{ include "cassandra.name" . }} + rolloutOptions: + maxSurgeReplicas: 0% + maxUnavailableReplicas: '1' + minReadySeconds: 60 + scalingPolicy: OrderedReady + securityOptions: + filesystemGroupId: 999 + supportDynamicTags: false diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index e69de29b..badaed62 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -0,0 +1,21 @@ +cassandra: + # Must match minScale/maxScale below. Use odd numbers: 3, 5, 7. + replicas: 3 + image: cassandra:5.0 + cpu: 2 + memory: 8Gi + # JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) + # Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. + jvmHeapSize: 2G + clusterName: cp-cassandra + suspend: false + + volumes: + data: + initialCapacity: 10 + performanceClass: general-purpose-ssd + fileSystemType: ext4 + autoscaling: + maxCapacity: 100 + minFreePercentage: 20 + scalingFactor: 1.5 From c9a9e6cbfae93c5a9dd420b58deb1c6278116d11 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:59:55 -0700 Subject: [PATCH 03/58] init script now queries for sentinel for master, updated components around that feature, added cpln-common tags (#244) * init 3.3.0 * added sentinel password env to redis workload, init now queries sentinel for master * added cpln-common tagging * added sentinel secret to redis policy * added master discovery on publicAccess mode * working changes --- redis/versions/3.3.0/Chart.yaml | 17 ++ redis/versions/3.3.0/README.md | 207 +++++++++++++++ redis/versions/3.3.0/templates/_helpers.tpl | 243 ++++++++++++++++++ .../3.3.0/templates/domain-redis.yaml | 20 ++ .../3.3.0/templates/domain-sentinel.yaml | 20 ++ .../3.3.0/templates/identity-redis.yaml | 23 ++ .../3.3.0/templates/identity-sentinel.yaml | 4 + .../3.3.0/templates/policy-backup.yaml | 17 ++ .../3.3.0/templates/policy-redis.yaml | 20 ++ .../3.3.0/templates/policy-sentinel.yaml | 20 ++ .../3.3.0/templates/secret-backup.yaml | 17 ++ .../3.3.0/templates/secret-redis-config.yaml | 12 + .../templates/secret-redis-password.yaml | 7 + .../templates/secret-sentinel-config.yaml | 16 ++ .../templates/secret-sentinel-password.yaml | 7 + .../3.3.0/templates/volumeset-redis.yaml | 26 ++ .../3.3.0/templates/volumeset-sentinel.yaml | 26 ++ .../3.3.0/templates/workload-backup.yaml | 75 ++++++ .../3.3.0/templates/workload-redis.yaml | 211 +++++++++++++++ .../3.3.0/templates/workload-sentinel.yaml | 179 +++++++++++++ redis/versions/3.3.0/values.yaml | 129 ++++++++++ 21 files changed, 1296 insertions(+) create mode 100644 redis/versions/3.3.0/Chart.yaml create mode 100644 redis/versions/3.3.0/README.md create mode 100644 redis/versions/3.3.0/templates/_helpers.tpl create mode 100644 redis/versions/3.3.0/templates/domain-redis.yaml create mode 100644 redis/versions/3.3.0/templates/domain-sentinel.yaml create mode 100644 redis/versions/3.3.0/templates/identity-redis.yaml create mode 100644 redis/versions/3.3.0/templates/identity-sentinel.yaml create mode 100644 redis/versions/3.3.0/templates/policy-backup.yaml create mode 100644 redis/versions/3.3.0/templates/policy-redis.yaml create mode 100644 redis/versions/3.3.0/templates/policy-sentinel.yaml create mode 100644 redis/versions/3.3.0/templates/secret-backup.yaml create mode 100644 redis/versions/3.3.0/templates/secret-redis-config.yaml create mode 100644 redis/versions/3.3.0/templates/secret-redis-password.yaml create mode 100644 redis/versions/3.3.0/templates/secret-sentinel-config.yaml create mode 100644 redis/versions/3.3.0/templates/secret-sentinel-password.yaml create mode 100644 redis/versions/3.3.0/templates/volumeset-redis.yaml create mode 100644 redis/versions/3.3.0/templates/volumeset-sentinel.yaml create mode 100644 redis/versions/3.3.0/templates/workload-backup.yaml create mode 100644 redis/versions/3.3.0/templates/workload-redis.yaml create mode 100644 redis/versions/3.3.0/templates/workload-sentinel.yaml create mode 100644 redis/versions/3.3.0/values.yaml diff --git a/redis/versions/3.3.0/Chart.yaml b/redis/versions/3.3.0/Chart.yaml new file mode 100644 index 00000000..4b1619a1 --- /dev/null +++ b/redis/versions/3.3.0/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: redis +description: A master-replica Redis configuration with Redis Sentinel +type: application +version: 3.3.0 +appVersion: "custom" + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2026-01-29" + lastModified: "2026-04-21" + category: "cache" + createsGvc: false \ No newline at end of file diff --git a/redis/versions/3.3.0/README.md b/redis/versions/3.3.0/README.md new file mode 100644 index 00000000..71acce83 --- /dev/null +++ b/redis/versions/3.3.0/README.md @@ -0,0 +1,207 @@ +## Redis Sentinel + +Creates a Redis Sentinel cluster on Control Plane with automatic leader election, failover, and an optional backup configuration. + +### Configuration + +**Redis and Sentinel** — set replicas, resources, and timeouts for each. Sentinel replicas must be an odd number for quorum: +```yaml +redis: + replicas: 2 + resources: + minCpu: 80m + minMemory: 128Mi + cpu: 200m + memory: 256Mi + +sentinel: + replicas: 3 + quorumAutoCalculation: true # calculates as (replicas/2)+1 +``` + +**Authentication** — enable one method. Apply the same config under both `redis.auth` and `sentinel.auth`: +```yaml +redis: + auth: + password: + enabled: true + value: your-password + # fromSecret: + # enabled: true + # name: my-redis-secret + # passwordKey: password +``` + +**Persistence** — disabled by default. Enable to attach a persistent volume to Redis: +```yaml +redis: + persistence: + enabled: true + volumes: + data: + initialCapacity: 10 + performanceClass: general-purpose-ssd # or high-throughput-ssd (min 1000 GiB) + fileSystemType: ext4 +``` + +**Firewall** — set the internal access scope for both Redis and Sentinel: +```yaml +firewall: + internal_inboundAllowType: same-gvc # same-gvc, same-org, or workload-list +``` + +### Public Access (External TCP) + +Redis and Sentinel can be exposed over the internet via TCP using Control Plane's domain resource with per-replica port routing. + +#### Prerequisites + +1. **A domain you control** with DNS managed by your registrar (e.g. Cloudflare) +2. **Dedicated Load Balancer** enabled on your GVC — required for arbitrary TCP port routing. Enable this under your GVC settings in the Control Plane console. +3. **DNS records added before deploying** — Control Plane will reject the domain resource on first deploy if ownership has not been proven. Add the following records in your DNS provider for each address before running the deployment. **Disable proxying** (e.g. Cloudflare's orange cloud) — TCP traffic must pass through directly: + +| Type | Name | Value | +|------|------|-------| +| TXT | `_cpln-` | your Control Plane org name or org ID (either is accepted) | +| CNAME | `` | `.cpln.app` | + +Your GVC alias is visible in the Control Plane console under GVC settings. The TXT record proves domain ownership — without it, the first deploy will fail with an `Unable to apply domain` error. + +#### Configuration + +Enable `publicAccess` for Redis and/or Sentinel and set a subdomain you own: + +```yaml +redis: + publicAccess: + enabled: true + address: redis.your-domain.com + firewall: + internal_inboundAllowType: same-gvc + external_inboundAllowCIDR: "0.0.0.0/0" # or restrict to specific CIDRs + +sentinel: + publicAccess: + enabled: true + address: redis-sentinel.your-domain.com + firewall: + internal_inboundAllowType: same-gvc + external_inboundAllowCIDR: "0.0.0.0/0" +``` + +When enabled, a Control Plane `domain` resource is created for each address. Port mapping is one port per replica: +- **Redis**: ports `6380`, `6381`, ... (replica 0, 1, ...) +- **Sentinel**: ports `26380`, `26381`, `26382`, ... (replica 0, 1, 2, ...) + +After DNS propagates, the domain status in Control Plane will show **Ready**. You can verify the full DNS chain resolves correctly with: + +```bash +dig .your-domain.com CNAME # should return .cpln.app +dig .cpln.app # should return an IP address +``` + +#### Connecting Externally + +```bash +# Redis replica 0 +redis-cli -h redis.your-domain.com -p 6380 ping + +# Redis replica 1 +redis-cli -h redis.your-domain.com -p 6381 ping + +# Sentinel replica 0 +redis-cli -h redis-sentinel.your-domain.com -p 26380 ping +``` + +### Connecting + +Redis is accessible internally on port 6379: +``` +RELEASE_NAME-redis.GVC_NAME.cpln.local:6379 +``` + +Sentinel is accessible on port 26379: +``` +RELEASE_NAME-sentinel.GVC_NAME.cpln.local:26379 +``` + +To route writes to the current master: +```bash +MASTER_INFO=$(redis-cli -h RELEASE_NAME-sentinel.GVC_NAME.cpln.local -p 26379 SENTINEL get-master-addr-by-name mymaster) +MASTER_HOST=$(echo $MASTER_INFO | cut -d' ' -f1) +MASTER_PORT=$(echo $MASTER_INFO | cut -d' ' -f2) +redis-cli -h $MASTER_HOST -p $MASTER_PORT SET my-key "value" +``` + +### Backing Up + +Set `backup.enabled` to `true`, configure your provider, and set your desired schedule. The backup image is compatible with all Redis versions. + +```yaml +backup: + enabled: true + schedule: "0 2 * * *" # daily at 2am UTC + provider: aws # Options: aws or gcp +``` + +#### AWS S3 + +For the backup cron job to access an S3 bucket, complete the following in your AWS account first: + +1. Create your bucket. Set `backup.aws.bucket` to its name and `backup.aws.region` to its region. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Set `backup.aws.cloudAccountName` to its name. + +3. Create a new IAM policy with the following JSON (replace `YOUR_BUCKET_NAME`) and set `backup.aws.policyName` to match: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetObjectVersion", + "s3:DeleteObjectVersion" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + } + ] +} +``` + +#### GCS + +For the backup cron job to access a GCS bucket, complete the following in your GCP account first: + +1. Create your bucket. Set `backup.gcp.bucket` to its name. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Set `backup.gcp.cloudAccountName` to its name. + +**Important**: You must add the `Storage Admin` role when creating your GCP service account. + +### Restoring a Backup + +Run the following command from a client with access to the bucket (replace `aws s3 cp` with `gsutil cp` for GCS): + +```sh +aws s3 cp s3://BUCKET_NAME/PREFIX/BACKUP_FILE.rdb /tmp/dump.rdb +redis-cli \ + -h RELEASE_NAME-redis.GVC_NAME.cpln.local \ + -p 6379 \ + --rdb /tmp/dump.rdb +``` + +### Supported External Services +- [Redis Documentation](https://redis.io/docs/) +- [Redis Sentinel Documentation](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/) + +### Release Notes +See [RELEASES.md](https://github.com/controlplane-com/templates/blob/main/redis/RELEASES.md) diff --git a/redis/versions/3.3.0/templates/_helpers.tpl b/redis/versions/3.3.0/templates/_helpers.tpl new file mode 100644 index 00000000..10f8995b --- /dev/null +++ b/redis/versions/3.3.0/templates/_helpers.tpl @@ -0,0 +1,243 @@ +{{/* Resource Naming */}} + +{{/* +Redis Workload Name +*/}} +{{- define "redis.name" -}} +{{- printf "%s-redis" .Release.Name }} +{{- end }} + +{{/* +Redis Sentinel Workload Name +*/}} +{{- define "redis.sentinel.name" -}} +{{- printf "%s-sentinel" .Release.Name }} +{{- end }} + +{{/* +Redis Secret Config Name +*/}} +{{- define "redis.secretConfig.name" -}} +{{- printf "%s-redis-config" .Release.Name }} +{{- end }} + +{{/* +Redis Secret Auth Password Name +*/}} +{{- define "redis.secretPassword.name" -}} +{{- printf "%s-redis-auth-password" .Release.Name }} +{{- end }} + +{{/* +Redis Sentinel Secret Config Name +*/}} +{{- define "redis.sentinelSecretConfig.name" -}} +{{- printf "%s-sentinel-config" .Release.Name }} +{{- end }} + +{{/* +Redis Sentinel Secret Auth Password Name +*/}} +{{- define "redis.sentinelSecretPassword.name" -}} +{{- printf "%s-sentinel-auth-password" .Release.Name }} +{{- end }} + +{{/* +Redis Identity Name +*/}} +{{- define "redis.identity.name" -}} +{{- printf "%s-redis-identity" .Release.Name }} +{{- end }} + +{{/* +Redis Sentinel Identity Name +*/}} +{{- define "redis.sentinelIdentity.name" -}} +{{- printf "%s-sentinel-identity" .Release.Name }} +{{- end }} + +{{/* +Redis Policy Name +*/}} +{{- define "redis.policy.name" -}} +{{- printf "%s-redis-policy" .Release.Name }} +{{- end }} + +{{/* +Redis Sentinel Policy Name +*/}} +{{- define "redis.sentinelPolicy.name" -}} +{{- printf "%s-sentinel-policy" .Release.Name }} +{{- end }} + +{{/* +Redis Volume Set Name +*/}} +{{- define "redis.volume.name" -}} +{{- printf "%s-redis-vs" .Release.Name }} +{{- end }} + +{{/* +Redis Sentinel Volume Set Name +*/}} +{{- define "redis.sentinelVolume.name" -}} +{{- printf "%s-sentinel-vs" .Release.Name }} +{{- end }} + + +{{/* +Redis Backup Workload Name +*/}} +{{- define "redis.backup.name" -}} +{{- printf "%s-redis-backup" .Release.Name }} +{{- end }} + +{{/* +Redis Backup Secret Config Name +*/}} +{{- define "redis.secretBackup.name" -}} +{{- printf "%s-redis-backup-config" .Release.Name }} +{{- end }} + +{{/* +Redis Backup Policy Name +*/}} +{{- define "redis.backupPolicy.name" -}} +{{- printf "%s-redis-backup-policy" .Release.Name }} +{{- end }} + + +{{/* Validation */}} + +{{/* +Validate backup configuration - when backup is enabled, backup.provider must be set to 'aws' or 'gcp' +*/}} +{{- define "redis.validateBackupConfig" -}} +{{- if .Values.backup.enabled -}} + {{- $provider := .Values.backup.provider -}} + {{- if not (or (eq $provider "aws") (eq $provider "gcp")) -}} + {{- fail "Invalid backup configuration: backup.provider must be set to 'aws' or 'gcp'." -}} + {{- end -}} + {{- if eq $provider "aws" -}} + {{- if not .Values.backup.aws.bucket -}} + {{- fail "All fields are required for AWS backup. Missing: backup.aws.bucket" -}} + {{- end -}} + {{- if not .Values.backup.aws.region -}} + {{- fail "All fields are required for AWS backup. Missing: backup.aws.region" -}} + {{- end -}} + {{- if not .Values.backup.aws.cloudAccountName -}} + {{- fail "All fields are required for AWS backup. Missing: backup.aws.cloudAccountName" -}} + {{- end -}} + {{- if not .Values.backup.aws.policyName -}} + {{- fail "All fields are required for AWS backup. Missing: backup.aws.policyName" -}} + {{- end -}} + {{- end -}} + {{- if eq $provider "gcp" -}} + {{- if not .Values.backup.gcp.bucket -}} + {{- fail "All fields are required for GCP backup. Missing: backup.gcp.bucket" -}} + {{- end -}} + {{- if not .Values.backup.gcp.cloudAccountName -}} + {{- fail "All fields are required for GCP backup. Missing: backup.gcp.cloudAccountName" -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end }} + +{{- define "calculateWorkloadCounts" -}} +{{- $quorumCount := int .Values.sentinel.quorum }} +{{- $workloadCount := 0 }} +{{- if eq $quorumCount 1 }} + {{- $workloadCount = 1 }} +{{- else }} + {{- $workloadCount = int (add $quorumCount 1) }} +{{- end }} +{{- $locations := default (list) .Values.locations }} +{{- if and $locations (gt (len $locations) 0) }} + {{- $locationCount := (len $locations) }} + {{- $baseCount := int (div $workloadCount $locationCount) }} + {{- $remainderCount := int (mod $workloadCount $locationCount) }} + {{- if not .Values.global }} + {{- $ := set .Values "global" (dict) }} + {{- end }} + {{- $ := set .Values.global "baseCount" $baseCount }} + {{- $ := set .Values.global "remainderCount" $remainderCount }} + {{- $ := set .Values.global "locationCount" $locationCount }} + {{- $ := set .Values.global "workloadCount" $workloadCount }} +{{- end }} +{{- end }} + + +{{ include "redis.auth" (dict "auth" .Values.redis.auth) }} + +redis: + image: redis/redis-stack:7.4.0-v3 + resources: + cpu: 200m + memory: 256Mi + minCpu: 80m + minMemory: 128Mi + replicas: 3 + timeoutSeconds: 15 + auth: + fromSecret: + enabled: false + name: example-redis-auth-password + passwordKey: password + password: + enabled: true + value: fu3h4f9834f8 + +{{/* +Validate auth configuration block +*/}} +{{- define "validateAuth" -}} +{{- $auth := .auth -}} + +{{- /* Check if auth block exists */ -}} +{{- if $auth -}} + {{- /* Count enabled auth methods */ -}} + {{- $enabledCount := 0 -}} + + {{- /* Check fromSecret */ -}} + {{- if and (hasKey $auth "fromSecret") $auth.fromSecret.enabled -}} + {{- $enabledCount = add1 $enabledCount -}} + {{- end -}} + + {{- /* Check password */ -}} + {{- if and (hasKey $auth "password") $auth.password.enabled -}} + {{- $enabledCount = add1 $enabledCount -}} + {{- end -}} + + {{- /* Validate that at most one method is enabled */ -}} + {{- if gt $enabledCount 1 -}} + {{- fail "Only one authentication method can be enabled at a time" -}} + {{- end -}} + + {{- /* If fromSecret is enabled, validate its configuration */ -}} + {{- if and (hasKey $auth "fromSecret") $auth.fromSecret.enabled -}} + {{- if not (hasKey $auth.fromSecret "name") -}} + {{- fail "fromSecret authentication requires a name" -}} + {{- end -}} + {{- if not (hasKey $auth.fromSecret "passwordKey") -}} + {{- fail "fromSecret authentication requires a passwordKey" -}} + {{- end -}} + {{- end -}} + + {{- /* If password is enabled, validate its configuration */ -}} + {{- if and (hasKey $auth "password") $auth.password.enabled -}} + {{- if not (hasKey $auth.password "value") -}} + {{- fail "password authentication requires a value" -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + + +{{/* Labeling */}} + +{{/* +Common labels - delegated to cpln-common +*/}} +{{- define "redis.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/domain-redis.yaml b/redis/versions/3.3.0/templates/domain-redis.yaml new file mode 100644 index 00000000..313c6e67 --- /dev/null +++ b/redis/versions/3.3.0/templates/domain-redis.yaml @@ -0,0 +1,20 @@ +{{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} +kind: domain +name: {{ .Values.redis.publicAccess.address }} +description: {{ .Values.redis.publicAccess.address }} +spec: + acceptAllHosts: false + dnsMode: cname + {{- if gt (.Values.redis.replicas | int) 0 }} + ports: + {{- range $i := until (int .Values.redis.replicas) }} + - number: {{ add 6380 $i }} + protocol: tcp + routes: + - port: {{ add 6380 $i }} + prefix: / + replica: {{ $i }} + workloadLink: //gvc/{{ $.Values.global.cpln.gvc }}/workload/{{ include "redis.name" $ }} + {{- end }} + {{- end }} +{{- end }} diff --git a/redis/versions/3.3.0/templates/domain-sentinel.yaml b/redis/versions/3.3.0/templates/domain-sentinel.yaml new file mode 100644 index 00000000..0c5c6464 --- /dev/null +++ b/redis/versions/3.3.0/templates/domain-sentinel.yaml @@ -0,0 +1,20 @@ +{{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} +kind: domain +name: {{ .Values.sentinel.publicAccess.address }} +description: {{ .Values.sentinel.publicAccess.address }} +spec: + acceptAllHosts: false + dnsMode: cname + {{- if gt (.Values.sentinel.replicas | int) 0 }} + ports: + {{- range $i := until (int .Values.sentinel.replicas) }} + - number: {{ add 26380 $i }} + protocol: tcp + routes: + - port: {{ add 26380 $i }} + prefix: / + replica: {{ $i }} + workloadLink: //gvc/{{ $.Values.global.cpln.gvc }}/workload/{{ include "redis.sentinel.name" $ }} + {{- end }} + {{- end }} +{{- end }} diff --git a/redis/versions/3.3.0/templates/identity-redis.yaml b/redis/versions/3.3.0/templates/identity-redis.yaml new file mode 100644 index 00000000..9cd19d54 --- /dev/null +++ b/redis/versions/3.3.0/templates/identity-redis.yaml @@ -0,0 +1,23 @@ +{{- include "redis.validateBackupConfig" . -}} +kind: identity +name: {{ include "redis.identity.name" . }} +description: Redis Identity +gvc: {{ .Values.global.cpln.gvc }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.backup.aws.cloudAccountName }} + policyRefs: + - cpln-connector + - aws::ReadOnlyAccess + - {{ .Values.backup.aws.policyName | quote }} +{{- end }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "gcp") }} +gcp: + bindings: + - resource: //storage.googleapis.com/projects/_/buckets/{{ .Values.backup.gcp.bucket }} + roles: + - roles/storage.objectAdmin + cloudAccountLink: //cloudaccount/{{ .Values.backup.gcp.cloudAccountName }} + scopes: + - https://www.googleapis.com/auth/cloud-platform +{{- end }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/identity-sentinel.yaml b/redis/versions/3.3.0/templates/identity-sentinel.yaml new file mode 100644 index 00000000..c7311c98 --- /dev/null +++ b/redis/versions/3.3.0/templates/identity-sentinel.yaml @@ -0,0 +1,4 @@ +kind: identity +name: {{ include "redis.sentinelIdentity.name" . }} +description: Redis Sentinel Identity +gvc: {{ .Values.global.cpln.gvc }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/policy-backup.yaml b/redis/versions/3.3.0/templates/policy-backup.yaml new file mode 100644 index 00000000..3185c454 --- /dev/null +++ b/redis/versions/3.3.0/templates/policy-backup.yaml @@ -0,0 +1,17 @@ +{{- if .Values.backup.enabled }} +kind: policy +name: {{ include "redis.backupPolicy.name" . }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "redis.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "redis.secretBackup.name" . }} + {{- if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled }} + - //secret/{{ .Values.redis.auth.fromSecret.name }} + {{- else if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} + - //secret/{{ include "redis.secretPassword.name" . }} + {{- end }} +{{- end }} diff --git a/redis/versions/3.3.0/templates/policy-redis.yaml b/redis/versions/3.3.0/templates/policy-redis.yaml new file mode 100644 index 00000000..322d26a1 --- /dev/null +++ b/redis/versions/3.3.0/templates/policy-redis.yaml @@ -0,0 +1,20 @@ +kind: policy +name: {{ include "redis.policy.name" . }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "redis.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "redis.secretConfig.name" . }} + {{- if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled }} + - //secret/{{ .Values.redis.auth.fromSecret.name }} + {{- else if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} + - //secret/{{ include "redis.secretPassword.name" . }} + {{- end }} + {{- if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled }} + - //secret/{{ .Values.sentinel.auth.fromSecret.name }} + {{- else if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled }} + - //secret/{{ include "redis.sentinelSecretPassword.name" . }} + {{- end }} diff --git a/redis/versions/3.3.0/templates/policy-sentinel.yaml b/redis/versions/3.3.0/templates/policy-sentinel.yaml new file mode 100644 index 00000000..f7aff9f5 --- /dev/null +++ b/redis/versions/3.3.0/templates/policy-sentinel.yaml @@ -0,0 +1,20 @@ +kind: policy +name: {{ include "redis.sentinelPolicy.name" . }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "redis.sentinelIdentity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "redis.sentinelSecretConfig.name" . }} + {{- if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled }} + - //secret/{{ .Values.sentinel.auth.fromSecret.name }} + {{- else if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled }} + - //secret/{{ include "redis.sentinelSecretPassword.name" . }} + {{- end }} + {{- if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled }} + - //secret/{{ .Values.redis.auth.fromSecret.name }} + {{- else if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} + - //secret/{{ include "redis.secretPassword.name" . }} + {{- end }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/secret-backup.yaml b/redis/versions/3.3.0/templates/secret-backup.yaml new file mode 100644 index 00000000..d244ca02 --- /dev/null +++ b/redis/versions/3.3.0/templates/secret-backup.yaml @@ -0,0 +1,17 @@ +{{- include "redis.validateBackupConfig" . }} +{{- if .Values.backup.enabled }} +kind: secret +name: {{ include "redis.secretBackup.name" . }} +description: Redis backup configuration +tags: + {{- include "redis.tags" . | nindent 4 }} +type: dictionary +data: +{{- if eq .Values.backup.provider "aws" }} + backup-bucket: {{ .Values.backup.aws.bucket | quote }} + aws-region: {{ .Values.backup.aws.region | quote }} +{{- end }} +{{- if eq .Values.backup.provider "gcp" }} + backup-bucket: {{ .Values.backup.gcp.bucket | quote }} +{{- end }} +{{- end }} diff --git a/redis/versions/3.3.0/templates/secret-redis-config.yaml b/redis/versions/3.3.0/templates/secret-redis-config.yaml new file mode 100644 index 00000000..7b35f60d --- /dev/null +++ b/redis/versions/3.3.0/templates/secret-redis-config.yaml @@ -0,0 +1,12 @@ +kind: secret +name: {{ include "redis.secretConfig.name" . }} +type: opaque +data: + encoding: plain + payload: |- + bind 0.0.0.0 + protected-mode no + save 900 1 + save 300 10 + save 60 10000 + appendonly yes \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/secret-redis-password.yaml b/redis/versions/3.3.0/templates/secret-redis-password.yaml new file mode 100644 index 00000000..51b90dc7 --- /dev/null +++ b/redis/versions/3.3.0/templates/secret-redis-password.yaml @@ -0,0 +1,7 @@ +{{ if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} +kind: secret +name: {{ include "redis.secretPassword.name" . }} +type: dictionary +data: + password: {{ .Values.redis.auth.password.value | quote }} +{{- end }} diff --git a/redis/versions/3.3.0/templates/secret-sentinel-config.yaml b/redis/versions/3.3.0/templates/secret-sentinel-config.yaml new file mode 100644 index 00000000..6465128d --- /dev/null +++ b/redis/versions/3.3.0/templates/secret-sentinel-config.yaml @@ -0,0 +1,16 @@ +kind: secret +name: {{ include "redis.sentinelSecretConfig.name" . }} +type: opaque +data: + encoding: plain + payload: |- + {{- if and (hasKey .Values.sentinel "persistence") .Values.sentinel.persistence.enabled }} + dir /etc/sentinel/data + {{- else }} + dir /tmp + {{- end }} + sentinel announce-hostnames yes + sentinel resolve-hostnames yes + sentinel down-after-milliseconds mymaster 5000 + sentinel failover-timeout mymaster 10000 + sentinel parallel-syncs mymaster 1 \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/secret-sentinel-password.yaml b/redis/versions/3.3.0/templates/secret-sentinel-password.yaml new file mode 100644 index 00000000..1e752fc0 --- /dev/null +++ b/redis/versions/3.3.0/templates/secret-sentinel-password.yaml @@ -0,0 +1,7 @@ +{{ if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled }} +kind: secret +name: {{ include "redis.sentinelSecretPassword.name" . }} +type: dictionary +data: + password: {{ .Values.sentinel.auth.password.value | quote}} +{{- end }} diff --git a/redis/versions/3.3.0/templates/volumeset-redis.yaml b/redis/versions/3.3.0/templates/volumeset-redis.yaml new file mode 100644 index 00000000..5b2eb0ef --- /dev/null +++ b/redis/versions/3.3.0/templates/volumeset-redis.yaml @@ -0,0 +1,26 @@ +{{- if and (hasKey .Values.redis "persistence") .Values.redis.persistence.enabled }} +kind: volumeset +name: {{ include "redis.volume.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +spec: + fileSystemType: {{ .Values.redis.persistence.volumes.data.fileSystemType }} + initialCapacity: {{ .Values.redis.persistence.volumes.data.initialCapacity }} + performanceClass: {{ .Values.redis.persistence.volumes.data.performanceClass }} + {{- if and .Values.redis.persistence.volumes.data.customEncryption .Values.redis.persistence.volumes.data.customEncryption.enabled }} + customEncryption: + regions: + {{ .Values.redis.persistence.volumes.data.customEncryption.region }}: + keyId: '{{ .Values.redis.persistence.volumes.data.customEncryption.keyId }}' + {{- end }} + {{- if .Values.redis.persistence.volumes.data.snapshots }} + snapshots: + retentionDuration: {{ .Values.redis.persistence.volumes.data.snapshots.retentionDuration }} + schedule: {{ .Values.redis.persistence.volumes.data.snapshots.schedule }} + {{- end }} + {{- if .Values.redis.persistence.volumes.data.autoscaling }} + autoscaling: + maxCapacity: {{ .Values.redis.persistence.volumes.data.autoscaling.maxCapacity }} + minFreePercentage: {{ .Values.redis.persistence.volumes.data.autoscaling.minFreePercentage }} + scalingFactor: {{ .Values.redis.persistence.volumes.data.autoscaling.scalingFactor }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/volumeset-sentinel.yaml b/redis/versions/3.3.0/templates/volumeset-sentinel.yaml new file mode 100644 index 00000000..976c315d --- /dev/null +++ b/redis/versions/3.3.0/templates/volumeset-sentinel.yaml @@ -0,0 +1,26 @@ +{{- if and (hasKey .Values.sentinel "persistence") .Values.sentinel.persistence.enabled }} +kind: volumeset +name: {{ include "redis.sentinelVolume.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +spec: + fileSystemType: {{ .Values.sentinel.persistence.volumes.data.fileSystemType }} + initialCapacity: {{ .Values.sentinel.persistence.volumes.data.initialCapacity }} + performanceClass: {{ .Values.sentinel.persistence.volumes.data.performanceClass }} + {{- if and .Values.sentinel.persistence.volumes.data.customEncryption .Values.sentinel.persistence.volumes.data.customEncryption.enabled }} + customEncryption: + regions: + {{ .Values.sentinel.persistence.volumes.data.customEncryption.region }}: + keyId: '{{ .Values.sentinel.persistence.volumes.data.customEncryption.keyId }}' + {{- end }} + {{- if .Values.sentinel.persistence.volumes.data.snapshots }} + snapshots: + retentionDuration: {{ .Values.sentinel.persistence.volumes.data.snapshots.retentionDuration }} + schedule: {{ .Values.sentinel.persistence.volumes.data.snapshots.schedule }} + {{- end }} + {{- if .Values.sentinel.persistence.volumes.data.autoscaling }} + autoscaling: + maxCapacity: {{ .Values.sentinel.persistence.volumes.data.autoscaling.maxCapacity }} + minFreePercentage: {{ .Values.sentinel.persistence.volumes.data.autoscaling.minFreePercentage }} + scalingFactor: {{ .Values.sentinel.persistence.volumes.data.autoscaling.scalingFactor }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/workload-backup.yaml b/redis/versions/3.3.0/templates/workload-backup.yaml new file mode 100644 index 00000000..56106fcb --- /dev/null +++ b/redis/versions/3.3.0/templates/workload-backup.yaml @@ -0,0 +1,75 @@ +{{- include "redis.validateBackupConfig" . }} +{{- if .Values.backup.enabled }} +kind: workload +name: {{ include "redis.backup.name" . }} +description: Redis Backup +tags: + {{- include "redis.tags" . | nindent 2 }} +spec: + type: cron + containers: + - name: backup-redis + cpu: {{ .Values.backup.resources.cpu | quote }} + memory: {{ .Values.backup.resources.memory | quote }} + env: + {{- if eq .Values.backup.provider "aws" }} + - name: AWS_REGION + value: cpln://secret/{{ include "redis.secretBackup.name" . }}.aws-region + - name: BACKUP_PROVIDER + value: aws + - name: BACKUP_BUCKET + value: cpln://secret/{{ include "redis.secretBackup.name" . }}.backup-bucket + - name: BACKUP_PREFIX + value: {{ .Values.backup.aws.prefix | quote }} + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + - name: BACKUP_PROVIDER + value: gcp + - name: BACKUP_BUCKET + value: cpln://secret/{{ include "redis.secretBackup.name" . }}.backup-bucket + - name: BACKUP_PREFIX + value: {{ .Values.backup.gcp.prefix | quote }} + {{- end }} + - name: REDIS_HOST + value: {{ include "redis.name" . }}.{{ .Values.global.cpln.gvc }}.cpln.local + {{- if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled }} + - name: REDIS_PASSWORD + value: cpln://secret/{{ .Values.redis.auth.fromSecret.name }}.{{ .Values.redis.auth.fromSecret.passwordKey }} + {{- else if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} + - name: REDIS_PASSWORD + value: cpln://secret/{{ include "redis.secretPassword.name" . }}.password + {{- end }} + image: {{ .Values.backup.image }} + inheritEnv: false + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 1 + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: none + inboundAllowWorkload: [] + identityLink: //identity/{{ include "redis.identity.name" . }} + job: + concurrencyPolicy: Forbid + historyLimit: 5 + restartPolicy: Never + schedule: {{ .Values.backup.schedule }} + supportDynamicTags: false +{{- end }} diff --git a/redis/versions/3.3.0/templates/workload-redis.yaml b/redis/versions/3.3.0/templates/workload-redis.yaml new file mode 100644 index 00000000..ef123a45 --- /dev/null +++ b/redis/versions/3.3.0/templates/workload-redis.yaml @@ -0,0 +1,211 @@ +{{ include "validateAuth" (dict "auth" .Values.redis.auth) }} +kind: workload +name: {{ include "redis.name" . }} +description: Redis +tags: + {{- if .Values.redis.tags }} +{{ toYaml .Values.redis.tags | indent 2 }} + {{- end }} + {{- include "redis.tags" . | nindent 2 }} +spec: + type: stateful + containers: + - name: redis + env: + {{- if .Values.redis.env }} +{{ toYaml .Values.redis.env | indent 8 }} + {{- end }} + {{- if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} + - name: REPLICA_DIRECT + value: "true" + {{- end }} + {{- if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled }} + - name: CUSTOM_REDIS_PASSWORD + value: cpln://secret/{{ .Values.redis.auth.fromSecret.name }}.{{ .Values.redis.auth.fromSecret.passwordKey }} + {{- else if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} + - name: CUSTOM_REDIS_PASSWORD + value: cpln://secret/{{ include "redis.secretPassword.name" . }}.password + {{- end }} + {{- if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled }} + - name: CUSTOM_SENTINEL_PASSWORD + value: cpln://secret/{{ .Values.sentinel.auth.fromSecret.name }}.{{ .Values.sentinel.auth.fromSecret.passwordKey }} + {{- else if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled }} + - name: CUSTOM_SENTINEL_PASSWORD + value: cpln://secret/{{ include "redis.sentinelSecretPassword.name" . }}.password + {{- end }} + {{- $hasSentinelAuth := or (and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled) (and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled) }} + {{- if not (or .Values.redis.env .Values.redis.replicaDirect (and (hasKey .Values.redis "auth") (or (and (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled) (and (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled))) $hasSentinelAuth) }} + [] + {{- end }} + args: + - '-c' + - |- + mkdir /etc/redis + + cp /config/redis.conf /etc/redis/redis.conf + + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + echo "\nrequirepass $CUSTOM_REDIS_PASSWORD" >> /etc/redis/redis.conf + echo "\nmasterauth $CUSTOM_REDIS_PASSWORD" >> /etc/redis/redis.conf + fi + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((6380 + POD_ID)) + echo "\nport $PORT" >> /etc/redis/redis.conf + echo "\nreplica-announce-ip {{ .Values.redis.publicAccess.address }}" >> /etc/redis/redis.conf + echo "\nreplica-announce-port $PORT" >> /etc/redis/redis.conf + {{ else }} + echo "\nport 6379" >> /etc/redis/redis.conf + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + LOCATION=${CPLN_LOCATION##*/} + CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" + if [ -n "$REPLICA_DIRECT" ]; then + echo "\nreplica-announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/redis/redis.conf + else + echo "\nreplica-announce-ip ${HOSTNAME}.{{ include "redis.name" . }}" >> /etc/redis/redis.conf + fi + echo "\nreplica-announce-port 6379" >> /etc/redis/redis.conf + {{ end }} + + if [ "$(hostname)" = "{{ include "redis.name" . }}-0" ]; then + {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} + else + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + SENTINEL_HOST="{{ include "redis.sentinel.name" . }}-0.{{ include "redis.sentinel.name" . }}" + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$MASTER_INFO" | head -1) + MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) + if [ -z "$MASTER_HOST" ]; then + MASTER_HOST="{{ .Values.redis.publicAccess.address }}" + MASTER_PORT=6380 + fi + {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT + {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} + SENTINEL_HOST="replica-0.{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$MASTER_INFO" | head -1) + MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) + if [ -z "$MASTER_HOST" ]; then + MASTER_HOST="replica-0.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" + MASTER_PORT=6379 + fi + {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT + {{- else }} + SENTINEL_HOST="{{ include "redis.sentinel.name" . }}-0.{{ include "redis.sentinel.name" . }}" + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$MASTER_INFO" | head -1) + MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) + if [ -z "$MASTER_HOST" ]; then + MASTER_HOST="{{ include "redis.name" . }}-0.{{ include "redis.name" . }}" + MASTER_PORT=6379 + fi + {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT + {{- end }} + fi + command: /bin/sh + cpu: {{ .Values.redis.resources.cpu }} + memory: {{ .Values.redis.resources.memory }} + minCpu: {{ .Values.redis.resources.minCpu }} + minMemory: {{ .Values.redis.resources.minMemory }} + image: {{ .Values.redis.image }} + readinessProbe: + exec: + command: + - /bin/bash + - "-c" + - |- + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((6380 + POD_ID)) + {{- else }} + PORT=6379 + {{- end }} + if [ ! -z "$CUSTOM_REDIS_PASSWORD" ]; then + redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping; + else + redis-cli -p $PORT ping; + fi + failureThreshold: 10 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 4 + inheritEnv: false + ports: +{{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled (gt (.Values.redis.replicas | int) 0) }} + {{- $startPort := 6380 }} + {{- $replicas := $.Values.redis.replicas | int }} + {{- range $replicaIndex := until $replicas }} + - number: {{ add $startPort $replicaIndex }} + protocol: tcp + {{- end }} +{{- else }} + - number: 6379 + protocol: tcp +{{- end }} + volumes: + - path: /config/redis.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "redis.secretConfig.name" . }} + {{- if and (hasKey .Values.redis "persistence") .Values.redis.persistence.enabled }} + - path: {{ .Values.redis.dataDir }} + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "redis.volume.name" . }} + {{- end }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + metric: disabled + minScale: {{ .Values.redis.replicas }} + maxScale: {{ .Values.redis.replicas }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: {{ .Values.redis.timeoutSeconds }} + {{- if .Values.redis.multiZone }} + multiZone: + enabled: true + {{- else }} + multiZone: + enabled: false + {{- end }} +{{- if .Values.redis.firewall }} + firewallConfig: + {{- if or (hasKey .Values.redis.firewall "external_inboundAllowCIDR") (hasKey .Values.redis.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.redis.firewall.external_inboundAllowCIDR }}{{ .Values.redis.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.redis.firewall.external_outboundAllowCIDR }}{{ .Values.redis.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.redis.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.redis.firewall.internal_inboundAllowType }} + {{- if .Values.redis.firewall.inboundAllowWorkload }} + inboundAllowWorkload: {{ .Values.redis.firewall.inboundAllowWorkload | toYaml | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "redis.identity.name" . }} +{{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + loadBalancer: + replicaDirect: true +{{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} + loadBalancer: + replicaDirect: true +{{- else }} + loadBalancer: + replicaDirect: false +{{- end }} diff --git a/redis/versions/3.3.0/templates/workload-sentinel.yaml b/redis/versions/3.3.0/templates/workload-sentinel.yaml new file mode 100644 index 00000000..6ef60f7c --- /dev/null +++ b/redis/versions/3.3.0/templates/workload-sentinel.yaml @@ -0,0 +1,179 @@ +{{ include "validateAuth" (dict "auth" .Values.sentinel.auth) }} +kind: workload +name: {{ include "redis.sentinel.name" . }} +description: Redis Sentinel +tags: + {{- if .Values.sentinel.tags }} +{{ toYaml .Values.sentinel.tags | indent 2 }} + {{- end }} + {{- include "redis.tags" . | nindent 2 }} +spec: + type: stateful + containers: + - name: sentinel + args: + - '-c' + - |- + {{- if and (hasKey .Values.sentinel "persistence") .Values.sentinel.persistence.enabled }} + mkdir -p /etc/sentinel/data + {{- else }} + mkdir -p /etc/sentinel + {{- end }} + cp /config/sentinel.conf /etc/sentinel/sentinel.conf + + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + echo "\nsentinel auth-pass mymaster $CUSTOM_REDIS_PASSWORD" >> /etc/sentinel/sentinel.conf + fi + + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + echo "\nrequirepass $CUSTOM_SENTINEL_PASSWORD" >> /etc/sentinel/sentinel.conf + fi + + {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((26380 + POD_ID)) + echo "\nport $PORT" >> /etc/sentinel/sentinel.conf + echo "\nsentinel announce-ip {{ .Values.sentinel.publicAccess.address }}" >> /etc/sentinel/sentinel.conf + echo "\nsentinel announce-port $PORT" >> /etc/sentinel/sentinel.conf + {{ else }} + echo "\nport 26379" >> /etc/sentinel/sentinel.conf + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + LOCATION=${CPLN_LOCATION##*/} + CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" + if [ -n "$REPLICA_DIRECT" ]; then + echo "\nsentinel announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/sentinel/sentinel.conf + else + echo "\nsentinel announce-ip ${HOSTNAME}.{{ include "redis.sentinel.name" . }}" >> /etc/sentinel/sentinel.conf + fi + echo "\nsentinel announce-port 26379" >> /etc/sentinel/sentinel.conf + {{ end }} + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + echo "sentinel monitor mymaster {{ .Values.redis.publicAccess.address }} 6380 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} + echo "sentinel monitor mymaster replica-0.{{ include "redis.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- else }} + echo "sentinel monitor mymaster {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- end }} + + redis-sentinel /etc/sentinel/sentinel.conf + command: /bin/sh + cpu: {{ .Values.sentinel.resources.cpu }} + memory: {{ .Values.sentinel.resources.memory }} + minCpu: {{ .Values.sentinel.resources.minCpu }} + minMemory: {{ .Values.sentinel.resources.minMemory }} + env: + - name: REDIS_SENTINEL_QUORUM + value: '{{ if .Values.sentinel.quorumAutoCalculation }}{{ add (div (int .Values.sentinel.replicas) 2) 1 }}{{ else }}{{ .Values.sentinel.quorumOverride }}{{ end }}' + - name: REDIS_SENTINEL_DATA_DIR + value: /etc/sentinel/data + {{- if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled }} + - name: CUSTOM_REDIS_PASSWORD + value: cpln://secret/{{ .Values.redis.auth.fromSecret.name }}.{{ .Values.redis.auth.fromSecret.passwordKey }} + {{- else if and (hasKey .Values.redis "auth") (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled }} + - name: CUSTOM_REDIS_PASSWORD + value: cpln://secret/{{ include "redis.secretPassword.name" . }}.password + {{- end }} + {{- if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled }} + - name: CUSTOM_SENTINEL_PASSWORD + value: cpln://secret/{{ .Values.sentinel.auth.fromSecret.name }}.{{ .Values.sentinel.auth.fromSecret.passwordKey }} + {{- else if and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled }} + - name: CUSTOM_SENTINEL_PASSWORD + value: cpln://secret/{{ include "redis.sentinelSecretPassword.name" . }}.password + {{- end }} + {{- if and (hasKey .Values.sentinel "replicaDirect") .Values.sentinel.replicaDirect }} + - name: REPLICA_DIRECT + value: "true" + {{- end }} + {{- if .Values.sentinel.env }} +{{ toYaml .Values.sentinel.env | indent 8 }} + {{- end }} + image: {{ .Values.sentinel.image }} + readinessProbe: + exec: + command: + - /bin/bash + - "-c" + - |- + {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((26380 + POD_ID)) + {{- else }} + PORT=26379 + {{- end }} + if [ ! -z "$CUSTOM_SENTINEL_PASSWORD" ]; then + redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" ping; + else + redis-cli -p $PORT ping; + fi + failureThreshold: 10 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 4 + inheritEnv: false + ports: +{{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled (gt (.Values.sentinel.replicas | int) 0) }} + {{- $startPort := 26380 }} + {{- $replicas := $.Values.sentinel.replicas | int }} + {{- range $replicaIndex := until $replicas }} + - number: {{ add $startPort $replicaIndex }} + protocol: tcp + {{- end }} +{{- else }} + - number: 26379 + protocol: tcp +{{- end }} + volumes: + - path: /config/sentinel.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "redis.sentinelSecretConfig.name" . }} + {{- if and (hasKey .Values.sentinel "persistence") .Values.sentinel.persistence.enabled }} + - path: /etc/sentinel + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "redis.sentinelVolume.name" . }} + {{- end }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + minScale: {{ .Values.sentinel.replicas }} + maxScale: {{ .Values.sentinel.replicas }} + metric: disabled + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: {{ .Values.sentinel.timeoutSeconds }} + {{- if .Values.sentinel.multiZone }} + multiZone: + enabled: true + {{- else }} + multiZone: + enabled: false + {{- end }} +{{- if .Values.sentinel.firewall }} + firewallConfig: + {{- if or (hasKey .Values.sentinel.firewall "external_inboundAllowCIDR") (hasKey .Values.sentinel.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.sentinel.firewall.external_inboundAllowCIDR }}{{ .Values.sentinel.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.sentinel.firewall.external_outboundAllowCIDR }}{{ .Values.sentinel.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.sentinel.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.sentinel.firewall.internal_inboundAllowType }} + {{- if .Values.sentinel.firewall.inboundAllowWorkload }} + inboundAllowWorkload: {{ .Values.sentinel.firewall.inboundAllowWorkload | toYaml | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "redis.sentinelIdentity.name" . }} +{{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + loadBalancer: + replicaDirect: true +{{- else if and (hasKey .Values.sentinel "replicaDirect") .Values.sentinel.replicaDirect }} + loadBalancer: + replicaDirect: true +{{- else }} + loadBalancer: + replicaDirect: false +{{- end }} diff --git a/redis/versions/3.3.0/values.yaml b/redis/versions/3.3.0/values.yaml new file mode 100644 index 00000000..480f054e --- /dev/null +++ b/redis/versions/3.3.0/values.yaml @@ -0,0 +1,129 @@ +redis: + image: redis:7.4 + resources: + cpu: 200m + memory: 256Mi + minCpu: 80m + minMemory: 128Mi + replicas: 2 + timeoutSeconds: 15 + multiZone: false + replicaDirect: false # https://docs.controlplane.com/reference/workload/general#internal-endpoint-formatting + auth: + fromSecret: + enabled: false + name: example-redis-auth-password + passwordKey: password + password: + enabled: false + value: your-password + serverCommand: redis-server # Can be overridden based on the version of redis image + # extraArgs: "--maxclients 20000 --maxmemory 200mb --maxmemory-policy allkeys-lru" + publicAccess: + enabled: false + address: redis-test.example-cpln.com + firewall: + internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc(Recommended) / workload-list + # external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list + # # You can specify additional workloads with either same-gvc or workload-list: + # inboundAllowWorkload: + # - //gvc/main-redis/workload/main-redis-sentinel + # - //gvc/client-gvc/workload/client + # external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list + env: [] + tags: {} + dataDir: /data + persistence: + enabled: false + volumes: + data: + initialCapacity: 10 # In GB + performanceClass: general-purpose-ssd # general-purpose-ssd / high-throughput-ssd (Min 1000GB) + fileSystemType: ext4 # ext4 / xfs + snapshots: + retentionDuration: 7d + schedule: 0 0 * * * # UTC + autoscaling: + maxCapacity: 100 # In GB + minFreePercentage: 20 + scalingFactor: 1.2 + # customEncryption: + # enabled: true + # region: aws-us-east-1 # Replace with the appropriate region + # keyId: arn:aws:kms:us-east-1:1234567890:key/d411f35a-1d31-4515-9934-4f193e042d80 # Replace with your AWS KMS key ARN + +sentinel: + image: redis:7.4 + resources: + cpu: 200m + memory: 256Mi + minCpu: 80m + minMemory: 128Mi + replicas: 3 + timeoutSeconds: 10 + multiZone: false + replicaDirect: false # https://docs.controlplane.com/reference/workload/general#internal-endpoint-formatting + quorumAutoCalculation: true # Set to false if you want to override quorum. Quorum is (replicas/2)+1 + quorumOverride: null # Only used if quorumAutoCalculation is false + auth: + fromSecret: + enabled: false + name: example-redis-auth-password + passwordKey: password + password: + enabled: false + value: your-password + publicAccess: + enabled: false + address: redis-sentinel-test.example-cpln.com + firewall: + internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc(Recommended) + # external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list + # # You can specify additional workloads with either same-gvc or workload-list: + # inboundAllowWorkload: + # - //gvc/main-redis/workload/main-redis-sentinel + # - //gvc/client-gvc/workload/client + # external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list + env: [] + tags: {} + persistence: + enabled: false + volumes: + data: + initialCapacity: 10 # In GB + performanceClass: general-purpose-ssd # general-purpose-ssd / high-throughput-ssd (Min 1000GB) + fileSystemType: ext4 # ext4 / xfs + snapshots: + retentionDuration: 7d + schedule: 0 0 * * * # UTC + autoscaling: + maxCapacity: 50 # In GB + minFreePercentage: 20 + scalingFactor: 1.2 + # customEncryption: + # enabled: true + # region: aws-us-east-1 # Replace with the appropriate region + # keyId: arn:aws:kms:us-east-1:1234567890:key/d411f35a-1d31-4515-9934-4f193e042d80 # Replace with your AWS KMS key ARN + +backup: + enabled: false + image: controlplanecorporation/redis-backup:1.0 + schedule: "0 2 * * *" # daily at 2am UTC + + resources: + cpu: 100m + memory: 128Mi + + provider: aws # Options: aws or gcp + + aws: + bucket: my-backup-bucket + region: us-east-1 + cloudAccountName: my-backup-cloudaccount + policyName: my-backup-policy + prefix: redis/backups # folder name where your backups will be stored + + gcp: + bucket: my-backup-bucket + cloudAccountName: my-backup-cloudaccount + prefix: redis/backups # folder name where your backups will be stored From 7e7e0d1880112b000d6cd2fed6f3b28fac6c69d7 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 23 Apr 2026 15:34:25 -0700 Subject: [PATCH 04/58] fixed probes --- .../versions/1.0.0/templates/workload-cassandra.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index cbc054fa..27359329 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -26,18 +26,18 @@ spec: - number: 7000 protocol: tcp livenessProbe: - failureThreshold: 10 - initialDelaySeconds: 120 + failureThreshold: 5 + initialDelaySeconds: 60 periodSeconds: 30 successThreshold: 1 tcpSocket: port: 9042 timeoutSeconds: 10 readinessProbe: - failureThreshold: 20 - initialDelaySeconds: 60 - periodSeconds: 15 - successThreshold: 3 + failureThreshold: 10 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 tcpSocket: port: 9042 timeoutSeconds: 5 From 4e80a3d46fc31a1f62524c499a0418e6f7d88154 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 23 Apr 2026 16:36:49 -0700 Subject: [PATCH 05/58] added tagging names, cpln-common tags --- cassandra/versions/1.0.0/Chart.yaml | 7 +- .../versions/1.0.0/templates/_helpers.tpl | 70 +++++++++++++++++-- .../versions/1.0.0/templates/identity.yaml | 5 +- .../versions/1.0.0/templates/policy.yaml | 8 +-- .../1.0.0/templates/secret-config.yaml | 2 +- .../versions/1.0.0/templates/secret-init.yaml | 28 ++++++-- .../versions/1.0.0/templates/volumeset.yaml | 5 +- .../1.0.0/templates/workload-cassandra.yaml | 15 ++-- 8 files changed, 113 insertions(+), 27 deletions(-) diff --git a/cassandra/versions/1.0.0/Chart.yaml b/cassandra/versions/1.0.0/Chart.yaml index 645755d5..8ba87801 100644 --- a/cassandra/versions/1.0.0/Chart.yaml +++ b/cassandra/versions/1.0.0/Chart.yaml @@ -7,6 +7,11 @@ appVersion: "5.0" annotations: created: "2026-04-21" - lastModified: "2026-04-21" + lastModified: "2026-04-23" category: "database" createsGvc: false + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" diff --git a/cassandra/versions/1.0.0/templates/_helpers.tpl b/cassandra/versions/1.0.0/templates/_helpers.tpl index 907a9e46..9a943472 100644 --- a/cassandra/versions/1.0.0/templates/_helpers.tpl +++ b/cassandra/versions/1.0.0/templates/_helpers.tpl @@ -1,9 +1,69 @@ -{{- define "cassandra.name" -}} -{{- default "cassandra" .Values.global.cpln.workloadName | trunc 63 | trimSuffix "-" }} +{{/* Resource Naming */}} + +{{/* +Cassandra Workload Name +*/}} +{{- define "cassandra.workload.name" -}} +{{- printf "%s-cassandra" .Release.Name }} {{- end }} -{{- define "cassandra.tags" -}} -{{- if .Values.global.cpln.tags }} -{{- toYaml .Values.global.cpln.tags }} +{{/* +Cassandra Init Script Secret Name +*/}} +{{- define "cassandra.secret.init.name" -}} +{{- printf "%s-cassandra-init" .Release.Name }} +{{- end }} + +{{/* +Cassandra Config Secret Name +*/}} +{{- define "cassandra.secret.config.name" -}} +{{- printf "%s-cassandra-config" .Release.Name }} +{{- end }} + +{{/* +Cassandra Identity Name +*/}} +{{- define "cassandra.identity.name" -}} +{{- printf "%s-cassandra-identity" .Release.Name }} +{{- end }} + +{{/* +Cassandra Policy Name +*/}} +{{- define "cassandra.policy.name" -}} +{{- printf "%s-cassandra-policy" .Release.Name }} +{{- end }} + +{{/* +Cassandra VolumeSet Name +*/}} +{{- define "cassandra.volumeset.name" -}} +{{- printf "%s-cassandra-data" .Release.Name }} +{{- end }} + + +{{/* Validation */}} + +{{/* +Validate that replicas is an odd number >= 3 +*/}} +{{- define "cassandra.validateReplicas" -}} +{{- $replicas := .Values.cassandra.replicas | int -}} +{{- if lt $replicas 3 -}} + {{- fail "cassandra.replicas must be at least 3." -}} +{{- end -}} +{{- if eq (mod $replicas 2) 0 -}} + {{- fail "cassandra.replicas must be an odd number (3, 5, 7, ...) for quorum." -}} +{{- end -}} {{- end }} + + +{{/* Labeling */}} + +{{/* +Common tags - delegated to cpln-common +*/}} +{{- define "cassandra.tags" -}} +{{- include "cpln-common.tags" . }} {{- end }} diff --git a/cassandra/versions/1.0.0/templates/identity.yaml b/cassandra/versions/1.0.0/templates/identity.yaml index 12e99946..d0086cab 100644 --- a/cassandra/versions/1.0.0/templates/identity.yaml +++ b/cassandra/versions/1.0.0/templates/identity.yaml @@ -1,4 +1,5 @@ kind: identity -name: {{ include "cassandra.name" . }} -description: {{ include "cassandra.name" . }} identity +name: {{ include "cassandra.identity.name" . }} +description: {{ include "cassandra.workload.name" . }} identity gvc: {{ .Values.global.cpln.gvc }} +tags: {{- include "cassandra.tags" . | nindent 2 }} diff --git a/cassandra/versions/1.0.0/templates/policy.yaml b/cassandra/versions/1.0.0/templates/policy.yaml index ef291ec6..51bfc516 100644 --- a/cassandra/versions/1.0.0/templates/policy.yaml +++ b/cassandra/versions/1.0.0/templates/policy.yaml @@ -1,12 +1,12 @@ kind: policy -name: {{ include "cassandra.name" . }} +name: {{ include "cassandra.policy.name" . }} origin: default bindings: - permissions: - reveal principalLinks: - - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.name" . }} + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} targetKind: secret targetLinks: - - //secret/{{ include "cassandra.name" . }}-init - - //secret/{{ include "cassandra.name" . }}-config + - //secret/{{ include "cassandra.secret.init.name" . }} + - //secret/{{ include "cassandra.secret.config.name" . }} diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml index be952d11..0d86188e 100644 --- a/cassandra/versions/1.0.0/templates/secret-config.yaml +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -1,5 +1,5 @@ kind: secret -name: {{ include "cassandra.name" . }}-config +name: {{ include "cassandra.secret.config.name" . }} type: opaque data: encoding: plain diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 8231543f..42db61ae 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -1,5 +1,6 @@ +{{ include "cassandra.validateReplicas" . -}} kind: secret -name: {{ include "cassandra.name" . }}-init +name: {{ include "cassandra.secret.init.name" . }} type: opaque data: encoding: plain @@ -53,11 +54,28 @@ data: sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml echo "cassandra.yaml written. Starting Cassandra..." - # Drop from root to the cassandra user (uid 999) using gosu, which is bundled - # in the official Cassandra image. Cassandra refuses to start as root without -R. + + # Graceful shutdown handler: drain the node before exit so gossip broadcasts + # DOWN immediately, preventing other nodes from retrying a dead IP for minutes. + # nodetool decommission is intentionally NOT used here — that redistributes + # tokens and permanently removes the node, which is wrong for a rolling restart. + shutdown_handler() { + echo "SIGTERM received. Draining Cassandra node before shutdown..." + nodetool drain 2>/dev/null || true + echo "Drain complete. Stopping Cassandra..." + kill -TERM "$CASS_PID" 2>/dev/null || true + wait "$CASS_PID" 2>/dev/null || true + exit 0 + } + trap shutdown_handler SIGTERM SIGINT + + # Run Cassandra in the background so the trap above can fire. + # (exec would replace the shell process and lose the signal handler.) if [ "$(id -u)" = "0" ]; then chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true - exec gosu cassandra cassandra -f + gosu cassandra cassandra -f & else - exec cassandra -f + cassandra -f & fi + CASS_PID=$! + wait "$CASS_PID" diff --git a/cassandra/versions/1.0.0/templates/volumeset.yaml b/cassandra/versions/1.0.0/templates/volumeset.yaml index b48ee13d..45188205 100644 --- a/cassandra/versions/1.0.0/templates/volumeset.yaml +++ b/cassandra/versions/1.0.0/templates/volumeset.yaml @@ -1,7 +1,8 @@ kind: volumeset -name: {{ include "cassandra.name" . }}-data -description: {{ include "cassandra.name" . }} data +name: {{ include "cassandra.volumeset.name" . }} +description: {{ include "cassandra.workload.name" . }} data gvc: {{ .Values.global.cpln.gvc }} +tags: {{- include "cassandra.tags" . | nindent 2 }} spec: initialCapacity: {{ .Values.cassandra.volumes.data.initialCapacity }} performanceClass: {{ .Values.cassandra.volumes.data.performanceClass }} diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 27359329..577f97ca 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -1,7 +1,8 @@ kind: workload -name: {{ include "cassandra.name" . }} +name: {{ include "cassandra.workload.name" . }} description: Cassandra cluster gvc: {{ .Values.global.cpln.gvc }} +tags: {{- include "cassandra.tags" . | nindent 2 }} spec: type: stateful containers: @@ -27,7 +28,7 @@ spec: protocol: tcp livenessProbe: failureThreshold: 5 - initialDelaySeconds: 60 + initialDelaySeconds: 120 periodSeconds: 30 successThreshold: 1 tcpSocket: @@ -44,13 +45,13 @@ spec: volumes: - path: /var/lib/cassandra recoveryPolicy: retain - uri: 'cpln://volumeset/{{ include "cassandra.name" . }}-data' + uri: 'cpln://volumeset/{{ include "cassandra.volumeset.name" . }}' - path: /config-template/cassandra-template.yaml recoveryPolicy: retain - uri: 'cpln://secret/{{ include "cassandra.name" . }}-config' + uri: 'cpln://secret/{{ include "cassandra.secret.config.name" . }}' - path: /scripts/cassandra-init.sh recoveryPolicy: retain - uri: 'cpln://secret/{{ include "cassandra.name" . }}-init' + uri: 'cpln://secret/{{ include "cassandra.secret.init.name" . }}' defaultOptions: autoscaling: maxConcurrency: 0 @@ -62,7 +63,7 @@ spec: capacityAI: false debug: false suspend: {{ .Values.cassandra.suspend }} - timeoutSeconds: 30 + timeoutSeconds: 90 firewallConfig: external: outboundAllowCIDR: @@ -74,7 +75,7 @@ spec: enabled: false ports: [] replicaDirect: true - identityLink: //identity/{{ include "cassandra.name" . }} + identityLink: //identity/{{ include "cassandra.identity.name" . }} rolloutOptions: maxSurgeReplicas: 0% maxUnavailableReplicas: '1' From c99d21cb643822d3eed2c72b47d2d7ab0a973797 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 23 Apr 2026 17:04:04 -0700 Subject: [PATCH 06/58] added post stop hook to handle shut down --- .../1.0.0/charts/cpln-common-1.0.0.tgz | Bin 0 -> 680 bytes .../versions/1.0.0/templates/secret-init.yaml | 25 +++--------------- .../1.0.0/templates/workload-cassandra.yaml | 9 ++++++- 3 files changed, 12 insertions(+), 22 deletions(-) create mode 100644 cassandra/versions/1.0.0/charts/cpln-common-1.0.0.tgz diff --git a/cassandra/versions/1.0.0/charts/cpln-common-1.0.0.tgz b/cassandra/versions/1.0.0/charts/cpln-common-1.0.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..c41e09051d41e5bd4feeabac39714835119f385c GIT binary patch literal 680 zcmV;Z0$2SXiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PIyykJCO7?Kz)fWN9z^Q(X67eo}6z2RHyiRNCvxO(uzjV+Y$Q zRNejV$oW&UX_rM}#RBSF;yBNq_g*~Xq>?I3bXjUOT^HOqy62^wJZmtgGsqpLF_1Z%M{COg4Xm|tMoai0xk4zp`Gl^LJV9_+R?-s)6fZG`!ATC> zzQXSkc74(rWt0UkZTTM+!}>ouI=wAWR%k4B)id;hC+dF|MaxnBN9=b9wq%YG-x(C970Gj6AFP0 z&KPZw1i}td8KGPTHP5^TyLQ@p;TxH0@c6tp6Ras;d7ZBdy0S z>!qH6@8EN--B|W;eaRKiF%7r-hi+}q>_pP6^w8/dev/null || true - echo "Drain complete. Stopping Cassandra..." - kill -TERM "$CASS_PID" 2>/dev/null || true - wait "$CASS_PID" 2>/dev/null || true - exit 0 - } - trap shutdown_handler SIGTERM SIGINT - - # Run Cassandra in the background so the trap above can fire. - # (exec would replace the shell process and lose the signal handler.) + # nodetool drain is run via the workload preStop hook before SIGTERM reaches + # this process, so exec here is safe — Cassandra stays PID 1. if [ "$(id -u)" = "0" ]; then chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true - gosu cassandra cassandra -f & + exec gosu cassandra cassandra -f else - cassandra -f & + exec cassandra -f fi - CASS_PID=$! - wait "$CASS_PID" diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 577f97ca..5360bd6a 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -17,6 +17,13 @@ spec: env: - name: MAX_HEAP_SIZE value: {{ .Values.cassandra.jvmHeapSize | quote }} + lifecycle: + preStop: + exec: + command: + - bash + - '-c' + - nodetool drain image: {{ .Values.cassandra.image }} cpu: '{{ .Values.cassandra.cpu }}' memory: {{ .Values.cassandra.memory }} @@ -63,7 +70,7 @@ spec: capacityAI: false debug: false suspend: {{ .Values.cassandra.suspend }} - timeoutSeconds: 90 + timeoutSeconds: 60 firewallConfig: external: outboundAllowCIDR: From d702dd65b86ca733c839a80cb3bbf2fab0f0067a Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 23 Apr 2026 17:28:40 -0700 Subject: [PATCH 07/58] added multi location support --- cassandra/versions/1.0.0/Chart.yaml | 2 +- .../1.0.0/charts/cpln-common-1.0.0.tgz | Bin 680 -> 0 bytes .../versions/1.0.0/templates/_helpers.tpl | 45 +++++++----------- cassandra/versions/1.0.0/templates/gvc.yaml | 11 +++++ .../versions/1.0.0/templates/identity.yaml | 2 +- .../versions/1.0.0/templates/policy.yaml | 2 +- .../1.0.0/templates/secret-config.yaml | 4 +- .../versions/1.0.0/templates/secret-init.yaml | 41 ++++++++++++---- .../versions/1.0.0/templates/volumeset.yaml | 2 +- .../1.0.0/templates/workload-cassandra.yaml | 33 +++++++++++-- cassandra/versions/1.0.0/values.yaml | 18 +++++-- 11 files changed, 108 insertions(+), 52 deletions(-) delete mode 100644 cassandra/versions/1.0.0/charts/cpln-common-1.0.0.tgz create mode 100644 cassandra/versions/1.0.0/templates/gvc.yaml diff --git a/cassandra/versions/1.0.0/Chart.yaml b/cassandra/versions/1.0.0/Chart.yaml index 8ba87801..b4b5fc7a 100644 --- a/cassandra/versions/1.0.0/Chart.yaml +++ b/cassandra/versions/1.0.0/Chart.yaml @@ -9,7 +9,7 @@ annotations: created: "2026-04-21" lastModified: "2026-04-23" category: "database" - createsGvc: false + createsGvc: true dependencies: - name: cpln-common diff --git a/cassandra/versions/1.0.0/charts/cpln-common-1.0.0.tgz b/cassandra/versions/1.0.0/charts/cpln-common-1.0.0.tgz deleted file mode 100644 index c41e09051d41e5bd4feeabac39714835119f385c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 680 zcmV;Z0$2SXiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PIyykJCO7?Kz)fWN9z^Q(X67eo}6z2RHyiRNCvxO(uzjV+Y$Q zRNejV$oW&UX_rM}#RBSF;yBNq_g*~Xq>?I3bXjUOT^HOqy62^wJZmtgGsqpLF_1Z%M{COg4Xm|tMoai0xk4zp`Gl^LJV9_+R?-s)6fZG`!ATC> zzQXSkc74(rWt0UkZTTM+!}>ouI=wAWR%k4B)id;hC+dF|MaxnBN9=b9wq%YG-x(C970Gj6AFP0 z&KPZw1i}td8KGPTHP5^TyLQ@p;TxH0@c6tp6Ras;d7ZBdy0S z>!qH6@8EN--B|W;eaRKiF%7r-hi+}q>_pP6^w8= 3 +Validate locations: requires at least 1. If more than 1, requires at least 3. +Each location must have odd replicas >= 3. */}} -{{- define "cassandra.validateReplicas" -}} -{{- $replicas := .Values.cassandra.replicas | int -}} -{{- if lt $replicas 3 -}} - {{- fail "cassandra.replicas must be at least 3." -}} +{{- define "cassandra.validateLocations" -}} +{{- $locations := .Values.gvc.locations -}} +{{- if lt (len $locations) 1 -}} + {{- fail "gvc.locations must contain at least one location." -}} {{- end -}} -{{- if eq (mod $replicas 2) 0 -}} - {{- fail "cassandra.replicas must be an odd number (3, 5, 7, ...) for quorum." -}} +{{- if and (gt (len $locations) 1) (lt (len $locations) 3) -}} + {{- fail "Multi-location Cassandra requires at least 3 locations for cross-DC quorum." -}} +{{- end -}} +{{- range $locations -}} + {{- $r := .replicas | int -}} + {{- if lt $r 3 -}} + {{- fail (printf "Location %s: replicas must be at least 3." .name) -}} + {{- end -}} + {{- if eq (mod $r 2) 0 -}} + {{- fail (printf "Location %s: replicas must be an odd number (3, 5, 7, ...) for quorum." .name) -}} + {{- end -}} {{- end -}} {{- end }} {{/* Labeling */}} -{{/* -Common tags - delegated to cpln-common -*/}} {{- define "cassandra.tags" -}} {{- include "cpln-common.tags" . }} {{- end }} diff --git a/cassandra/versions/1.0.0/templates/gvc.yaml b/cassandra/versions/1.0.0/templates/gvc.yaml new file mode 100644 index 00000000..744088fb --- /dev/null +++ b/cassandra/versions/1.0.0/templates/gvc.yaml @@ -0,0 +1,11 @@ +kind: gvc +name: {{ .Values.gvc.name }} +description: {{ .Values.gvc.name }} +tags: {{- include "cassandra.tags" . | nindent 2 }} +spec: + endpointNamingFormat: org + staticPlacement: + locationLinks: + {{- range .Values.gvc.locations }} + - //location/{{ .name }} + {{- end }} diff --git a/cassandra/versions/1.0.0/templates/identity.yaml b/cassandra/versions/1.0.0/templates/identity.yaml index d0086cab..9513b667 100644 --- a/cassandra/versions/1.0.0/templates/identity.yaml +++ b/cassandra/versions/1.0.0/templates/identity.yaml @@ -1,5 +1,5 @@ kind: identity name: {{ include "cassandra.identity.name" . }} description: {{ include "cassandra.workload.name" . }} identity -gvc: {{ .Values.global.cpln.gvc }} +gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} diff --git a/cassandra/versions/1.0.0/templates/policy.yaml b/cassandra/versions/1.0.0/templates/policy.yaml index 51bfc516..8b98159d 100644 --- a/cassandra/versions/1.0.0/templates/policy.yaml +++ b/cassandra/versions/1.0.0/templates/policy.yaml @@ -5,7 +5,7 @@ bindings: - permissions: - reveal principalLinks: - - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} + - //gvc/{{ .Values.gvc.name }}/identity/{{ include "cassandra.identity.name" . }} targetKind: secret targetLinks: - //secret/{{ include "cassandra.secret.init.name" . }} diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml index 0d86188e..b344f3ee 100644 --- a/cassandra/versions/1.0.0/templates/secret-config.yaml +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -8,9 +8,9 @@ data: # Networking — replaced at pod startup by the init script listen_address: LISTEN_ADDRESS_PLACEHOLDER - broadcast_address: LISTEN_ADDRESS_PLACEHOLDER + broadcast_address: BROADCAST_ADDRESS_PLACEHOLDER rpc_address: 0.0.0.0 - broadcast_rpc_address: LISTEN_ADDRESS_PLACEHOLDER + broadcast_rpc_address: BROADCAST_ADDRESS_PLACEHOLDER # Seeds — replaced at pod startup by the init script seed_provider: diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 1c1ea4d6..e3550db7 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -1,4 +1,4 @@ -{{ include "cassandra.validateReplicas" . -}} +{{ include "cassandra.validateLocations" . -}} kind: secret name: {{ include "cassandra.secret.init.name" . }} type: opaque @@ -8,7 +8,7 @@ data: #!/bin/bash set -euo pipefail - # Derive own FQDN from /etc/hosts. + # Derive own internal FQDN from /etc/hosts. # Control Plane inserts a line like: # 10.x.x.x cassandra-0.cassandra..svc.cluster.local cassandra-0 MY_FQDN=$(grep -E "^[0-9]" /etc/hosts | grep "${HOSTNAME}" | awk '{print $2}') @@ -24,34 +24,55 @@ data: SERVICE=$(echo "${MY_FQDN}" | cut -d'.' -f2) REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') + # Location and GVC from environment (injected by Control Plane / workload env). + LOCATION=$(basename "${CPLN_LOCATION}") + GVC="${CASSANDRA_GVC}" + WORKLOAD="${CASSANDRA_WORKLOAD}" + + # cpln.local address — reachable cross-location, used as broadcast_address. + MY_CPLN_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD}.${LOCATION}.${GVC}.cpln.local" + echo "HOSTNAME: ${HOSTNAME}" echo "MY_FQDN: ${MY_FQDN}" + echo "MY_CPLN_FQDN: ${MY_CPLN_FQDN}" echo "NAMESPACE_HASH: ${NAMESPACE_HASH}" echo "SERVICE: ${SERVICE}" echo "REPLICA_INDEX: ${REPLICA_INDEX}" + echo "LOCATION: ${LOCATION}" + echo "GVC: ${GVC}" - # Build seed list from replicas 0 and 1 only (avoids "only seed" warning and - # keeps seed count stable regardless of cluster size). + # Build seed list: 2 replicas from each location using cpln.local addresses. + # This ensures cross-DC gossip bootstrap and keeps seed count stable. SEED_COUNT=2 SEEDS="" - for i in $(seq 0 $((SEED_COUNT - 1))); do - SEED_FQDN="${SERVICE}-${i}.${SERVICE}.${NAMESPACE_HASH}.svc.cluster.local" - SEEDS="${SEEDS}${SEED_FQDN}," + LOCATIONS="{{ range .Values.gvc.locations }}{{ .name }} {{ end }}" + for loc in $LOCATIONS; do + for i in $(seq 0 $((SEED_COUNT - 1))); do + SEED_FQDN="replica-${i}.${WORKLOAD}.${loc}.${GVC}.cpln.local" + SEEDS="${SEEDS}${SEED_FQDN}," + done done SEEDS="${SEEDS%,}" echo "SEEDS: ${SEEDS}" - # Cassandra data dir ownership (cassandra runs as uid 999 in the official image) + # Cassandra data dir ownership (cassandra runs as uid 999 in the official image). mkdir -p /var/lib/cassandra chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true rm -rf /var/lib/cassandra/lost+found 2>/dev/null || true + # Write cassandra-rackdc.properties for DC/rack awareness. + # DC = location name (e.g. aws-us-east-2), rack = rack1. + mkdir -p /etc/cassandra + printf 'dc=%s\nrack=rack1\n' "${LOCATION}" > /etc/cassandra/cassandra-rackdc.properties + echo "cassandra-rackdc.properties written: dc=${LOCATION} rack=rack1" + # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) cp /config-template/cassandra-template.yaml /etc/cassandra/cassandra.yaml - sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml - sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml + sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml + sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${MY_CPLN_FQDN}|g" /etc/cassandra/cassandra.yaml + sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml echo "cassandra.yaml written. Starting Cassandra..." # nodetool drain is run via the workload preStop hook before SIGTERM reaches diff --git a/cassandra/versions/1.0.0/templates/volumeset.yaml b/cassandra/versions/1.0.0/templates/volumeset.yaml index 45188205..de173320 100644 --- a/cassandra/versions/1.0.0/templates/volumeset.yaml +++ b/cassandra/versions/1.0.0/templates/volumeset.yaml @@ -1,7 +1,7 @@ kind: volumeset name: {{ include "cassandra.volumeset.name" . }} description: {{ include "cassandra.workload.name" . }} data -gvc: {{ .Values.global.cpln.gvc }} +gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} spec: initialCapacity: {{ .Values.cassandra.volumes.data.initialCapacity }} diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 5360bd6a..2f3a67cd 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -1,7 +1,8 @@ +{{- include "cassandra.validateLocations" . -}} kind: workload name: {{ include "cassandra.workload.name" . }} description: Cassandra cluster -gvc: {{ .Values.global.cpln.gvc }} +gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} spec: type: stateful @@ -17,6 +18,10 @@ spec: env: - name: MAX_HEAP_SIZE value: {{ .Values.cassandra.jvmHeapSize | quote }} + - name: CASSANDRA_GVC + value: {{ .Values.gvc.name | quote }} + - name: CASSANDRA_WORKLOAD + value: {{ include "cassandra.workload.name" . | quote }} lifecycle: preStop: exec: @@ -62,9 +67,9 @@ spec: defaultOptions: autoscaling: maxConcurrency: 0 - maxScale: {{ .Values.cassandra.replicas }} + maxScale: {{ (index .Values.gvc.locations 0).replicas | int }} metric: disabled - minScale: {{ .Values.cassandra.replicas }} + minScale: {{ (index .Values.gvc.locations 0).replicas | int }} scaleToZeroDelay: 300 target: 95 capacityAI: false @@ -76,13 +81,31 @@ spec: outboundAllowCIDR: - 0.0.0.0/0 internal: - inboundAllowType: same-gvc + inboundAllowType: {{ .Values.internal_access.type }} + {{- if .Values.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + {{- end }} + identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "cassandra.identity.name" . }} loadBalancer: direct: enabled: false ports: [] replicaDirect: true - identityLink: //identity/{{ include "cassandra.identity.name" . }} + localOptions: + {{- range $location := .Values.gvc.locations }} + - autoscaling: + maxConcurrency: 0 + maxScale: {{ if eq ($location.replicas | int) 0 }}1{{ else }}{{ $location.replicas | int }}{{ end }} + metric: disabled + minScale: {{ if eq ($location.replicas | int) 0 }}0{{ else }}{{ $location.replicas | int }}{{ end }} + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + location: //location/{{ $location.name }} + suspend: {{ if eq ($location.replicas | int) 0 }}true{{ else }}false{{ end }} + timeoutSeconds: 60 + {{- end }} rolloutOptions: maxSurgeReplicas: 0% maxUnavailableReplicas: '1' diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index badaed62..f28d169d 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,8 +1,16 @@ +gvc: + name: cassandra-gvc + locations: + - name: aws-us-east-2 + replicas: 3 + - name: aws-us-west-2 + replicas: 3 + - name: aws-eu-central-1 + replicas: 3 + cassandra: - # Must match minScale/maxScale below. Use odd numbers: 3, 5, 7. - replicas: 3 image: cassandra:5.0 - cpu: 2 + cpu: '2' memory: 8Gi # JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) # Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. @@ -19,3 +27,7 @@ cassandra: maxCapacity: 100 minFreePercentage: 20 scalingFactor: 1.5 + +internal_access: + type: same-gvc + workloads: From 96be16646b937ec4101cd5013901f0331ac4c07c Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:38:13 -0700 Subject: [PATCH 08/58] improved ha proxy health check and patroni resiliancy with dcs (#245) * init 2.3.1 * switched ha proxy to use http over tcp check, added 2 health endpoints for ha proxy, increased dcs retry limit --- .../versions/2.3.1/Chart.yaml | 21 ++ .../versions/2.3.1/README.md | 282 ++++++++++++++++++ .../versions/2.3.1/templates/_helpers.tpl | 140 +++++++++ .../versions/2.3.1/templates/identity.yaml | 25 ++ .../versions/2.3.1/templates/policy.yaml | 20 ++ .../2.3.1/templates/secret-config.yaml | 17 ++ .../2.3.1/templates/secret-ha-proxy.yaml | 91 ++++++ .../2.3.1/templates/secret-startup.yaml | 128 ++++++++ .../2.3.1/templates/secret-wal-g-script.yaml | 39 +++ .../versions/2.3.1/templates/volumeset.yaml | 20 ++ .../2.3.1/templates/workload-ha-proxy.yaml | 56 ++++ .../templates/workload-logical-backup.yaml | 74 +++++ .../templates/workload-patroni-postgres.yaml | 148 +++++++++ .../2.3.1/templates/workload-pgbouncer.yaml | 67 +++++ .../versions/2.3.1/values.yaml | 96 ++++++ 15 files changed, 1224 insertions(+) create mode 100644 postgres-highly-available/versions/2.3.1/Chart.yaml create mode 100644 postgres-highly-available/versions/2.3.1/README.md create mode 100644 postgres-highly-available/versions/2.3.1/templates/_helpers.tpl create mode 100644 postgres-highly-available/versions/2.3.1/templates/identity.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/policy.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/secret-config.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/secret-startup.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/secret-wal-g-script.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/volumeset.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/workload-ha-proxy.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/workload-logical-backup.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml create mode 100644 postgres-highly-available/versions/2.3.1/templates/workload-pgbouncer.yaml create mode 100644 postgres-highly-available/versions/2.3.1/values.yaml diff --git a/postgres-highly-available/versions/2.3.1/Chart.yaml b/postgres-highly-available/versions/2.3.1/Chart.yaml new file mode 100644 index 00000000..ae05057a --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: postgres-highly-available +description: Multi-replica PostgreSQL with Patroni and etcd for Control Plane + +type: application +version: 2.3.1 +appVersion: "17" + +annotations: + created: "2025-10-02" + lastModified: "2026-04-24" + category: "database" + createsGvc: false + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + - name: etcd + version: 1.4.0 + repository: "oci://ghcr.io/controlplane-com/templates" diff --git a/postgres-highly-available/versions/2.3.1/README.md b/postgres-highly-available/versions/2.3.1/README.md new file mode 100644 index 00000000..4661f3a8 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/README.md @@ -0,0 +1,282 @@ +# PostgreSQL 17 Highly Available with Patroni + +This app deploys a highly available PostgreSQL 17 cluster using Patroni for automatic failover and etcd for distributed consensus. The setup delivers automatic leader election, health checking, and seamless failover capabilities in a single location with multi-zone capability and provides optional backup features. + +## Architecture + +- **PostgreSQL with Patroni**: Multi-replica PostgreSQL cluster managed by Patroni +- **etcd**: Distributed key-value store for consensus and configuration allowing high availability +- **HA Proxy** (optional): Leader-routing proxy that directs write traffic to the current primary replica +- **PgBouncer** (optional): Connection pooler that sits in front of HAProxy, multiplexing application connections into a smaller pool of real database connections +- **Backup**: (optional): Logical or native WAL-G backup + +## Configuration + +### PostgreSQL Settings + +Configure your PostgreSQL cluster in the values file: + +```yaml +replicas: 3 # Number of PostgreSQL replicas (minimum 3 recommended for HA) + +resources: + minCpu: 500m # Minimum CPU per replica + minMemory: 1Gi # Minimum memory per replica + maxCpu: 1 # Maximum CPU per replica + maxMemory: 2Gi # Maximum memory per replica + +postgres: + username: username # PostgreSQL username + password: password # PostgreSQL password + database: test # Auto created database name +``` + +**Volume** — set the initial storage capacity (minimum 10 GiB). Optionally enable autoscaling to expand the volume as data grows: + +```yaml +volumeset: + capacity: 10 + autoscaling: + enabled: true + maxCapacity: 100 + minFreePercentage: 10 + scalingFactor: 1.2 +``` + +Configure which workloads can access PostgreSQL: + +```yaml +internal_access: + type: same-gvc # Options: same-gvc, same-org, workload-list + workloads: + # Uncomment and specify workloads if using same-gvc or workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME +``` + +- `same-gvc`: Allow access from all workloads in the same GVC +- `same-org`: Allow access from all workloads in the org +- `workload-list`: Allow access only from specified workloads + +### etcd Configuration + +The embedded etcd cluster manages cluster state and consensus: + +```yaml +etcd: + replicas: 3 # Number of etcd replicas (must be odd number, minimum 3 for HA) + + resources: # resources specific to etcd + cpu: 500m + memory: 512Mi + + internal_access: # same behavior as postgres settings +``` + +### HA Proxy (Strongly Recommended) + +In a Patroni cluster, only the leader replica accepts writes, other replicas are read-only. The HA Proxy provides a stable endpoint that automatically routes traffic to the current leader, ensuring write operations always reach the correct replica. + +```yaml +proxy: + enabled: true # Enable leader-routing proxy + resources: + cpu: 100m + memory: 128Mi + minReplicas: 2 + maxReplicas: 2 +``` + +**Required for:** +- **External write access**: External clients must connect through the proxy to perform write operations +- **Backup feature**: The proxy must be enabled for logical backups to function correctly (WAL-G backups work internally - proxy not required) + +When enabled, connect to the proxy workload on port 5432 for write operations. + +### PgBouncer Connection Pooling (Optional) + +PgBouncer multiplexes application connections into a smaller pool of real database connections, reducing overhead and protecting Postgres from connection exhaustion under high concurrency. It sits in front of HAProxy so leader routing and failover are handled transparently. + +HAProxy is automatically enabled when PgBouncer is enabled, as it is required for leader-aware routing in the HA cluster. + +When enabled, PgBouncer becomes the primary connection endpoint. Connect to `{release-name}-pgbouncer.{gvc}.cpln.local:5432` instead of the proxy workload directly. + +```yaml +pgbouncer: + enabled: true + poolMode: transaction # options: session, transaction, statement + defaultPoolSize: 25 # real Postgres connections per PgBouncer pod + maxClientConn: 1000 # max app connections per PgBouncer pod + maxDbConnections: 100 # hard cap on total Postgres connections regardless of how many PgBouncer pods are running + minReplicas: 2 + maxReplicas: 4 +``` + +**Pool modes:** +- `transaction` — connection held only for the duration of a transaction. Best for most web and API workloads. Not compatible with session-level features like `SET` variables, temporary tables, or advisory locks. +- `session` — connection held for the entire client session. Compatible with all Postgres features but provides less connection reuse. Increase `defaultPoolSize` to match your expected concurrent client count. +- `statement` — connection returned after every statement. Transactions are not supported. Rarely used. + +**`maxDbConnections`** is a hard cap on the total number of real Postgres connections PgBouncer will open, shared across all PgBouncer pods. This prevents connection blowout when PgBouncer scales up — set it to a value your Postgres primary can safely handle. + +**Scaling:** PgBouncer autoscales on RPS between `minReplicas` and `maxReplicas`. Increase `maxReplicas` for high-throughput workloads where PgBouncer becomes the bottleneck before Postgres does. + +## Connecting to PostgreSQL + +Connect to the PostgreSQL cluster using the appropriate endpoint: + +| Setup | Host | +|---|---| +| PgBouncer enabled | `{release-name}-pgbouncer.{gvc}.cpln.local` | +| Proxy only | `{release-name}-postgres-ha-proxy.{gvc}.cpln.local` | + +``` +Port: 5432 +Database: {postgres.database} +Username: {postgres.username} +Password: {postgres.password} +``` + +## Important Notes + +- **Minimum Replicas**: For production use, maintain at least 3 PostgreSQL replicas and 3 etcd replicas +- **Odd Number for etcd**: Always use an odd number of etcd replicas (3, 5, 7) for proper quorum +- **Resource Allocation**: Ensure adequate CPU and memory resources for both PostgreSQL and etcd workloads +- **Multi-zone**: Verify your selected location supports multi-zone + +## Backing Up + +There are two backup options: +- **Logical backups** create portable SQL dumps ideal for smaller databases and cross-version migrations. +- **WAL-G backups** provide continuous archiving with point-in-time recovery, suited for larger databases requiring minimal data loss. + +**Note:** The HA Proxy must be enabled (`proxy.enabled: true`) for logical backups to function correctly. + +Set `backup.enabled: true`, choose a `mode` (`logical` or `wal-g`), then set `backup.provider` to `aws` or `gcp` and fill in the corresponding block: + +```yaml +backup: + enabled: true + mode: logical # logical or wal-g + provider: aws # aws or gcp + + logical: + schedule: "0 2 * * *" + + walg: + intervalSeconds: 21600 + + aws: + bucket: my-bucket + region: us-east-1 + cloudAccountName: my-cloud-account + policyName: my-backup-policy + prefix: postgres/backups + + gcp: + bucket: my-bucket + cloudAccountName: my-cloud-account + prefix: postgres/backups +``` + +### AWS S3 + +For the workload to have access to a S3 bucket, ensure the following prerequisites are completed in your AWS account before installing: + +1. Create your bucket. Update the value `bucket` to include its name and `region` to include its region. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Update the value `cloudAccountName`. + +3. Create a new AWS IAM policy with the following JSON (replace `YOUR_BUCKET_NAME`) + +```JSON +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetObjectVersion", + "s3:DeleteObjectVersion" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + } + ] +} +``` + +4. Update `cloudAccountName` in your values file with the name of your Cloud Account. + +5. Set `policyName` to match the policy created in step 3. + +### GCS + +For the workload to have access to a GCS bucket, ensure the following prerequisites are completed in your GCP account before installing: + +1. Create your bucket. Update the value `bucket` to include its name. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Update the value `cloudAccountName`. + +**Important**: You must add the `Storage Admin` role to the created GCP service account. + +## Restoring Backup + +### Logical + +Run the following command with password from a client with access to the bucket. Set `WORKLOAD_NAME` to match the proxy workload so restores write to the leader. + +S3 +```SH +export PGPASSWORD="PASSWORD" + +aws s3 cp "s3://BUCKET_NAME/PREFIX/BACKUP_FILE.sql.gz" - \ + | gunzip \ + | psql \ + --host=WORKLOAD_NAME \ + --port=5432 \ + --username=USERNAME \ + --dbname=postgres + +unset PGPASSWORD +``` + +GCS +```SH +export PGPASSWORD="PASSWORD" + +gsutil cp "gs://BUCKET_NAME/PREFIX/BACKUP_FILE.sql.gz" - \ + | gunzip \ + | psql \ + --host=WORKLOAD_NAME \ + --port=5432 \ + --username=USERNAME \ + --dbname=postgres + +unset PGPASSWORD +``` + +### WAL-G + +Because a point-in-time restore from WAL-G requires an empty data directory, follow the steps below. + +1. Run `wal-g backup-list` to get desired backup. +2. Stop the postgres workload. +3. Create a new volume set to restore to. +4. Run a one-off restore workload with the new volume set mounted at `/var/lib/postgresql/data` and run the following command: +```SH +wal-g backup-fetch /var/lib/postgresql/data/pgdata +``` +5. Re-point the postgres workload to the restored volume set and restart the workload. +6. **After restore**: Change the WAL-G prefix before re-enabling backups to avoid system identifier conflicts. + +## Supported External Services + +- [Patroni Documentation](https://patroni.readthedocs.io/) +- [Postgres Doccumentation](https://www.postgresql.org/docs/) +- [etcd Documentation](https://etcd.io/docs/v3.6/) \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/_helpers.tpl b/postgres-highly-available/versions/2.3.1/templates/_helpers.tpl new file mode 100644 index 00000000..d9532a5d --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/_helpers.tpl @@ -0,0 +1,140 @@ +{{/* Resource Naming */}} + +{{/* +Postgres HA Workload Name +*/}} +{{- define "pg-ha.name" -}} +{{- printf "%s-postgres-ha" .Release.Name }} +{{- end }} + +{{/* +Postgres HA etcd Workload Name +*/}} +{{- define "pg-ha.etcd.name" -}} +{{- printf "%s-etcd" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Proxy Workload Name +*/}} +{{- define "pg-ha.proxy.name" -}} +{{- printf "%s-postgres-ha-proxy" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Workload Logical Backup Name +*/}} +{{- define "pg-ha.backup.name" -}} +{{- printf "%s-postgres-ha-backup" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Secret Database Config Name +*/}} +{{- define "pg-ha.secretDatabase.name" -}} +{{- printf "%s-postgres-config" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Secret Startup Name +*/}} +{{- define "pg-ha.secretStartup.name" -}} +{{- printf "%s-postgres-proxy-startup" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Secret Proxy Startup Name +*/}} +{{- define "pg-ha.secretProxyStartup.name" -}} +{{- printf "%s-patroni-startup" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Secret WAL-G Backup Startup Name +*/}} +{{- define "pg-ha.secretWALGStartup.name" -}} +{{- printf "%s-wal-g-backup-script" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Identity Name +*/}} +{{- define "pg-ha.identity.name" -}} +{{- printf "%s-postgres-ha-identity" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Policy Name +*/}} +{{- define "pg-ha.policy.name" -}} +{{- printf "%s-postgres-ha-policy" .Release.Name }} +{{- end }} + +{{/* +Postgres HA Volume Set Name +*/}} +{{- define "pg-ha.volume.name" -}} +{{- printf "%s-postgres-ha-vs" .Release.Name }} +{{- end }} + +{{/* +PgBouncer Workload Name +*/}} +{{- define "pg-ha.pgbouncer.name" -}} +{{- printf "%s-pgbouncer" .Release.Name }} +{{- end }} + + +{{/* Validation */}} + +{{/* +Validate backup mode - must be "logical" or "wal-g" +*/}} +{{- define "pg-ha.validateBackupMode" -}} +{{- $mode := .Values.backup.mode -}} +{{- if and .Values.backup.enabled (not (or (eq $mode "logical") (eq $mode "wal-g"))) -}} + {{- fail (printf "Invalid backup.mode: '%s'. Must be either 'logical' or 'wal-g'." $mode) -}} +{{- end -}} +{{- end }} + +{{/* +Validate backup configuration - when backup is enabled, backup.provider must be set to 'aws' or 'gcp' +*/}} +{{- define "pg-ha.validateBackupConfig" -}} +{{- include "pg-ha.validateBackupMode" . -}} +{{- if .Values.backup.enabled -}} + {{- $provider := .Values.backup.provider -}} + {{- if not (or (eq $provider "aws") (eq $provider "gcp")) -}} + {{- fail "Invalid backup configuration: backup.provider must be set to 'aws' or 'gcp'." -}} + {{- end -}} + {{- if eq $provider "aws" -}} + {{- if not .Values.backup.aws.bucket -}} + {{- fail "Invalid backup configuration: backup.aws.bucket is required when provider is 'aws'." -}} + {{- end -}} + {{- if not .Values.backup.aws.region -}} + {{- fail "Invalid backup configuration: backup.aws.region is required when provider is 'aws'." -}} + {{- end -}} + {{- if not .Values.backup.aws.cloudAccountName -}} + {{- fail "Invalid backup configuration: backup.aws.cloudAccountName is required when provider is 'aws'." -}} + {{- end -}} + {{- end -}} + {{- if eq $provider "gcp" -}} + {{- if not .Values.backup.gcp.bucket -}} + {{- fail "Invalid backup configuration: backup.gcp.bucket is required when provider is 'gcp'." -}} + {{- end -}} + {{- if not .Values.backup.gcp.cloudAccountName -}} + {{- fail "Invalid backup configuration: backup.gcp.cloudAccountName is required when provider is 'gcp'." -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels - delegated to cpln-common +*/}} +{{- define "pg-ha.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/identity.yaml b/postgres-highly-available/versions/2.3.1/templates/identity.yaml new file mode 100644 index 00000000..838626cf --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/identity.yaml @@ -0,0 +1,25 @@ +{{- include "pg-ha.validateBackupConfig" . -}} +kind: identity +gvc: {{ .Values.global.cpln.gvc }} +name: {{ include "pg-ha.identity.name" . }} +description: Postgres Highly Available identity +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.backup.aws.cloudAccountName }} + policyRefs: + - cpln-connector + - aws::ReadOnlyAccess + - "{{ .Values.backup.aws.policyName }}" +{{- end }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "gcp") }} +gcp: + bindings: + - resource: //storage.googleapis.com/projects/_/buckets/{{ .Values.backup.gcp.bucket }} + roles: + - roles/storage.objectAdmin + cloudAccountLink: //cloudaccount/{{ .Values.backup.gcp.cloudAccountName }} + scopes: + - https://www.googleapis.com/auth/cloud-platform +{{- end }} \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/policy.yaml b/postgres-highly-available/versions/2.3.1/templates/policy.yaml new file mode 100644 index 00000000..3fe654f0 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/policy.yaml @@ -0,0 +1,20 @@ +kind: policy +name: {{ include "pg-ha.policy.name" . }} +description: Postgres Highly Available policy +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "pg-ha.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "pg-ha.secretStartup.name" . }} + - //secret/{{ include "pg-ha.secretDatabase.name" . }} + {{- if .Values.proxy.enabled }} + - //secret/{{ include "pg-ha.secretProxyStartup.name" . }} + {{- end }} + {{- if and .Values.backup.enabled (eq .Values.backup.mode "wal-g") }} + - //secret/{{ include "pg-ha.secretWALGStartup.name" . }} + {{- end }} \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/secret-config.yaml b/postgres-highly-available/versions/2.3.1/templates/secret-config.yaml new file mode 100644 index 00000000..f3cd9b45 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/secret-config.yaml @@ -0,0 +1,17 @@ +kind: secret +name: {{ include "pg-ha.secretDatabase.name" . }} +description: Postgres Highly Available config +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +type: dictionary +data: + username: {{ .Values.postgres.username | quote }} + password: {{ .Values.postgres.password | quote }} + database: {{ .Values.postgres.database | quote }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} + backup-bucket: {{ .Values.backup.aws.bucket | quote }} + aws-region: {{ .Values.backup.aws.region | quote }} +{{- end }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "gcp") }} + backup-bucket: {{ .Values.backup.gcp.bucket | quote }} +{{- end }} \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml b/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml new file mode 100644 index 00000000..f42d367d --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml @@ -0,0 +1,91 @@ +{{- if or .Values.proxy.enabled .Values.pgbouncer.enabled }} +kind: secret +name: {{ include "pg-ha.secretProxyStartup.name" . }} +description: HAProxy startup script for Postgres Highly Available +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env sh + set -eu + + LOCATION="$(basename "${CPLN_LOCATION:-}")" + if [ -z "${LOCATION}" ]; then + echo "ERROR: CPLN_LOCATION is empty; cannot derive location" + exit 1 + fi + + GVC="{{ .Values.global.cpln.gvc }}" + WORKLOAD="{{ include "pg-ha.name" . }}" + REPLICAS="{{ .Values.replicas }}" + + echo "Starting HAProxy Patroni leader proxy" + echo "Derived location: ${LOCATION}" + echo "Target workload: ${WORKLOAD}" + echo "Replicas: ${REPLICAS}" + + CFG="/tmp/haproxy.cfg" + + SERVERS="" + i=0 + while [ "${i}" -lt "${REPLICAS}" ]; do + HOST="replica-${i}.${WORKLOAD}.${LOCATION}.${GVC}.cpln.local" + SERVERS="${SERVERS} + server pg${i} ${HOST}:5432 check port 8008" + i=$((i + 1)) + done + + cat > "${CFG}" < "$CONFIG_FILE" <> "$CONFIG_FILE" < + sh -lc 'psql -h 127.0.0.1 -p 5432 -U {{ .Values.postgres.username }} -d postgres -c "CREATE DATABASE {{ .Values.postgres.database }}"' + EOF + else + echo "PGDATA exists — starting Patroni normally" + fi + + # --- drop privileges and start Patroni --- + echo "Dropping privileges to postgres user..." + exec gosu postgres patroni "$CONFIG_FILE" \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/secret-wal-g-script.yaml b/postgres-highly-available/versions/2.3.1/templates/secret-wal-g-script.yaml new file mode 100644 index 00000000..88ebe64a --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/secret-wal-g-script.yaml @@ -0,0 +1,39 @@ +kind: secret +name: {{ include "pg-ha.secretWALGStartup.name" . }} +description: WAL-G base backup sidecar script +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env bash + set -euo pipefail + + : "${PGDATA:?Missing PGDATA}" + : "${PATRONI_API:=http://127.0.0.1:8008}" + : "${WALG_BACKUP_INTERVAL_SECONDS:=3600}" + + echo "WAL-G base backup sidecar started" + echo "PGDATA=${PGDATA}" + echo "PATRONI_API=${PATRONI_API}" + echo "Interval=${WALG_BACKUP_INTERVAL_SECONDS}s" + + # small jitter so fleets don't backup at the same second + sleep $(( RANDOM % 30 )) + + while true; do + code="$(curl -s -o /dev/null -w "%{http_code}" "${PATRONI_API}/primary" || true)" + + if [ "${code}" = "200" ]; then + echo "Leader confirmed. Running wal-g backup-push ${PGDATA}" + wal-g backup-push "${PGDATA}" + echo "Backup complete" + + # Optional retention (uncomment if you want it) + # : "${WALG_RETAIN_FULL:=7}" + # wal-g delete retain FULL "${WALG_RETAIN_FULL}" --confirm || true + else + echo "Not leader (/primary=${code}). Skipping backup." + fi + + sleep "${WALG_BACKUP_INTERVAL_SECONDS}" + done \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/volumeset.yaml b/postgres-highly-available/versions/2.3.1/templates/volumeset.yaml new file mode 100644 index 00000000..6a39ba8c --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/volumeset.yaml @@ -0,0 +1,20 @@ +kind: volumeset +name: {{ include "pg-ha.volume.name" . }} +description: Postgres Highly Available volumeset +gvc: {{ .Values.global.cpln.gvc }} +tags: + {{- include "pg-ha.tags" . | nindent 2 }} + workload: {{ include "pg-ha.name" . }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.volumeset.capacity }} + {{- if .Values.volumeset.autoscaling.enabled }} + autoscaling: + maxCapacity: {{ .Values.volumeset.autoscaling.maxCapacity }} + minFreePercentage: {{ .Values.volumeset.autoscaling.minFreePercentage }} + scalingFactor: {{ .Values.volumeset.autoscaling.scalingFactor }} + {{- end }} + performanceClass: general-purpose-ssd + snapshots: + createFinalSnapshot: true + retentionDuration: 7d \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/workload-ha-proxy.yaml b/postgres-highly-available/versions/2.3.1/templates/workload-ha-proxy.yaml new file mode 100644 index 00000000..4ed0855f --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/workload-ha-proxy.yaml @@ -0,0 +1,56 @@ +{{- if or .Values.proxy.enabled .Values.pgbouncer.enabled }} +kind: workload +name: {{ include "pg-ha.proxy.name" . }} +description: HAProxy leader-only endpoint for Patroni Postgres +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: haproxy + image: {{ .Values.proxy.image }} + inheritEnv: false + command: /bin/sh + args: + - /proxy/start.sh + cpu: {{ .Values.proxy.resources.cpu }} + memory: {{ .Values.proxy.resources.memory }} + ports: + - number: 5432 + protocol: tcp + - number: 8404 + protocol: http + - number: 8405 + protocol: http + volumes: + - path: /proxy/start.sh + uri: cpln://secret/{{ include "pg-ha.secretProxyStartup.name" . }}.payload + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "pg-ha.identity.name" . }} + defaultOptions: + autoscaling: + metric: rps + minScale: {{ .Values.proxy.minReplicas }} + maxScale: {{ .Values.proxy.maxReplicas }} + maxConcurrency: 0 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + multiZone: + enabled: {{ .Values.multiZone }} + timeoutSeconds: 5 + firewallConfig: + internal: + inboundAllowType: {{ .Values.internal_access.type }} + {{- if .Values.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + {{- end }} + external: + inboundAllowCIDR: [] + outboundAllowCIDR: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false +{{- end }} \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/workload-logical-backup.yaml b/postgres-highly-available/versions/2.3.1/templates/workload-logical-backup.yaml new file mode 100644 index 00000000..a9897b60 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/workload-logical-backup.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.backup.enabled (eq .Values.backup.mode "logical") }} +kind: workload +name: {{ include "pg-ha.backup.name" . }} +description: Scheduled logical backup +tags: + {{- include "pg-ha.tags" . | nindent 2 }} +spec: + type: cron + containers: + - name: backup + image: {{ .Values.backup.logical.image }} + cpu: {{ .Values.backup.resources.cpu | quote }} + memory: {{ .Values.backup.resources.memory | quote }} + inheritEnv: false + env: + - name: PG_HOST + value: {{ include "pg-ha.proxy.name" . }}.{{ .Values.global.cpln.gvc }}.cpln.local + - name: PG_PORT + value: '5432' + - name: PG_USER + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.username + - name: PG_PASSWORD + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.password + - name: BACKUP_BUCKET + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.backup-bucket + {{- if eq .Values.backup.provider "aws" }} + - name: AWS_REGION + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.aws-region + - name: BACKUP_PROVIDER + value: aws + - name: BACKUP_PREFIX + value: {{ .Values.backup.aws.prefix }} + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + - name: BACKUP_PROVIDER + value: gcp + - name: BACKUP_PREFIX + value: {{ .Values.backup.gcp.prefix }} + {{- end }} + identityLink: //identity/{{ include "pg-ha.identity.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 1 + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + multiZone: + enabled: false + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowPort: + - number: 443 + protocol: tcp + {{- if eq .Values.backup.provider "aws" }} + outboundAllowHostname: + - s3.{{ .Values.backup.aws.region }}.amazonaws.com + - s3.amazonaws.com + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + outboundAllowHostname: + - storage.googleapis.com + {{- end }} + job: + schedule: {{ .Values.backup.logical.schedule | quote }} + concurrencyPolicy: Forbid + restartPolicy: Never + historyLimit: 5 +{{- end }} \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml b/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml new file mode 100644 index 00000000..9376ddb1 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml @@ -0,0 +1,148 @@ +kind: workload +name: {{ include "pg-ha.name" . }} +description: Postgres Highly Available +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +spec: + type: stateful + containers: + - name: patroni-postgres + command: "/bin/bash" + args: + - "/patroni/start.sh" + minCpu: {{ .Values.resources.minCpu | quote }} + minMemory: {{ .Values.resources.minMemory | quote }} + cpu: {{ .Values.resources.maxCpu | quote }} + memory: {{ .Values.resources.maxMemory | quote }} + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + {{- if and .Values.backup.enabled (eq .Values.backup.mode "wal-g") }} + - name: WALG_COMPRESSION_METHOD + value: zstd + {{- if .Values.backup.aws.enabled }} + - name: WALG_S3_PREFIX + value: s3://{{ .Values.backup.aws.s3.bucket }}/{{ .Values.backup.aws.s3.prefix }} + - name: AWS_REGION + value: {{ .Values.backup.aws.s3.region | quote }} + - name: AWS_S3_FORCE_PATH_STYLE + value: "true" + {{- end }} + {{- if .Values.backup.gcp.enabled }} + - name: WALG_GS_PREFIX + value: gs://{{ .Values.backup.gcp.gcs.bucket }}/{{ .Values.backup.gcp.gcs.prefix }} + {{- end }} + {{- end }} + image: {{ .Values.image }} + inheritEnv: false + ports: + - number: 8008 + protocol: tcp + - number: 5432 + protocol: tcp + volumes: + - path: /var/lib/postgresql/data + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "pg-ha.volume.name" . }} + - path: /patroni/start.sh + uri: cpln://secret/{{ include "pg-ha.secretStartup.name" . }}.payload + # WAL-G backup sidecar + {{- if and .Values.backup.enabled (eq .Values.backup.mode "wal-g") }} + - name: wal-g-backup + image: {{ .Values.image }} + command: "/bin/bash" + args: + - "/walg/backup.sh" + cpu: {{ .Values.backup.resources.cpu | quote }} + memory: {{ .Values.backup.resources.memory | quote }} + inheritEnv: false + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: PATRONI_API + value: http://127.0.0.1:8008 + - name: WALG_BACKUP_INTERVAL_SECONDS + value: {{ .Values.backup.walg.intervalSeconds | quote }} + - name: WALG_COMPRESSION_METHOD + value: zstd + - name: PGHOST + value: 127.0.0.1 + - name: PGPORT + value: "5432" + - name: PGUSER + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.username + - name: PGPASSWORD + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.password + - name: PGDATABASE + value: postgres + {{- if .Values.backup.aws.enabled }} + - name: WALG_S3_PREFIX + value: s3://{{ .Values.backup.aws.s3.bucket }}/{{ .Values.backup.aws.s3.prefix }} + - name: AWS_REGION + value: {{ .Values.backup.aws.s3.region | quote }} + - name: AWS_S3_FORCE_PATH_STYLE + value: "true" + {{- end }} + {{- if .Values.backup.gcp.enabled }} + - name: WALG_GS_PREFIX + value: gs://{{ .Values.backup.gcp.gcs.bucket }}/{{ .Values.backup.gcp.gcs.prefix }} + {{- end }} + volumes: + - path: /var/lib/postgresql/data + uri: cpln://volumeset/{{ include "pg-ha.volume.name" . }} + recoveryPolicy: retain + - path: /walg/backup.sh + uri: cpln://secret/{{ include "pg-ha.secretWALGStartup.name" . }}.payload + {{- end }} + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "pg-ha.identity.name" . }} + defaultOptions: + autoscaling: + keda: + cooldownPeriod: 1 + initialCooldownPeriod: 1 + pollingInterval: 1 + triggers: [] + maxConcurrency: 0 + maxScale: {{ .Values.replicas }} + metric: disabled + minScale: {{ .Values.replicas }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + multiZone: + enabled: {{ .Values.multiZone }} + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowPort: [] + {{- if and .Values.backup.enabled (eq .Values.backup.mode "wal-g") }} + {{- if .Values.backup.aws.enabled }} + outboundAllowHostname: + - s3.{{ .Values.backup.aws.s3.region }}.amazonaws.com + - s3.amazonaws.com + {{- end }} + {{- end }} + {{- if and .Values.backup.enabled (eq .Values.backup.mode "wal-g") }} + {{- if .Values.backup.gcp.enabled }} + outboundAllowHostname: + - storage.googleapis.com + {{- end }} + {{- end }} + outboundBlockedCIDR: [] + internal: + inboundAllowType: {{ .Values.internal_access.type }} + {{- if .Values.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + {{- end }} + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + securityOptions: + filesystemGroupId: 999 + supportDynamicTags: false \ No newline at end of file diff --git a/postgres-highly-available/versions/2.3.1/templates/workload-pgbouncer.yaml b/postgres-highly-available/versions/2.3.1/templates/workload-pgbouncer.yaml new file mode 100644 index 00000000..4969b460 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/templates/workload-pgbouncer.yaml @@ -0,0 +1,67 @@ +{{- if .Values.pgbouncer.enabled }} +kind: workload +name: {{ include "pg-ha.pgbouncer.name" . }} +description: PgBouncer Connection Pooler +tags: + {{- include "pg-ha.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: pgbouncer + image: {{ .Values.pgbouncer.image }} + cpu: {{ .Values.pgbouncer.resources.cpu | quote }} + memory: {{ .Values.pgbouncer.resources.memory | quote }} + inheritEnv: false + env: + - name: DB_HOST + value: {{ include "pg-ha.proxy.name" . }}.{{ .Values.global.cpln.gvc }}.cpln.local + - name: DB_PORT + value: '5432' + - name: DB_USER + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.username + - name: DB_PASSWORD + value: cpln://secret/{{ include "pg-ha.secretDatabase.name" . }}.password + - name: DB_NAME + value: {{ .Values.postgres.database | quote }} + - name: POOL_MODE + value: {{ .Values.pgbouncer.poolMode | quote }} + - name: DEFAULT_POOL_SIZE + value: {{ .Values.pgbouncer.defaultPoolSize | quote }} + - name: MAX_CLIENT_CONN + value: {{ .Values.pgbouncer.maxClientConn | quote }} + - name: MAX_DB_CONNECTIONS + value: {{ .Values.pgbouncer.maxDbConnections | quote }} + - name: AUTH_TYPE + value: plain + ports: + - number: 5432 + protocol: tcp + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "pg-ha.identity.name" . }} + defaultOptions: + autoscaling: + metric: rps + minScale: {{ .Values.pgbouncer.minReplicas }} + maxScale: {{ .Values.pgbouncer.maxReplicas }} + maxConcurrency: 0 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + multiZone: + enabled: {{ .Values.multiZone }} + timeoutSeconds: 5 + firewallConfig: + internal: + inboundAllowType: {{ .Values.internal_access.type }} + {{- if .Values.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + {{- end }} + external: + inboundAllowCIDR: [] + outboundAllowCIDR: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false +{{- end }} diff --git a/postgres-highly-available/versions/2.3.1/values.yaml b/postgres-highly-available/versions/2.3.1/values.yaml new file mode 100644 index 00000000..3b7315e7 --- /dev/null +++ b/postgres-highly-available/versions/2.3.1/values.yaml @@ -0,0 +1,96 @@ +replicas: 3 + +resources: + minCpu: 500m + minMemory: 1Gi + maxCpu: 1 + maxMemory: 2Gi + +image: controlplanecorporation/patroni-postgres:0.7 + +postgres: + username: username + password: password + database: test + +multiZone: false + +volumeset: + capacity: 10 # initial capacity in GiB (minimum is 10) + autoscaling: + enabled: false # Set to true to enable autoscaling + maxCapacity: 100 # Maximum capacity in GiB when autoscaling is enabled + minFreePercentage: 10 # Minimum free percentage to trigger scaling when autoscaling is enabled + scalingFactor: 1.2 # Scaling factor to determine how much to scale up when autoscaling is triggered + +internal_access: + type: same-gvc # options: same-gvc, same-org, workload-list + workloads: # Note: can only be used if type is same-gvc or workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + +etcd: + replicas: 3 + resources: + cpu: 500m + memory: 512Mi + multiZone: false + volumeset: + capacity: 10 # initial capacity in GiB (minimum is 10) + internal_access: + type: same-gvc # options: same-gvc, same-org, workload-list + workloads: # Note: can only be used if type is same-gvc or workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + +pgbouncer: + enabled: false + image: edoburu/pgbouncer:v1.25.1-p0 + poolMode: transaction # options: session, transaction, statement + defaultPoolSize: 25 # number of real Postgres connections PgBouncer maintains per pod + maxClientConn: 1000 # maximum number of client connections PgBouncer accepts per pod + maxDbConnections: 100 # hard cap on total Postgres connections regardless of how many PgBouncer pods are running + minReplicas: 2 + maxReplicas: 4 + + resources: + cpu: 200m + memory: 128Mi + +proxy: # HA Proxy endpoint to write to leader replica. Automatically enabled when pgbouncer is enabled. + enabled: true + image: haproxy:2.9 + resources: + cpu: 100m + memory: 128Mi + minReplicas: 2 + maxReplicas: 2 + +backup: + enabled: false + mode: logical # logical or wal-g + resources: # applies to whichever mode is enabled + cpu: 100m + memory: 128Mi + + logical: # logical backup settings + image: controlplanecorporation/pg-backup:17.1.0 # tag 17.1.0 = Postgres 17, no other tags currently supported + schedule: "0 2 * * *" # cron schedule, default isdaily at 2am UTC + + walg: # wal-g backup settings + intervalSeconds: 21600 # interval in seconds between backups, default is every 6 hours + + # storage settings are applied to whichever mode is enabled + provider: aws # Options: aws or gcp + + aws: + bucket: pg-ha-backup-bucket + region: us-east-1 + cloudAccountName: my-s3-cloud-account + policyName: pg-ha-backup-policy + prefix: postgres/backups # folder name where your backups will be stored + + gcp: + bucket: pg-ha-backup-bucket + cloudAccountName: my-gcs-cloud-account + prefix: postgres/backups # folder name where your backups will be stored \ No newline at end of file From 115f099f27f2cc026c0e8e80c2878d260b1c04fa Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Fri, 24 Apr 2026 17:23:53 -0700 Subject: [PATCH 09/58] lowered proxy rise to 1 (#246) --- .../versions/2.3.1/templates/secret-ha-proxy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml b/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml index f42d367d..362460e8 100644 --- a/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml +++ b/postgres-highly-available/versions/2.3.1/templates/secret-ha-proxy.yaml @@ -76,7 +76,7 @@ data: http-check connect port 8008 http-check send meth GET uri /primary ver HTTP/1.0 http-check expect status 200 - default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions + default-server inter 3s fall 3 rise 1 on-marked-down shutdown-sessions ${SERVERS} EOF From 94fd4e6095afb416fe14eaa66033e8641321309c Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Sun, 26 Apr 2026 19:50:52 -0700 Subject: [PATCH 10/58] added secondary index value, added publishNotReadyAddresses tag, moved searcbd log dir (#247) * init 2.0.1 * added secondary index value * added secondary indexes * moved searchd log directory to prevent loading wrong config from donor * added publish not ready addresses tag * defaulted secondary indexes to false --- manticore/versions/2.0.1/Chart.yaml | 18 + manticore/versions/2.0.1/README.md | 258 ++++++++ .../versions/2.0.1/templates/_helpers.tpl | 242 ++++++++ .../versions/2.0.1/templates/domain.yaml | 34 ++ .../versions/2.0.1/templates/identity.yaml | 66 ++ .../versions/2.0.1/templates/policy.yaml | 99 +++ .../versions/2.0.1/templates/secret.yaml | 330 ++++++++++ .../2.0.1/templates/volumeset-shared.yaml | 11 + .../versions/2.0.1/templates/volumeset.yaml | 15 + .../versions/2.0.1/templates/workload.yaml | 566 ++++++++++++++++++ manticore/versions/2.0.1/values.yaml | 220 +++++++ 11 files changed, 1859 insertions(+) create mode 100644 manticore/versions/2.0.1/Chart.yaml create mode 100644 manticore/versions/2.0.1/README.md create mode 100644 manticore/versions/2.0.1/templates/_helpers.tpl create mode 100644 manticore/versions/2.0.1/templates/domain.yaml create mode 100644 manticore/versions/2.0.1/templates/identity.yaml create mode 100644 manticore/versions/2.0.1/templates/policy.yaml create mode 100644 manticore/versions/2.0.1/templates/secret.yaml create mode 100644 manticore/versions/2.0.1/templates/volumeset-shared.yaml create mode 100644 manticore/versions/2.0.1/templates/volumeset.yaml create mode 100644 manticore/versions/2.0.1/templates/workload.yaml create mode 100644 manticore/versions/2.0.1/values.yaml diff --git a/manticore/versions/2.0.1/Chart.yaml b/manticore/versions/2.0.1/Chart.yaml new file mode 100644 index 00000000..fc33a7ac --- /dev/null +++ b/manticore/versions/2.0.1/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: manticore +description: Distributed Manticore Search cluster with intelligent orchestration. + +type: application +version: 2.0.1 +appVersion: "25.0.0" + +annotations: + created: "2026-01-05" + lastModified: "2026-04-17" + category: "search" + createsGvc: false + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" \ No newline at end of file diff --git a/manticore/versions/2.0.1/README.md b/manticore/versions/2.0.1/README.md new file mode 100644 index 00000000..fe07cae8 --- /dev/null +++ b/manticore/versions/2.0.1/README.md @@ -0,0 +1,258 @@ +# Manticore Search Cluster + +Deploys a distributed Manticore Search cluster on Control Plane with automatic Galera-based replication, zero-downtime data imports, multi-table support, backup/restore, and a web UI for cluster management. + +## Architecture + +The template deploys several components that work together: + +- **Manticore Workload** - Stateful replicas running Manticore searchd, each with a sidecar agent for local operations +- **Orchestrator API** - REST API that coordinates cluster-wide operations (initialization, imports, repairs, backups) +- **Orchestrator Job** - Cron workload for on-demand job execution +- **UI** - Web dashboard for monitoring and managing the cluster + +The orchestrator handles cluster initialization, coordinates imports across all replicas using a dual-slot (A/B) system for zero-downtime swaps, and provides automatic repair for split-brain scenarios. All replicas stay in sync via Galera cluster replication. + +## Prerequisites + +1. **S3 Bucket** - Create an S3 bucket to store your CSV source files +2. **Control Plane Cloud Account** - Follow the [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account) guide to establish trust between Control Plane and your AWS account + +## Installation + +1. **Configure S3 access** in `values.yaml`: + ```yaml + buckets: + cloudAccountName: your-cloud-account + awsPolicyRefs: + - aws::AmazonS3ReadOnlyAccess # or your custom policy + sourceBucket: your-bucket-name + ``` + +2. **Define your tables**: + ```yaml + tables: + - name: products + csvPath: imports/products/data.csv + config: + haStrategy: noerrors # HA strategy for distributed queries; 'noerrors' skips agents that return errors + agentRetryCount: 3 # Number of times to retry failed agent connections + clusterMain: false # Set to true to replicate the main table across all cluster nodes + memLimit: 2G # Memory limit for indexer during import (max = 2G) + hasHeader: true # Set to true if the CSV file includes a header row + schema: + columns: + - name: title + type: field + - name: price + type: attr_float + ``` + +3. **Generate an authentication token**: + ```bash + openssl rand -base64 32 + ``` + Set this in `orchestrator.agent.token`. This bearer token secures all internal API communication between components. + +**Note:** After installation, the cluster will be initialized but tables will be empty until you run an import. See [Operations](#operations) below. + +## Authentication + +All internal communication is secured with the bearer token set in `orchestrator.agent.token`. This token is shared across the orchestrator, agents, and UI. + +- Must be set before deployment +- Should be cryptographically random (use `openssl rand -base64 32`) +- Rotating requires redeploying all components + +**Security note:** The UI injects this token automatically, so anyone with network access to the UI can perform admin operations. Restrict access by setting `orchestrator.ui.allowExternalAccess: false` or using a domain with authentication. + +## Configuration Reference + +### Core Settings + +| Path | Description | Default | +|------|-------------|---------| +| `buckets.cloudAccountName` | AWS Cloud Account name | - | +| `buckets.sourceBucket` | S3 bucket with CSV files | - | +| `manticore.clusterName` | Galera cluster name | `manticore` | +| `manticore.autoscaling.minScale` | Minimum replicas | `3` | +| `manticore.autoscaling.maxScale` | Maximum replicas | `4` | + +### Table Configuration + +Each entry in `tables[]` supports: + +| Field | Description | +|-------|-------------| +| `name` | Table name | +| `csvPath` | Path to CSV in S3 bucket, or a list of paths for multi-segment tables (see [Multi-Segment Tables](#multi-segment-tables)) | +| `config.haStrategy` | HA strategy: `noerrors`, `nodeads`, etc. | +| `config.agentRetryCount` | Retry count for distributed queries | +| `config.clusterMain` | Replicate main tables across cluster | +| `config.segmentCount` | Number of distributed table segments; must match the number of entries in `csvPath` (default: `1`) | +| `config.importMethod` | Import method: `indexer` or `sql` | +| `config.charsetTable` | Manticore `charset_table` tokenization preset (e.g., `non_cont`) — omit to use the Manticore default | +| `config.memLimit` | Memory limit for indexer operations (e.g., `2G`) | +| `config.hasHeader` | Whether the CSV file has a header row (`true`/`false`) | +| `schema.columns` | Column definitions (see column types below) | + +### Column Types + +| Type | Description | +|------|-------------| +| `field` | Full-text searchable field | +| `field_string` | Full-text field (string variant) | +| `attr_uint` | Unsigned integer attribute | +| `attr_bigint` | Big integer attribute | +| `attr_float` | Float attribute | +| `attr_bool` | Boolean attribute | +| `attr_string` | String attribute (not full-text indexed) | +| `attr_timestamp` | Timestamp attribute | +| `attr_multi` | Multi-value integer attribute | +| `attr_multi_64` | Multi-value 64-bit integer attribute | +| `attr_json` | JSON attribute | + +**Note**: If column 1 is numeric, it's used as the document ID (don't declare it). If not numeric, an ID is auto-generated. + +### Orchestrator Settings + +| Path | Description | Default | +|------|-------------|---------| +| `orchestrator.schedule` | Cron schedule for imports | `0 * * * *` | +| `orchestrator.action` | Action: `init`, `import`, `health`, `repair` | `import` | +| `orchestrator.tableName` | Table to import | - | +| `orchestrator.suspend` | Start suspended | `true` | +| `orchestrator.agent.token` | Bearer token for auth | **required** | + +## Multi-Segment Tables + +Large datasets can be split across multiple CSV files and imported as a distributed table with multiple independent segments. Manticore fans queries across all segments automatically. + +Set `csvPath` to a list of S3 paths and set `segmentCount` to match the number of entries: + +```yaml +tables: + - name: addresses + csvPath: + - large-file/part1.csv + - large-file/part2.csv + config: + segmentCount: 2 # must match the number of csvPath entries + importMethod: indexer + memLimit: 2G + hasHeader: true + schema: + columns: + - name: street_name + type: field +``` + +`segmentCount` must equal the number of items in `csvPath`. The template will fail at render time with a descriptive error if they don't match. + +When a table has multiple segments, a backup backs up **all segments** as one file. Restores on multi-segment tables will restore all segments on the table. + +## Operations + +Operations can be triggered via the **Orchestrator UI** or the **Control Plane CLI/API**. + +### Via Orchestrator UI + +The web dashboard provides controls for: +- **Import, Backup and Restore** - Select a table and trigger a coordinated import, backup, or restore process +- **Repair** - Recover the cluster from split-brain scenarios +- **Monitoring** - View cluster health, replica status, and table details + +### Via Control Plane + +Run the orchestrator cron workload to execute operations: + +```bash +# Trigger an import +cpln workload run-cron {release-name}-orchestrator-job --gvc {gvc-name} + +# Trigger a repair (set ACTION=repair on the workload first) +cpln workload run-cron {release-name}-orchestrator-job --gvc {gvc-name} +``` + +## Load Testing + +Enable k6 load testing to validate search performance: + +```yaml +loadTest: + enabled: true + vus: 10 + duration: "5m" + query: + index: products + query: + match: + "*": "test" +``` + +Trigger via Control Plane: +```bash +cpln workload run-cron {release-name}-load-test-controller --gvc {gvc-name} +``` + +Or set `loadTest.controller.schedule` to run on a cron schedule. + +## Backup & Restore + +Backup and restore is available for both **delta** (real-time updates) and **main** (full indexed dataset) tables. Backups are stored as compressed archives in S3. + +### Prerequisites + +1. **S3 Bucket** for storing backups (can be shared with or separate from source data) +2. **IAM Policy** with `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` permissions on the bucket +3. **Cloud Account** with the above policy attached + +### Configuration + +Enable backups in `values.yaml`: + +```yaml +orchestrator: + backup: + enabled: true + cloudAccountName: my-backup-cloud-account + s3Bucket: my-backup-bucket + s3Policy: + - my-backup-policy + s3Region: us-east-1 + prefix: manticore-backups + schedules: [ # Automated backup schedules (optional) + {"table": "products", "type": "delta", "schedule": "0 2 * * *"}, + {"table": "products", "type": "main", "schedule": "0 3 * * 0"} + ] +``` + +### Usage + +**Via Orchestrator UI:** +- **Backup**: Select a type (delta/main) and click "Backup" +- **Restore**: Select a type, choose a backup file from the list, and confirm +- **Rotate Main**: After a main restore, swap the active slot + +**Via API:** +```bash +# Backup +curl -X POST "https://{orchestrator-api-url}/api/backup" \ + -H "Authorization: Bearer {token}" \ + -H "Content-Type: application/json" \ + -d '{"tableName": "products", "type": "delta"}' + +# List backups +curl "https://{orchestrator-api-url}/api/backups/files?tableName=products" \ + -H "Authorization: Bearer {token}" + +# Restore +curl -X POST "https://{orchestrator-api-url}/api/restore" \ + -H "Authorization: Bearer {token}" \ + -H "Content-Type: application/json" \ + -d '{"tableName": "products", "type": "delta", "filename": "products_delta-2024-01-28T22-50-49Z.tar.gz"}' +``` + +## Links +- [Manticore Search Docs](https://manual.manticoresearch.com/) +- [Orchestrator, Agent, UI and Backup source code](https://github.com/controlplane-com/manticore-orchestrator) \ No newline at end of file diff --git a/manticore/versions/2.0.1/templates/_helpers.tpl b/manticore/versions/2.0.1/templates/_helpers.tpl new file mode 100644 index 00000000..c0326b1d --- /dev/null +++ b/manticore/versions/2.0.1/templates/_helpers.tpl @@ -0,0 +1,242 @@ +{{/* Resource Naming */}} + +{{/* +Manticore Workload Name +*/}} +{{- define "manticore.name" -}} +{{- printf "%s-manticore" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Job Workload Name +*/}} +{{- define "manticore.orchestratorJobName" -}} +{{- printf "%s-orchestrator-job" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator API Workload Name +*/}} +{{- define "manticore.orchestratorAPIName" -}} +{{- printf "%s-orchestrator-api" .Release.Name }} +{{- end }} + +{{/* +Manticore UI Workload Name +*/}} +{{- define "manticore.UIName" -}} +{{- printf "%s-ui" .Release.Name }} +{{- end }} + +{{/* +Manticore Backup Workload Name +*/}} +{{- define "manticore.backupName" -}} +{{- printf "%s-manticore-backup" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Workload Name +*/}} +{{- define "manticore.loadTestName" -}} +{{- printf "%s-load-test" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Controller Workload Name +*/}} +{{- define "manticore.loadTestControllerName" -}} +{{- printf "%s-load-test-controller" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Config Name +*/}} +{{- define "manticore.secretConfigName" -}} +{{- printf "%s-manticore-config" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Startup Name +*/}} +{{- define "manticore.secretStartupName" -}} +{{- printf "%s-manticore-startup" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Schema Config Name +*/}} +{{- define "manticore.secretSchemaConfigName" -}} +{{- printf "%s-manticore-schema" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Agent Token Name +*/}} +{{- define "manticore.secretAgentTokenName" -}} +{{- printf "%s-manticore-agent-token" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret K6 Script Name +*/}} +{{- define "manticore.secretK6ScriptName" -}} +{{- printf "%s-manticore-k6-script" .Release.Name }} +{{- end }} + +{{/* +Manticore Identity Name +*/}} +{{- define "manticore.identityName" -}} +{{- printf "%s-manticore-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Identity Name +*/}} +{{- define "manticore.orchestratorIdentityName" -}} +{{- printf "%s-manticore-orchestrator-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Job Identity Name +*/}} +{{- define "manticore.orchestratorJobIdentityName" -}} +{{- printf "%s-manticore-orchestrator-job-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Identity Name +*/}} +{{- define "manticore.loadTestIdentityName" -}} +{{- printf "%s-manticore-load-test-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Controller Identity Name +*/}} +{{- define "manticore.loadTestControllerIdentityName" -}} +{{- printf "%s-manticore-load-test-controller-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Backup Identity Name +*/}} +{{- define "manticore.backupIdentityName" -}} +{{- printf "%s-manticore-backup-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Config Policy Name +*/}} +{{- define "manticore.configPolicyName" -}} +{{- printf "%s-manticore-config-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Exec Policy Name +*/}} +{{- define "manticore.execPolicyName" -}} +{{- printf "%s-manticore-exec-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Policy Name +*/}} +{{- define "manticore.orchestratorPolicyName" -}} +{{- printf "%s-manticore-orchestrator-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Policy Name +*/}} +{{- define "manticore.loadTestPolicyName" -}} +{{- printf "%s-manticore-load-test-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Controller Policy Name +*/}} +{{- define "manticore.loadTestControllerPolicyName" -}} +{{- printf "%s-manticore-load-test-controller-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Volume Set Name +*/}} +{{- define "manticore.volumeName" -}} +{{- printf "%s-manticore-vs" .Release.Name }} +{{- end }} + +{{/* +Manticore Shared Volume Set Name +*/}} +{{- define "manticore.sharedVolumeName" -}} +{{- printf "%s-manticore-vs-shared" .Release.Name }} +{{- end }} + + +{{/* Functions */}} + +{{/* +Generate JSON mapping of table names to CSV paths for orchestrator. +csvPath accepts a single string or a list for multi-segment tables. +Output (single): {"addresses":"imports/addresses/data.csv"} +Output (multi): {"addresses":["imports/addresses/data_1.csv","imports/addresses/data_2.csv"]} +*/}} +{{- define "manticore.tablesConfigJSON" -}} +{{- $config := dict -}} +{{- range . -}} +{{- $_ := set $config .name .csvPath -}} +{{- end -}} +{{- $config | toJson -}} +{{- end }} + +{{/* +Validate that each table's csvPath length matches its config.segmentCount. +csvPath may be a single string (segmentCount must be 1) or a list (length must equal segmentCount). +*/}} +{{- define "manticore.validateTables" -}} +{{- range .Values.tables -}} +{{- $tableName := .name -}} +{{- $segmentCount := .config.segmentCount | int -}} +{{- if kindIs "slice" .csvPath -}} + {{- $csvCount := len .csvPath -}} + {{- if ne $csvCount $segmentCount -}} + {{- fail (printf "Table %q: csvPath has %d entries but segmentCount is %d — they must match." $tableName $csvCount $segmentCount) -}} + {{- end -}} +{{- else -}} + {{- if ne $segmentCount 1 -}} + {{- fail (printf "Table %q: csvPath is a single string but segmentCount is %d — it must be 1 when csvPath is a single value." $tableName $segmentCount) -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +Calculate total load test duration in seconds (duration + buffer) +Parses duration strings like "5m", "1h", "30s" +*/}} +{{- define "loadTest.totalDurationSeconds" -}} +{{- $duration := .Values.loadTest.duration -}} +{{- $buffer := .Values.loadTest.controller.testDurationBuffer | int -}} +{{- $seconds := 0 -}} +{{- if hasSuffix "s" $duration -}} + {{- $seconds = trimSuffix "s" $duration | int -}} +{{- else if hasSuffix "m" $duration -}} + {{- $seconds = mul (trimSuffix "m" $duration | int) 60 -}} +{{- else if hasSuffix "h" $duration -}} + {{- $seconds = mul (trimSuffix "h" $duration | int) 3600 -}} +{{- end -}} +{{- add $seconds $buffer -}} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels - delegated to cpln-common +*/}} +{{- define "manticore.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/manticore/versions/2.0.1/templates/domain.yaml b/manticore/versions/2.0.1/templates/domain.yaml new file mode 100644 index 00000000..aad6906a --- /dev/null +++ b/manticore/versions/2.0.1/templates/domain.yaml @@ -0,0 +1,34 @@ +# ============================================================================= +# External Domain (Optional) +# ============================================================================= +# Routes /api/* to orchestrator-api, /* to UI +{{- if .Values.domain.enabled }} +kind: domain +name: {{ .Values.domain.name }} +description: External domain for Manticore cluster UI and API +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + dnsMode: {{ .Values.domain.dnsMode }} + gvcLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }} + acceptAllHosts: false + ports: + - number: 443 + protocol: http2 + cors: + allowOrigins: + - exact: '*' + allowMethods: + - GET + - POST + - OPTIONS + allowHeaders: + - '*' + allowCredentials: true + routes: + - prefix: /api/ + workloadLink: //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.orchestratorAPIName" . }} + port: 8080 + - prefix: / + workloadLink: //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.UIName" . }} + port: 3000 +{{- end }} diff --git a/manticore/versions/2.0.1/templates/identity.yaml b/manticore/versions/2.0.1/templates/identity.yaml new file mode 100644 index 00000000..cad0607e --- /dev/null +++ b/manticore/versions/2.0.1/templates/identity.yaml @@ -0,0 +1,66 @@ +# ============================================================================= +# Workload Identities +# ============================================================================= +# Manticore identity - S3 access via AWS Cloud Account +kind: identity +name: {{ include "manticore.identityName" . }} +description: Manticore workload identity for secret access +tags: {{- include "manticore.tags" . | nindent 4 }} + +--- +# Orchestrator identity - secret access + CPLN API for cluster operations + s3 access +kind: identity +name: {{ include "manticore.orchestratorIdentityName" . }} +description: Orchestrator identity for secrets, CPLN API, and S3 access +tags: {{- include "manticore.tags" . | nindent 4 }} +{{- if .Values.orchestrator.backup.enabled }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.orchestrator.backup.cloudAccountName }} + policyRefs: + {{- range .Values.orchestrator.backup.s3Policy }} + - {{ . }} + {{- end }} +{{- end }} + +--- +# Orchestrator job identity - secret access + S3 access for cron operations +kind: identity +name: {{ include "manticore.orchestratorJobIdentityName" . }} +description: Orchestrator job identity for secrets and S3 access +tags: {{- include "manticore.tags" . | nindent 4 }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.buckets.cloudAccountName }} + policyRefs: + {{- range .Values.buckets.awsPolicyRefs }} + - {{ . }} + {{- end }} + +{{- if .Values.orchestrator.backup.enabled }} +--- +# Backup identity - S3 access for backups +kind: identity +name: {{ include "manticore.backupIdentityName" . }} +description: Manticore backup identity for S3 access +tags: {{- include "manticore.tags" . | nindent 4 }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.orchestrator.backup.cloudAccountName }} + policyRefs: + {{- range .Values.orchestrator.backup.s3Policy }} + - {{ . }} + {{- end }} +{{- end }} + +{{- if .Values.loadTest.enabled }} +--- +# Load test identity - access to k6 script secret +kind: identity +name: {{ include "manticore.loadTestIdentityName" . }} +description: Load test workload identity for script access +tags: {{- include "manticore.tags" . | nindent 4 }} +--- +# Load test controller identity - workload scaling permissions +kind: identity +name: {{ include "manticore.loadTestControllerIdentityName" . }} +description: Controller identity for scaling load-test workload +tags: {{- include "manticore.tags" . | nindent 4 }} +{{- end }} diff --git a/manticore/versions/2.0.1/templates/policy.yaml b/manticore/versions/2.0.1/templates/policy.yaml new file mode 100644 index 00000000..78c046f7 --- /dev/null +++ b/manticore/versions/2.0.1/templates/policy.yaml @@ -0,0 +1,99 @@ +# ============================================================================= +# Access Policies +# ============================================================================= +# Secret access policy - grants reveal on config secrets +kind: policy +name: {{ include "manticore.configPolicyName" . }} +description: Secret access for manticore and orchestrator workloads +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.identityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorJobIdentityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.backupIdentityName" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "manticore.secretConfigName" . }} + - //secret/{{ include "manticore.secretStartupName" . }} + - //secret/{{ include "manticore.secretSchemaConfigName" . }} + - //secret/{{ include "manticore.secretAgentTokenName" . }} +targetQuery: + kind: secret + fetch: items + spec: + match: all + terms: [] +--- +# Cron execution policy - allows triggering orchestrator jobs +kind: policy +name: {{ include "manticore.execPolicyName" . }} +description: Permission to trigger orchestrator cron workload executions +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - view + - edit + - exec.runCronWorkload + principalLinks: + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.identityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} +targetKind: workload +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.orchestratorJobName" . }} + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.name" . }} + {{- if .Values.orchestrator.backup.enabled }} + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.backupName" . }} + {{- end }} + +--- +# Workload view policy - allows orchestrator to read Manticore workload config +kind: policy +name: {{ include "manticore.orchestratorPolicyName" . }} +description: Permission to view Manticore workload configuration +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - view + - edit + principalLinks: + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorJobIdentityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.backupIdentityName" . }} +targetKind: workload +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.name" . }} + +{{- if .Values.loadTest.enabled }} +--- +# Load test script access +kind: policy +name: {{ include "manticore.loadTestPolicyName" . }} +description: K6 workload access to load test script secret +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - use + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestIdentityName" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "manticore.secretK6ScriptName" . }} +--- +# Load test controller policy - allows scaling k6 workload +kind: policy +name: {{ include "manticore.loadTestControllerPolicyName" . }} +description: Controller permission to scale load-test workload +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - manage + - edit + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestControllerIdentityName" . }} +targetKind: workload +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.loadTestName" . }} +{{- end }} diff --git a/manticore/versions/2.0.1/templates/secret.yaml b/manticore/versions/2.0.1/templates/secret.yaml new file mode 100644 index 00000000..b8deb254 --- /dev/null +++ b/manticore/versions/2.0.1/templates/secret.yaml @@ -0,0 +1,330 @@ +{{- include "manticore.validateTables" . -}} +# ============================================================================= +# Manticore Search Configuration +# ============================================================================= +# Base searchd config (startup script generates runtime config with IP-bound listeners) +kind: secret +name: {{ include "manticore.secretConfigName" . }} +description: Manticore searchd base configuration +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + searchd { + listen = 9306:mysql + listen = 9308:http + listen = 9312 + data_dir = /var/lib/manticore + binlog_path = /var/lib/manticore/binlog + log = /dev/stdout + query_log = /dev/stdout + pid_file = /var/run/searchd.pid + seamless_rotate = 1 + preopen_tables = 1 + } +--- +# ============================================================================= +# Schema Registry +# ============================================================================= +# Table schemas for agent (creates RT delta tables, parses CSV, builds distributed tables) +kind: secret +name: {{ include "manticore.secretSchemaConfigName" . }} +description: Schema registry for multi-table configuration +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + # Schema Registry (YAML format, read by agent) + # See README.md for column types and configuration options + {{- range .Values.tables }} + + {{ .name }}: + config: + {{- .config | toYaml | nindent 8 }} + schema: + columns: + {{- range .schema.columns }} + - name: {{ .name }} + type: {{ .type }} + {{- end }} + {{- end }} +--- +# ============================================================================= +# Manticore Startup Script +# ============================================================================= +# Generates runtime config, starts searchd, handles graceful shutdown +# Cluster init (bootstrap/join) handled by agent via orchestrator API +kind: secret +name: {{ include "manticore.secretStartupName" . }} +description: Manticore searchd startup and shutdown handler +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env bash + set -euo pipefail + + # ============================================================================= + # CONFIGURATION + # ============================================================================= + + CLUSTER_NAME="{{ .Values.manticore.clusterName }}" + MYSQL_PORT=9306 + HTTP_PORT=9308 + REPL_PORT=9312 + WORKLOAD_NAME="$(echo "${HOSTNAME}" | sed 's/-[0-9]*$//')" + REPLICA_INDEX="$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}')" + LOCATION=$(basename "${CPLN_LOCATION}") + # Internal DNS format: {workloadName}-{replicaIndex}.{workloadName} + NODE0_FQDN="${WORKLOAD_NAME}-0.${WORKLOAD_NAME}" + NODE0_ADDR="${NODE0_FQDN}:${REPL_PORT}" + + echo "============================================" + echo "Manticore Startup" + echo "============================================" + echo "Hostname: ${HOSTNAME}" + echo "Replica Index: ${REPLICA_INDEX}" + echo "Cluster: ${CLUSTER_NAME}" + echo "Node 0 FQDN: ${NODE0_FQDN}" + echo "============================================" + echo "" + + # Create required directories + echo "Creating directories..." + mkdir -p /var/lib/manticore/binlog + mkdir -p /var/lib/manticore/data + echo "Directories created." + + RUNTIME_CONFIG="/tmp/manticore-runtime.conf" + + # ============================================================================= + # HELPER FUNCTIONS + # ============================================================================= + + mysql_exec() { + local host="${1:-127.0.0.1}" + shift || true + mysql --protocol=tcp -h "${host}" -P "${MYSQL_PORT}" -N -B "$@" + } + + wait_for_manticore() { + echo "Waiting for Manticore MySQL port ${MYSQL_PORT}..." + for i in $(seq 1 60); do + if mysql --protocol=tcp -h 127.0.0.1 -P "${MYSQL_PORT}" -e "SELECT 1" >/dev/null 2>&1; then + echo "Manticore is ready." + return 0 + fi + echo " [$i/60] not ready yet..." + sleep 1 + done + echo "ERROR: Manticore did not become ready on ${MYSQL_PORT}" + return 1 + } + + # ============================================================================= + # GRACEFUL SHUTDOWN HANDLER + # ============================================================================= + + shutdown() { + echo "" + echo "============================================" + echo "SIGTERM received: graceful shutdown" + echo "============================================" + + echo "Stopping searchd gracefully..." + if searchd --stopwait --config "${RUNTIME_CONFIG}"; then + echo "searchd stopped gracefully" + else + echo "searchd --stopwait failed, falling back to kill" + if [[ -n "${SEARCHD_PID:-}" ]] && kill -0 "$SEARCHD_PID" >/dev/null 2>&1; then + kill "$SEARCHD_PID" || true + for _ in $(seq 1 30); do + kill -0 "$SEARCHD_PID" >/dev/null 2>&1 || break + sleep 1 + done + fi + fi + + # Keep manticore.json on all replicas so they remember cluster state and + # table associations. This allows IST (incremental sync) instead of full SST + # when rejoining, and ensures local tables are recognized as cluster tables. + if [[ "${REPLICA_INDEX}" != "0" ]]; then + echo "Requesting cluster node refresh on replica-0..." + mysql_exec "${NODE0_FQDN}" -e "ALTER CLUSTER ${CLUSTER_NAME} UPDATE nodes" || true + fi + + echo "Shutdown sequence complete." + echo "============================================" + } + + trap shutdown TERM INT + + # ============================================================================= + # START MANTICORE + # ============================================================================= + + echo "" + echo "============================================" + echo "Preparing searchd configuration..." + echo "============================================" + + # Get the pod's IP address and FQDN for replication binding + MY_IP=$(hostname -i | awk '{print $1}') + MY_FQDN="${WORKLOAD_NAME}-${REPLICA_INDEX}.${WORKLOAD_NAME}" + echo "Pod IP address: ${MY_IP}" + echo "Pod FQDN: ${MY_FQDN}" + + # Create runtime config with dynamic replication listener + echo "Generating runtime config: ${RUNTIME_CONFIG}" + + echo "searchd {" > "${RUNTIME_CONFIG}" + echo " listen = 127.0.0.1:9306:mysql" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9306:mysql" >> "${RUNTIME_CONFIG}" + echo " listen = 127.0.0.1:9308:http" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9308:http" >> "${RUNTIME_CONFIG}" + echo " listen = 127.0.0.1:9312" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9312" >> "${RUNTIME_CONFIG}" + echo " listen = 0.0.0.0:9322-9323:replication" >> "${RUNTIME_CONFIG}" + + echo " node_address = ${MY_FQDN}" >> "${RUNTIME_CONFIG}" + echo " data_dir = /var/lib/manticore" >> "${RUNTIME_CONFIG}" + echo " binlog_path = /var/lib/manticore/binlog" >> "${RUNTIME_CONFIG}" + echo " log = /tmp/searchd.log" >> "${RUNTIME_CONFIG}" + echo " query_log = /dev/stdout" >> "${RUNTIME_CONFIG}" + echo " pid_file = /var/run/searchd.pid" >> "${RUNTIME_CONFIG}" + echo " seamless_rotate = 1" >> "${RUNTIME_CONFIG}" + echo " preopen_tables = 1" >> "${RUNTIME_CONFIG}" + echo " server_id = ${REPLICA_INDEX}" >> "${RUNTIME_CONFIG}" + echo " buddy_path = /usr/share/manticore/modules/manticore-buddy/bin/manticore-buddy" >> "${RUNTIME_CONFIG}" + echo "}" >> "${RUNTIME_CONFIG}" + echo "common {" >> "${RUNTIME_CONFIG}" + echo " plugin_dir = /usr/share/manticore/modules" >> "${RUNTIME_CONFIG}" + echo "}" >> "${RUNTIME_CONFIG}" + + echo "Runtime config generated:" + cat "${RUNTIME_CONFIG}" + echo "" + + chmod +x /usr/share/manticore/modules/manticore-buddy/bin/manticore-buddy + + echo "============================================" + echo "Starting searchd..." + echo "============================================" + + echo "" + echo "Launching searchd in background..." + touch /tmp/searchd.log + tail -f /tmp/searchd.log & + searchd --config "${RUNTIME_CONFIG}" --nodetach 2>&1 & + SEARCHD_PID="$!" + echo "searchd launched with PID: ${SEARCHD_PID}" + + sleep 2 + + if ! kill -0 "${SEARCHD_PID}" 2>/dev/null; then + echo "ERROR: searchd exited immediately! Check config." + exit 1 + fi + + echo "searchd is running. Waiting for MySQL port..." + wait_for_manticore + + # ============================================================================= + # CLUSTER STATUS CHECK (cluster setup is handled by orchestrator init) + # ============================================================================= + + echo "" + echo "============================================" + echo "Cluster Status" + echo "============================================" + + # Check if already part of a cluster (from preserved state in manticore.json) + # Wait a few seconds for cluster module to load state + sleep 3 + CLUSTER_STATUS=$(mysql_exec 127.0.0.1 -e "SHOW STATUS LIKE 'cluster_${CLUSTER_NAME}_status'" 2>/dev/null || echo "") + + if echo "${CLUSTER_STATUS}" | grep -qE "(primary|synced)"; then + echo "Already part of cluster ${CLUSTER_NAME}." + else + echo "Not part of any cluster. Run orchestrator init to bootstrap/join cluster." + fi + + # ============================================================================= + # KEEP CONTAINER ALIVE + # ============================================================================= + + echo "" + echo "============================================" + echo "Startup complete" + echo "============================================" + echo "searchd PID: ${SEARCHD_PID}" + echo "============================================" + echo "" + echo "Import operations are handled by the agent sidecar." + echo "Use the orchestrator workload for coordinated imports." + echo "" + + wait "${SEARCHD_PID}" +--- +# ============================================================================= +# Agent Authentication Token +# ============================================================================= +# Bearer token for orchestrator/agent/UI communication +# Generate with: openssl rand -base64 32 +kind: secret +name: {{ include "manticore.secretAgentTokenName" . }} +description: Bearer token for orchestrator-agent authentication +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: {{ required "agent.token is required" .Values.orchestrator.agent.token }} + +# ============================================================================= +# K6 Load Test Script +# ============================================================================= +{{- if .Values.loadTest.enabled }} +--- +# Generated k6 script from loadTest.* values +kind: secret +name: {{ include "manticore.secretK6ScriptName" . }} +description: K6 load test script for Manticore search +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + import http from 'k6/http'; + import { check } from 'k6'; + import { Rate } from 'k6/metrics'; + + const errorRate = new Rate('errors'); + + export const options = { + vus: {{ .Values.loadTest.vus }}, + duration: '{{ .Values.loadTest.duration }}', + thresholds: { + 'http_req_duration': ['p(95)<{{ .Values.loadTest.thresholds.p95ResponseTime }}'], + 'http_req_failed': ['rate<{{ .Values.loadTest.thresholds.errorRate }}'], + }, + }; + + const BASE_URL = 'http://{{ include "manticore.name" . }}.{{ .Values.global.cpln.gvc }}.cpln.local:{{ .Values.loadTest.target.port }}'; + + const QUERY = JSON.stringify({{ .Values.loadTest.query | toJson }}); + + export default function () { + const res = http.post(`${BASE_URL}/{{ .Values.loadTest.target.endpoint }}`, QUERY, { + headers: { 'Content-Type': 'application/json' }, + }); + + const success = check(res, { + 'status is 200': (r) => r.status === 200, + }); + + errorRate.add(!success); + } +{{- end }} diff --git a/manticore/versions/2.0.1/templates/volumeset-shared.yaml b/manticore/versions/2.0.1/templates/volumeset-shared.yaml new file mode 100644 index 00000000..77934a40 --- /dev/null +++ b/manticore/versions/2.0.1/templates/volumeset-shared.yaml @@ -0,0 +1,11 @@ +# ============================================================================= +# Shared Volumeset +# ============================================================================= +# Shared storage across replicas and orchestrator +kind: volumeset +name: {{ include "manticore.sharedVolumeName" . }} +description: Shared storage across Manticore replicas and orchestrator +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + fileSystemType: shared + initialCapacity: {{ .Values.manticore.sharedVolumeset.capacity }} diff --git a/manticore/versions/2.0.1/templates/volumeset.yaml b/manticore/versions/2.0.1/templates/volumeset.yaml new file mode 100644 index 00000000..7935d42b --- /dev/null +++ b/manticore/versions/2.0.1/templates/volumeset.yaml @@ -0,0 +1,15 @@ +# ============================================================================= +# Manticore Volumeset +# ============================================================================= +# Persistent storage per replica (data, binlog, cluster state) +kind: volumeset +name: {{ include "manticore.volumeName" . }} +description: Persistent storage for Manticore data and cluster state +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.manticore.volumeset.capacity }} + performanceClass: general-purpose-ssd + snapshots: + createFinalSnapshot: true + retentionDuration: 7d \ No newline at end of file diff --git a/manticore/versions/2.0.1/templates/workload.yaml b/manticore/versions/2.0.1/templates/workload.yaml new file mode 100644 index 00000000..74055172 --- /dev/null +++ b/manticore/versions/2.0.1/templates/workload.yaml @@ -0,0 +1,566 @@ +# ============================================================================= +# Manticore Search Workload (Stateful) +# ============================================================================= +# Each replica runs manticore (searchd) + agent sidecar for orchestrator coordination +kind: workload +name: {{ include "manticore.name" . }} +description: Manticore search cluster with agent sidecar +tags: {{- include "manticore.tags" . | nindent 4 }} + cpln/publishNotReadyAddresses: "true" +spec: + type: stateful + containers: + - name: manticore + cpu: {{ .Values.manticore.resources.cpu | quote }} + image: {{ .Values.manticore.image }} + command: "/bin/bash" + args: + - "/usr/local/bin/start.sh" + inheritEnv: false + memory: {{ .Values.manticore.resources.memory | quote }} + metrics: + path: "/metrics" + port: 9308 + ports: + - number: 9306 + protocol: tcp + - number: 9308 + protocol: http + - number: 9312 + protocol: tcp + # Galera gcomm + IST ports + - number: 9322 + protocol: tcp + - number: 9323 + protocol: tcp + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 15 + successThreshold: 1 + tcpSocket: + port: 9306 + timeoutSeconds: 10 + volumes: + - path: /var/lib/manticore + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.volumeName" . }} + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + - path: /etc/manticore/manticore.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretConfigName" . }}.payload + - path: /etc/manticore/schema.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretSchemaConfigName" . }}.payload + - path: /usr/local/bin/start.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretStartupName" . }}.payload + - name: agent + cpu: {{ .Values.orchestrator.agent.resources.cpu | quote }} + minCpu: {{ .Values.orchestrator.agent.resources.minCpu | default .Values.orchestrator.agent.resources.cpu | quote }} + image: {{ .Values.orchestrator.agent.image }}:{{ .Values.orchestrator.agent.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.agent.resources.memory | quote }} + minMemory: {{ .Values.orchestrator.agent.resources.minMemory | default .Values.orchestrator.agent.resources.memory | quote }} + ports: + - number: 8080 + protocol: http + env: + - name: MYSQL_HOST + value: "127.0.0.1" + - name: MYSQL_PORT + value: "9306" + - name: SCHEMA_FILE + value: "/etc/manticore/schema.conf" + - name: CLUSTER_NAME + value: "{{ .Values.manticore.clusterName }}" + - name: LISTEN_ADDR + value: ":8080" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + # Cluster recovery settings + - name: WORKLOAD_NAME + value: "{{ include "manticore.name" . }}" + - name: DATA_DIR + value: "/var/lib/manticore" + - name: MAX_SCALE + value: "{{ .Values.manticore.autoscaling.maxScale }}" + - name: RECOVERY_MAX_RETRIES + value: "{{ .Values.orchestrator.agent.recovery.maxRetries | default 5 }}" + - name: RECOVERY_INITIAL_BACKOFF_SEC + value: "{{ .Values.orchestrator.agent.recovery.initialBackoffSec | default 5 }}" + - name: RECOVERY_MAX_BACKOFF_SEC + value: "{{ .Values.orchestrator.agent.recovery.maxBackoffSec | default 60 }}" + - name: REPLICATION_PORT + value: "9312" + - name: IMPORT_BATCH_SIZE + value: "{{ .Values.orchestrator.agent.import.batchSize | default 1000 }}" + - name: ORCHESTRATOR_API_URL + value: "http://{{ include "manticore.orchestratorAPIName" . }}.{{ .Values.global.cpln.gvc }}.cpln.local:8080" + readinessProbe: + failureThreshold: 20 + initialDelaySeconds: 10 + periodSeconds: 15 + successThreshold: 1 + httpGet: + path: /api/ready + port: 8080 + timeoutSeconds: 5 + volumes: + - path: /etc/manticore/schema.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretSchemaConfigName" . }}.payload + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + - path: /var/lib/manticore + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.volumeName" . }} + defaultOptions: + autoscaling: + maxScale: {{ .Values.manticore.autoscaling.maxScale }} + metric: {{ .Values.manticore.autoscaling.metric }} + minScale: {{ .Values.manticore.autoscaling.minScale }} + scaleToZeroDelay: {{ .Values.manticore.autoscaling.scaleToZeroDelay }} + target: {{ .Values.manticore.autoscaling.target }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowHostname: [] + internal: + inboundAllowType: {{ .Values.manticore.firewall.internalAccess.type }} + {{- if .Values.manticore.firewall.internalAccess.workloads }} + inboundAllowWorkload: {{ .Values.manticore.firewall.internalAccess.workloads | toYaml | nindent 8 }} + {{- end }} + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.identityName" . }} + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + rolloutOptions: + maxSurgeReplicas: {{ .Values.manticore.rolloutOptions.maxSurgeReplicas }} + maxUnavailableReplicas: '{{ .Values.manticore.rolloutOptions.maxUnavailableReplicas }}' + minReadySeconds: {{ .Values.manticore.rolloutOptions.minReadySeconds }} + scalingPolicy: {{ .Values.manticore.rolloutOptions.scalingPolicy }} + terminationGracePeriodSeconds: {{ .Values.manticore.rolloutOptions.terminationGracePeriodSeconds }} + supportDynamicTags: false +--- +# ============================================================================= +# Orchestrator Cron Workload +# ============================================================================= +# Executes scheduled/manual actions: init, import, health, repair +# Starts suspended by default (trigger via UI/API) +kind: workload +name: {{ include "manticore.orchestratorJobName" . }} +description: Manticore orchestrator for scheduled/manual cluster operations +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: cron + containers: + - name: orchestrator + cpu: {{ .Values.orchestrator.resources.cpu | quote }} + image: {{ .Values.orchestrator.image }}:{{ .Values.orchestrator.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.resources.memory | quote }} + env: + - name: MODE + value: "cli" + - name: ACTION + value: "{{ .Values.orchestrator.action }}" + - name: REPLICA_COUNT + value: "{{ .Values.manticore.autoscaling.minScale }}" + - name: AGENT_PORT + value: "8080" + - name: WORKLOAD_NAME + value: "{{ include "manticore.name" . }}" + - name: GVC + value: "{{ .Values.global.cpln.gvc }}" + - name: LOCATION + value: "{{ .Values.global.cpln.location }}" + - name: TABLE_NAME + value: "{{ .Values.orchestrator.tableName }}" + - name: TABLES_CONFIG + value: '{{ include "manticore.tablesConfigJSON" .Values.tables }}' + - name: STATE_FILE + value: "/tmp/orchestrator_state.json" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + - name: LOG_LEVEL + value: "{{ .Values.orchestrator.logLevel }}" + - name: IMPORT_MEM_LIMIT + value: "{{ .Values.orchestrator.importMemLimit }}" + - name: INDEXER_WORK_DIR + value: "/mnt/s3/indexer-temp" + - name: S3_MOUNT + value: "/mnt/s3" + - name: SHARED_VOLUME_MOUNT + value: "/mnt/shared" + volumes: + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + - path: /mnt/s3 + uri: s3://{{ .Values.buckets.sourceBucket }} + recoveryPolicy: retain + defaultOptions: + autoscaling: + maxScale: 1 + minScale: 1 + capacityAI: false + debug: false + suspend: {{ .Values.orchestrator.suspend }} + timeoutSeconds: {{ .Values.orchestrator.timeoutSeconds }} + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + internal: + inboundAllowType: none + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorJobIdentityName" . }} + job: + schedule: {{ .Values.orchestrator.schedule | quote }} + concurrencyPolicy: Forbid + restartPolicy: Never + activeDeadlineSeconds: {{ .Values.orchestrator.activeDeadlineSeconds }} + supportDynamicTags: false +--- +# ============================================================================= +# Orchestrator API Workload (Standard) +# ============================================================================= +# REST API for cluster coordination (init, import, repair, cluster status) +# Backend for UI dashboard, communicates with agents via bearer token +kind: workload +name: {{ include "manticore.orchestratorAPIName" . }} +description: Manticore orchestrator REST API for cluster coordination +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: api + cpu: {{ .Values.orchestrator.api.resources.cpu | quote }} + image: {{ .Values.orchestrator.api.image }}:{{ .Values.orchestrator.api.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.api.resources.memory | quote }} + port: 8080 + readinessProbe: + httpGet: + path: /api/status + port: 8080 + periodSeconds: 10 + failureThreshold: 3 + env: + - name: MODE + value: "server" + - name: LISTEN_ADDR + value: ":8080" + - name: REPLICA_COUNT + value: "{{ .Values.manticore.autoscaling.minScale }}" + - name: AGENT_PORT + value: "8080" + - name: WORKLOAD_NAME + value: "{{ include "manticore.name" . }}" + # CPLN_GVC, CPLN_LOCATION auto-injected by Control Plane + - name: TABLES_CONFIG + value: '{{ include "manticore.tablesConfigJSON" .Values.tables }}' + - name: STATE_FILE + value: "/tmp/orchestrator_state.json" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + - name: LOG_LEVEL + value: "{{ .Values.orchestrator.logLevel }}" + # CPLN_TOKEN, CPLN_ORG, CPLN_GVC auto-injected by Control Plane + - name: ORCHESTRATOR_WORKLOAD + value: "{{ include "manticore.orchestratorJobName" . }}" + - name: IMPORT_POLL_INTERVAL + value: "{{ .Values.orchestrator.api.importPollInterval | default "30s" }}" + - name: IMPORT_POLL_TIMEOUT + value: "{{ .Values.orchestrator.api.importPollTimeout | default "2h" }}" + {{- if .Values.orchestrator.backup.enabled }} + - name: BACKUP_SCHEDULES + value: '{{ .Values.orchestrator.backup.schedules | toJson }}' + - name: BACKUP_BUCKET + value: {{ .Values.orchestrator.backup.s3Bucket }} + - name: BACKUP_PREFIX + value: {{ .Values.orchestrator.backup.prefix }} + - name: BACKUP_PROVIDER + value: aws + - name: BACKUP_REGION + value: {{ .Values.orchestrator.backup.s3Region }} + {{- end }} + defaultOptions: + autoscaling: + maxScale: {{ .Values.orchestrator.api.autoscaling.maxScale }} + minScale: {{ .Values.orchestrator.api.autoscaling.minScale }} + metric: {{ .Values.orchestrator.api.autoscaling.metric }} + target: {{ .Values.orchestrator.api.autoscaling.target }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 30 + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: same-gvc + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} + supportDynamicTags: false +--- +# ============================================================================= +# Web UI Workload (Standard) +# ============================================================================= +# Dashboard for cluster monitoring, table status, imports, and repairs +kind: workload +name: {{ include "manticore.UIName" . }} +description: Manticore cluster management dashboard +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: ui + cpu: {{ .Values.orchestrator.ui.resources.cpu | quote }} + image: {{ .Values.orchestrator.ui.image }}:{{ .Values.orchestrator.ui.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.ui.resources.memory | quote }} + port: 3000 + env: + - name: ORCHESTRATOR_API_URL + value: "http://{{ include "manticore.orchestratorAPIName" . }}.{{ .Values.global.cpln.gvc }}.cpln.local:8080" + - name: ORCHESTRATOR_AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + - name: PORT + value: "3000" + readinessProbe: + httpGet: + path: / + port: 3000 + periodSeconds: 10 + failureThreshold: 3 + defaultOptions: + autoscaling: + maxScale: {{ .Values.orchestrator.ui.autoscaling.maxScale }} + minScale: {{ .Values.orchestrator.ui.autoscaling.minScale }} + metric: {{ .Values.orchestrator.ui.autoscaling.metric }} + target: {{ .Values.orchestrator.ui.autoscaling.target }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 30 + firewallConfig: + external: + inboundAllowCIDR: {{ if .Values.orchestrator.ui.allowExternalAccess }}[ "0.0.0.0/0" ]{{ else }}[ ]{{ end }} + outboundAllowCIDR: [] + internal: + inboundAllowType: same-gvc + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} + supportDynamicTags: false + +{{- if .Values.orchestrator.backup.enabled }} +--- +# ============================================================================= +# Backup Workload (Cron) +# ============================================================================= +# Cron job for logical backups on delta table to S3 bucket +kind: workload +name: {{ include "manticore.backupName" . }} +description: manticore backup +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: cron + containers: + - name: backup + cpu: {{ .Values.orchestrator.backup.resources.cpu | quote }} + env: + - name: ACTION + value: backup + - name: TYPE + value: delta + - name: BACKUP_BUCKET + value: {{ .Values.orchestrator.backup.s3Bucket }} + - name: BACKUP_PREFIX + value: {{ .Values.orchestrator.backup.prefix }} + - name: BACKUP_PROVIDER + value: aws + - name: BACKUP_REGION + value: {{ .Values.buckets.awsRegion }} + - name: DATASET + value: {{ .Values.orchestrator.backup.dataSet }} + - name: MANTICORE_HOST + value: "{{ include "manticore.name" . }}" + - name: MANTICORE_PORT + value: "9306" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + image: {{ .Values.orchestrator.backup.image }}:{{ .Values.orchestrator.backup.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.backup.resources.memory | quote }} + ports: + - number: 8080 + protocol: http + volumes: + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + defaultOptions: + autoscaling: + maxScale: 1 + metric: disabled + minScale: 1 + capacityAI: false + debug: false + multiZone: + enabled: false + suspend: true + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.backupIdentityName" . }} + job: + concurrencyPolicy: Forbid + historyLimit: 5 + restartPolicy: Never + schedule: '0 2 * * *' + activeDeadlineSeconds: {{ .Values.orchestrator.backup.activeDeadlineSeconds }} + localOptions: [] + supportDynamicTags: false +{{- end }} + +# ============================================================================= +# K6 Load Test Workload (Standard) +# ============================================================================= +# Runs k6 load tests (starts at scale 0, scaled up by controller) +{{- if .Values.loadTest.enabled }} +--- +kind: workload +name: {{ include "manticore.loadTestName" . }} +description: K6 load test runner for Manticore search +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: k6 + image: {{ .Values.loadTest.image }} + cpu: {{ .Values.loadTest.resources.cpu | quote }} + memory: {{ .Values.loadTest.resources.memory | quote }} + inheritEnv: false + command: /bin/sh + args: + - '-c' + - >- + k6 run + --vus {{ .Values.loadTest.vus }} + --duration {{ .Values.loadTest.duration }} + {{- if .Values.loadTest.rps }} + --rps {{ .Values.loadTest.rps }} + {{- end }} + /scripts/payload + volumes: + - path: /scripts + recoveryPolicy: retain + uri: 'cpln://secret/{{ include "manticore.secretK6ScriptName" . }}.payload' + defaultOptions: + autoscaling: + maxScale: 0 + minScale: 0 + scaleToZeroDelay: 30 + target: 95 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 3600 + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: none + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestIdentityName" . }} + supportDynamicTags: false +--- +# ============================================================================= +# Load Test Controller (Cron) +# ============================================================================= +# Scales k6 workload up, waits for test duration, then scales back to 0 +# Suspended if no schedule (manual trigger only) +kind: workload +name: {{ include "manticore.loadTestControllerName" . }} +description: Controller for triggering and managing load tests +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: cron + containers: + - name: controller + image: {{ .Values.loadTest.controller.image }} + cpu: '0.25' + memory: 256Mi + inheritEnv: false + command: /bin/sh + args: + - '-c' + - | + set -e + WORKLOAD="{{ include "manticore.loadTestName" . }}" + REPLICAS="{{ .Values.loadTest.replicas }}" + + echo "Scaling $WORKLOAD to $REPLICAS replicas..." + curl -sf -X PATCH \ + -H "Authorization: Bearer $CPLN_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"spec\":{\"defaultOptions\":{\"autoscaling\":{\"minScale\":$REPLICAS,\"maxScale\":$REPLICAS}}}}" \ + "http://api.cpln.io/org/$CPLN_ORG/gvc/$CPLN_GVC/workload/$WORKLOAD" + + echo "Waiting for test duration + buffer..." + sleep {{ include "loadTest.totalDurationSeconds" . }} + + echo "Scaling $WORKLOAD back to 0..." + curl -sf -X PATCH \ + -H "Authorization: Bearer $CPLN_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"spec":{"defaultOptions":{"autoscaling":{"minScale":0,"maxScale":0}}}}' \ + "http://api.cpln.io/org/$CPLN_ORG/gvc/$CPLN_GVC/workload/$WORKLOAD" + + echo "Load test complete." + defaultOptions: + autoscaling: + maxScale: 1 + minScale: 1 + capacityAI: false + debug: false + suspend: {{ if .Values.loadTest.controller.schedule }}false{{ else }}true{{ end }} + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: none + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestControllerIdentityName" . }} + job: + schedule: '{{ .Values.loadTest.controller.schedule | default "0 0 1 1 *" }}' + concurrencyPolicy: Forbid + restartPolicy: Never + historyLimit: 10 + activeDeadlineSeconds: 7200 + supportDynamicTags: false +{{- end }} diff --git a/manticore/versions/2.0.1/values.yaml b/manticore/versions/2.0.1/values.yaml new file mode 100644 index 00000000..ebad6a68 --- /dev/null +++ b/manticore/versions/2.0.1/values.yaml @@ -0,0 +1,220 @@ +# ============================================================================= +# AWS Cloud Account and S3 Configuration +# ============================================================================= +# See README.md for Cloud Account and IAM policy setup instructions. +buckets: + cloudAccountName: my-cloudaccount # Name of your configured Cloud Account + awsPolicyRefs: # IAM policies for S3 access + - my-manticore-policy # Note: if using a custom policy, omit the aws:: prefix as this is only for AWS managed policies + awsRegion: us-east-1 # Region of your S3 bucket + sourceBucket: my-manticore-bucket # S3 bucket containing files to import + +# ============================================================================= +# Tables Configuration +# ============================================================================= +# See README.md for table structure, column types, and config options. +# Add/remove tables as needed. +tables: + - name: addresses + csvPath: + - imports/addresses.csv + config: + haStrategy: noerrors + agentRetryCount: 3 + clusterMain: false + segmentCount: 1 + charsetTable: non_cont + memLimit: 2G + hasHeader: true + secondaryIndexes: false + schema: + columns: + - name: address_id + type: attr_uint + - name: street_number + type: attr_uint + - name: street_name + type: field + - name: city + type: field + - name: county + type: field + - name: state + type: field + - name: postal_code + type: field + - name: country + type: field + - name: latitude + type: attr_float + - name: longitude + type: attr_float + +# ============================================================================= +# Manticore Search Configuration +# ============================================================================= +manticore: + image: manticoresearch/manticore:25.0.0 + clusterName: manticore # Galera cluster name + resources: + cpu: 4 + memory: 8Gi + volumeset: + capacity: 200 # GB per replica + sharedVolumeset: + capacity: 100 # GB shared across replicas and orchestrator + + # minScale = replica count used by orchestrator for coordination + autoscaling: + minScale: 3 + maxScale: 4 + metric: rps # rps for stateful, cpu for standard + target: 100 + scaleToZeroDelay: 300 + + rolloutOptions: + maxSurgeReplicas: 25% + maxUnavailableReplicas: '0' + minReadySeconds: 10 + scalingPolicy: OrderedReady + terminationGracePeriodSeconds: 60 + + # IMPORTANT: same-gvc required for Galera replication + firewall: + internalAccess: + type: same-gvc + workloads: [] + +# ============================================================================= +# Orchestrator Configuration +# ============================================================================= +orchestrator: + version: v6.0.5 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api + logLevel: debug # debug, info, warn, error + resources: + cpu: 1 + memory: 2Gi + + # Cron job settings + schedule: "0 * * * *" # Cron schedule (default = every hour) + action: import # init, import, health, repair + tableName: addresses # Must match a name in tables[] + suspend: true # Start suspended (trigger via UI/API) + timeoutSeconds: 900 # Container timeout (seconds, default 15 minutes) + importMemLimit: 2G # Memory limit for import jobs + activeDeadlineSeconds: 14400 # Max job runtime (seconds, default 4 hours) + + # Orchestrator API + api: + version: v6.0.5 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api + logLevel: debug + importPollInterval: 30s + importPollTimeout: 2h + resources: + cpu: 0.25 + memory: 256Mi + autoscaling: + maxScale: 3 + minScale: 2 + metric: cpu + target: 80 + + # Agent sidecar + agent: + version: v6.0.5 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-agent + # REQUIRED: Generate with `openssl rand -base64 32` + token: "6Gl5uO9KkKAh1u+ymoBW98WCtjTFpljuhpLdKb+tNAA=" + resources: + cpu: 250m + minCpu: 100m + memory: 512Mi + minMemory: 128Mi + import: + batchSize: 20000 # Rows per INSERT statement + recovery: + maxRetries: 5 # Retry attempts for cluster recovery + initialBackoffSec: 5 # Initial delay between retries + maxBackoffSec: 60 # Max backoff delay (exponential) + + # Web UI - See README.md "Authentication" section for security notes + ui: + version: v6.0.5 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-ui + resources: + cpu: 0.25 + memory: 0.25Gi + allowExternalAccess: true # false = internal GVC access only + autoscaling: + maxScale: 2 + minScale: 1 + metric: cpu + target: 80 + + # Backup Configuration (Optional) - Cron job runs logical backup on delta table to S3 bucket. + backup: + enabled: false + version: v6.0.5 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-backup + cloudAccountName: my-backup-cloud-account + s3Bucket: my-backup-bucket # S3 bucket for backups + s3Policy: # IAM policies for S3 access + - my-backup-policy # Custom policy created in S3 setup instructions + s3Region: us-east-1 + dataSet: addresses # Data set to back up + prefix: manticore-backups # S3 prefix/folder for backups + schedules: [ + {"table":"addresses","type":"delta","schedule":"0 2 * * *"}, # Daily at 2am UTC + {"table":"addresses","type":"main","schedule":"0 2 1 * *"} # Monthly full backup on 1st at 2am UTC + ] + activeDeadlineSeconds: 14400 # Max job runtime (seconds, default 4 hours) + resources: + cpu: 1 + memory: 1Gi + +# ============================================================================= +# Domain Configuration (Optional) +# ============================================================================= +# Routes /api/* to orchestrator-api, everything else to UI. +domain: + enabled: false + name: "" # FQDN, e.g., manticore.example.com + dnsMode: cname # cname (subdomains) or ns (zone delegation) + +# ============================================================================= +# Load Testing Configuration (Optional) +# ============================================================================= +loadTest: + enabled: false + image: grafana/k6:0.47.0 + resources: + cpu: 0.5 + memory: 512Mi + + vus: 10 # Virtual users + duration: "5m" # Test duration (e.g., 30s, 5m, 1h) + rps: null # Target RPS (null = unlimited) + replicas: 1 # Number of k6 pods to spawn + + controller: + image: alpine/curl # Alpine image with curl pre-installed + schedule: "" # Cron expression (empty = manual only) + testDurationBuffer: 60 # Seconds added to duration before scale-down + + target: + port: 9308 # Manticore HTTP API port + endpoint: search # "search" or "sql" + + # Full JSON body for /search endpoint + query: + index: addresses + query: + match: + "*": "test" + limit: 10 + + thresholds: + p95ResponseTime: 500 # ms + errorRate: 0.01 # 1% From 4317f64bf0187bbe6fe82578358e371e58eae0d1 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Sun, 26 Apr 2026 20:11:02 -0700 Subject: [PATCH 11/58] lowered proxy rise to 1, added liveness probe, readiness probe and prestop hook (#248) * lowered proxy rise to 1 * added prestop hook, liveness probe and readiness probe * added secondary index value, added publishNotReadyAddresses tag, moved searcbd log dir (#247) * init 2.0.1 * added secondary index value * added secondary indexes * moved searchd log directory to prevent loading wrong config from donor * added publish not ready addresses tag * defaulted secondary indexes to false --- .../templates/workload-patroni-postgres.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml b/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml index 9376ddb1..49a7b568 100644 --- a/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml +++ b/postgres-highly-available/versions/2.3.1/templates/workload-patroni-postgres.yaml @@ -46,6 +46,37 @@ spec: uri: cpln://volumeset/{{ include "pg-ha.volume.name" . }} - path: /patroni/start.sh uri: cpln://secret/{{ include "pg-ha.secretStartup.name" . }}.payload + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - | + set -uo pipefail + ROLE=$(curl -fs --max-time 2 http://localhost:8008/patroni | python3 -c 'import sys,json; print(json.load(sys.stdin).get("role",""))' 2>/dev/null || echo "") + if [ "$ROLE" = "master" ] || [ "$ROLE" = "primary" ]; then + curl -fs --max-time 10 -X POST http://localhost:8008/switchover -H 'Content-Type: application/json' -d "{\"leader\":\"${HOSTNAME}\"}" || true + for i in $(seq 1 20); do + R=$(curl -fs --max-time 1 http://localhost:8008/patroni | python3 -c 'import sys,json; print(json.load(sys.stdin).get("role",""))' 2>/dev/null) + [ "$R" = "replica" ] && break + sleep 1 + done + fi + livenessProbe: + httpGet: + path: /liveness + port: 8008 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readiness + port: 8008 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 # WAL-G backup sidecar {{- if and .Values.backup.enabled (eq .Values.backup.mode "wal-g") }} - name: wal-g-backup From a56e3815d34c7c76b8626bb0edce24ec67206507 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Mon, 27 Apr 2026 07:59:10 -0700 Subject: [PATCH 12/58] readded to template (#249) * readded to template * added cpln-common tags --- ollama/icon.png | Bin 0 -> 17686 bytes ollama/versions/1.0.0/.helmignore | 23 ++++ ollama/versions/1.0.0/Chart.yaml | 13 +++ ollama/versions/1.0.0/PREREQUISITES.md | 0 ollama/versions/1.0.0/README.md | 36 ++++++ ollama/versions/1.0.0/templates/_helpers.tpl | 6 + ollama/versions/1.0.0/templates/identity.yaml | 4 + ollama/versions/1.0.0/templates/policy.yaml | 11 ++ ollama/versions/1.0.0/templates/secret.yaml | 8 ++ .../versions/1.0.0/templates/volumeset.yaml | 10 ++ ollama/versions/1.0.0/templates/workload.yaml | 106 ++++++++++++++++++ ollama/versions/1.0.0/values.yaml | 56 +++++++++ ollama/versions/1.1.0/Chart.yaml | 18 +++ ollama/versions/1.1.0/README.md | 93 +++++++++++++++ ollama/versions/1.1.0/templates/_helpers.tpl | 46 ++++++++ ollama/versions/1.1.0/templates/identity.yaml | 5 + ollama/versions/1.1.0/templates/policy.yaml | 12 ++ ollama/versions/1.1.0/templates/secret.yaml | 9 ++ .../versions/1.1.0/templates/volumeset.yaml | 16 +++ ollama/versions/1.1.0/templates/workload.yaml | 104 +++++++++++++++++ ollama/versions/1.1.0/values.yaml | 67 +++++++++++ 21 files changed, 643 insertions(+) create mode 100644 ollama/icon.png create mode 100644 ollama/versions/1.0.0/.helmignore create mode 100644 ollama/versions/1.0.0/Chart.yaml create mode 100644 ollama/versions/1.0.0/PREREQUISITES.md create mode 100644 ollama/versions/1.0.0/README.md create mode 100644 ollama/versions/1.0.0/templates/_helpers.tpl create mode 100644 ollama/versions/1.0.0/templates/identity.yaml create mode 100644 ollama/versions/1.0.0/templates/policy.yaml create mode 100644 ollama/versions/1.0.0/templates/secret.yaml create mode 100644 ollama/versions/1.0.0/templates/volumeset.yaml create mode 100644 ollama/versions/1.0.0/templates/workload.yaml create mode 100644 ollama/versions/1.0.0/values.yaml create mode 100644 ollama/versions/1.1.0/Chart.yaml create mode 100644 ollama/versions/1.1.0/README.md create mode 100644 ollama/versions/1.1.0/templates/_helpers.tpl create mode 100644 ollama/versions/1.1.0/templates/identity.yaml create mode 100644 ollama/versions/1.1.0/templates/policy.yaml create mode 100644 ollama/versions/1.1.0/templates/secret.yaml create mode 100644 ollama/versions/1.1.0/templates/volumeset.yaml create mode 100644 ollama/versions/1.1.0/templates/workload.yaml create mode 100644 ollama/versions/1.1.0/values.yaml diff --git a/ollama/icon.png b/ollama/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..b087107a744c0b0ac3bedf752f22decef9490677 GIT binary patch literal 17686 zcmagFbzIcnvp7tLNK5xpA|>4*y28>(cZbw2-6<*{xga2|QqnBaxzeE^Aj?Xp2n#IT z@hqS3z4v?XbAP|*`D0(NeV>UlXXcza^PZV^oUXPi={?4KSXfx3Y9OE<78bS}=C?#d zfElqD(X_|H!fSRhF!43f(v-D_cnH}#K}D_H08moACm$$_Vc_BDYs((!;STnZ z4O9U9!&esb|87_q!2S=4ubTqEL`#?b8N}O>T|!7g=rKU?9=p7^gOjWtP~|^}F;5DB zm%hH9vckdv0Rch*VnPsaXJHW;85!ZnqQau0f*1@zpCGWWZJ;37hx3lbzjy#0eeAtm zJbhgtVD>wnwssIdUj+aF184sa=8l0b|K<+%`H$se6d`;E5f%}8Ed1ZmeO;XXFX(rW z|Ap@P65L(We4qPc6!9NQ z{U0*@=L9|mL7tAndX7F2KW}@-=l+ghU(SCtaiQs(IKtI~uz<_`du%w|@gP9l_3bkpJZL z59CfdcfULH8iHLg>k<7oH+PuQj4`qTK-tO3 z1*6U&cFgf{aRxhjvx~BeV$SBjl<{9w0WTdf9RKg@6aEkBgzrw?e~KqB{J$0;{|~}{ zJE|DEe~n>W0LGUH|CcktJp7kCas*?1kT=HJsI2t$VH{eN3&u46%OqG>521e0MgHg) z^aDyv*&iNKt=qZGA;sS}zi1S9b@}n$T58r!>6at~-B!GA)MEI)c^~zvE7E-3j1Ry) z-Npkpso9wSm9Z_OuyLM?mKmFr8N+syE-pF_CNB~*6fFX0sY)o91(*6h$#(-=PSQ2x^wGowH==~}3v)&ljeZYtDpnZ!1ehwef6 ztkSW0MYTb|*PXytw4Bk@Q^r;!XZ| zOjwhW&OTBh${5QWCxMr=>xqs6BjX z2hQ^PHY@Pn8O7s;Dv1diGW=b@6~0XCckIslnx`%qAP)P;i0%#N73C#MBw;k@*(_EB z*9`_io{V|>ry^840XqqQx9`?@MRLh+-27qsSK2*N8-^RYbr1tyC$VPioDtN&65rxU zYe5XuoAI%vw|OzZABUCu`nilAn4mFv#t!{2MPY^=R7r zi#Ktwdn4#AmX~Z$^$Yq<;e6z}SA9o$y|e6; zHR+VLjEF%@T(U|dh^v+SY7HI zd$VTf{W`&sgMCI4_jMn%Ri514q>imvZd)5NZ{Tj=d#E55hJ1DQi%x}XJs~xWZ+ zf7r<33%0M3h5Am<>BDKY>_Yf9lcp|BE-fF5RAPVU{gHE9Cb=@pVv_Q((-i(e((e84 zIN_XDHp+veR+UL4KlNc}+f1w$K)qpob-g|!<~$ZCLiP#wdB_jjqBca5t60AmTrkcq z)HFv#9k1-9Vxgf&`Znd^=%HMKe)*2wgoQJmKm7)4k!h8;QH+;5drr!qemkh3S`*I? zi}`oU;iF=b~{FkfZ>G6^Y^4m0zfheAGrMGKCgaj5 zGKDKT&)0VyI|)NczyFHUX(Y#E%jQk>`}bM+Jz=gZ+j`V;*Y3q9xJQb7AcVg0zKN12 zQ@?$iad6|RPt+`X;n8x})eaqW=D=se|30>yifeok4EJVQx3Yi`&6F6C_=}98k@|+t zxh;g%cgvA@<^Ten0l0)R-nu zuIK1m6q+yt4bWL?gDLRi;6)30HuHM6k%;G^JHaOlzMY%Z62~ zjf>!ZldIFWNfP*?cK2@FG?NqOc=ngW2I1Z|u4|G(I;8zKB|pFV_namsDBz2(=xz94 z4}a34S{&Q!01NrNoTLvum1e*zk*Vj8eWVq=jkqYlqY1g^@cPuaD^qrw*K2##VzGtH zuM&Q;b}~V@jW)67Q%59T&y=yE9Xb?iVLo|O2w5aUL4JGnq>}8E*k{p01O~x!*A3vQ;!`Mjr>#d21jS~;*ZTSjghavZ$RYRPYEoD)!Y(ymz0 zo3R)4dj?jmhrzIZHTI*Ea#|LFua20=0&63%rlk=LWQ~vbLk!gr3;OK&bR(cJ9T`NE zoUB4g`9+ih3o7K^Eyfn#^te@~)e*-*)&kGqXcm+?>8$b7a_116I{UDVRk@h^H*93; z2<#y66LBmHw8>(uMlNd703;El6J@b$!GUe*K}@E}Ze_hkZC&%x(^#GQl!ZU=X|XHhrkW(q$t)sWx!lH}$DB!iQ)01U zHru-f*5wWzWc+Cd3&>}zuv2yLbf7@~457{++qeqJD5V{?Y}BQn*j~GR*s=V5_YDK% z6DfxDN@Ix;ShWhcST5~YBho%nN*~H3D^Y?&NS;Y~iqnM`es6C5LCnTwy0?MC)1g)s z8l&hy?dT#_Q{1@~6XR{Ro8zc7*jF4hmZI7*l^>_M?M@4|Id!NgV&Q{)n%{7U6sP<~ z)mI&a;T~5wuPXEAuM#J+?o+&!pFXeL>{Mf}8D=#Xz6zF&4E{MEpb9r#V;uF6;qttC z9*Wa-BJ_zddfHtJ{>Q6hp;jR@d+HbM*^Tbo!1odIdj#%gHZY&7mW;lhYcJ`?)q&HU zHoO~W&IGY7l}+Oz9@PTqUWuMkQ?10F0N}-2Dj1kJl)TLceXeL&^_^w6R8#|<OxoGM3Z(|yLk9#g({Ss`K5!*-WU6@=YN!?a7TN=l^x{frviypiN}+9 zCS4w8OA_)M6+Wire>)H}K2kk*n`14D-8|?gtXex2-Vfh|&%?ct#EKK0Fq5wN=wo|c zsWFALhUu*gqkK4lu+{rY4U=hDHM7yHgh4@52v#Uv$S}HqSd4IXq~G>BP#96eRKI_F zWLL-98z8g8nB?ataO$vgd_=XDUfDs$`{s|DjpdTKd%;`k{dwIAI!*Y+arjb>I9qNY z52}p?<=W@-2*`<|l22>cfSFYVCRIwU_erId;Fj94F|rZx$o5ah^Wd<3ySS_{xt-qX z3D7xZW+h|#)B`B{PVt2 z!G0*%p~|5{Pn#3PTU-3vYLfg`gB9iKXs910yxf)m=4(U;2oW%&hGk7R6iFv4I8hu+ z9GH{;O%YP7mSLqeH=;e8MndB~}_|lUl@kAe8iW32AMVzBf45dle2lY`c$D zxehC~2bxbYfg`PE7p93#Y2@xZyGuSpL0Kr{+=f zdid)OrdwDlFX~t`Lh7uROpL)pva?Jq+&{3xqqzUNQ4LH~o5zfl!AD@6%}MO&DTC+h z{_|CvCYh;z#QAp)X|`2eiCGR<$76F0H4 z0xi4r&?aO_)j;mWbTEZ91CmJ2;OwS;882v&3PJt*Sf|?3B8%Ac)mT z>Yk{TXpt|Tec;`;`@S+}J-;#}$idQZ8vD{V?jp4p7RZP5*#)?>e^4SfCb~c-Cf7U9T(ic$gb`oEh)D?{-P7?Xjf}=|v}I zZ~Q9p4~;xhsy`mS#nuNjwq+)KW2I8R&PVX`s<;R6_OH(F2Aqpm$gw1#b0o$w&eS7r zr#bV2u(45~=T9+rdy~C&mwwkkgCj>{qPr{8((G`@N`HK(P^d{I^Scl~%po|pf7Vk>lk6qy<*6a{k zWvNT8D*;U9NxKJ=jgTUekEud@AQVW_?b@j8$UV`osdzLr<5q@z%W`w6=)MM%lSDU* zVD8xW%ZoM7FW6eO+sV1NevS4bY6=LygP2Fck$j3(0(~l_hqZW$slCp&Y>LzNFZ;UGlSS1POlApdUpp-M=qfwRq3PF(Onr-`b9DOm8DtKclAU8VtHO zkaokk|40K?&}A(<9ilsnek(4Xc0{Kul_@q8!O!(c9~4pL>+`#PC*UdL){LE zf_7Zc9sSS3%|UjD>dgN5y5+v#DV zA;EarT89&4J4`2R$FNF(P+Q>VqNuyPzpL7oR9*S{KwdzLZ&X3b%}j->4DN{BL{u5l zOQmeFy*vXSev#+j>igyxcB*tNVKU_{Lt{Fe8Lbs1r*>On*b@4AQaiKy08-RNP+_qP zKS&XW5jI}iZ%I`QuPc`OWw*YT!fqqevyL|u(y==SZ3QaS#vVcR(nmy>Qncl)S|>RE zzAmnFfcB9Z<9{$?|YSX6V(nQkqNb;QNd~5yuo)g7_MAC;&$rsnD zFKVP{kwi$m4>DJ7JLo2Y{$BT#LMY{ljvnF4W}d+g6VfSJ(E@}$l0GjF=@wKe`@LxF8swOIUcDty&r9p z#<*-alJ^(60Sa!S{^GwIif$Ru^#VBl#EaZPAF$;$FSNl;_p!R3kgPtJZ^x0&bQP2i zq#tp!bY%9g^WPfvC>6l#e8Vb5qh`yrbO}E!8pGdt&kwsoNNDqhl{P*a!!GDHvWx|p zO-9|qjXoBQl2#3GLNDO+#6{@uPMD-5iTMNa!dgf@ zA|CWm7$3|VHu};aLw}jtSrYe8N+&fc{Wm=1Z#AJeuysCgTXUSdXpG}{c+z0cYMK!) zScGm5UMEL}euy=gM687@to?v*?Ps$ZvF({p1kC)cyD!bKQ^+wPnAAs0P&6u7I0E?s z%9nX_lqb=G3r8O$H!abLjirS8{*^uJXP zpx|Tfi9a>m(kz`}X0Q6EGWa85UWX4&B)dXYVOmxBfV9DIbU%$%H@i{leE zLRy}3b71haz*0`!mQd*6Y2bu>QjnaPz93+;GAd7TJ@4|5uIFN54;7vqUV8(m768ER z$P_iw4^v|?&;Il|Gf2>g1~abN*m(csX*^LHJ&ud@-RjCinUhG@ zo{+2XqB1Z0aF8zRT>;hlz#{z(o8;&aGj6tIZA6G10gfka?OcmCB~)9mkM?Px>xbk4 z8@c(Q(UEQ1W-kB$*8X7e>WrB*Lma)$Jh+M{JFEii{YJm-#xf*U~9 zBbempaldk$T+^rW!TZYRMHhE3mGzh_U_(6}^SITXc06;9WU#j)h|FO@Vf6h?&Q#}^ z)pU_Wz^hUR$o1Aaa)23SJ}YOcv}KZs*}<6XP{Y7VP9?)v; zZIb2uYcVp&|IjtON^s1sut+(R79`a|BK@nZ8bc%JT;(A62Y}}UyJf&5UI=F zl!6OK5>of3$2ZhMl$mT7M+GAgGj!?r@Bjfs7e@~e#1@xlgm6xKyRdEok0=K_1wK>>UbX6PKyC&hUb*%Fe z(J$QcOsLPO{0qgOA5^#o-PbUnmPLDGs)IgzoAG0E4SXevSJio?@IO!6ALUY27o?uC zcPh=5UWiwCtVWVAU1BdLI@0ciPgL?j{xF@L^hkwe)BM;vJD%#n3Xn5uHfj#{K2^^a z9*BQIK(8nuh6};l{nm6nZ^haV2A=rsDfZU!x?~u3&J}t=E2FKI=J6Z3dMmnzmM?~; zT8o&ptdheNv>e^sP0ijlo~pM{AV1DaalI0OC;F`%Jd}c-TnO1yUtMgfM0Uv;nM_Tw zFIbbSW|H?oR^pFr^;zTZ^;S#-Wsxw?Bjl(TVrCOWjDjMrcvN75;bBKowY>K2kLDCD zlv=WsFJ2$xb@CHSb|dbUy8F~*6xc|4c4z|T%(gx|3Qbw?H8O>V`MM}Iu71coW=7px zFD!~^wpCc_(Dq}&ox8`pW>8SMm50IQE>FP60xIC@YH9pwd!Wk){ zM<^ojM?E_WxeAB8Gl}snVLLN+KdkWJM5K*)&otE1@EO*{*;~dBthD9XPNg1)m*IY) zL~*5c)c)|-i=Yv*=kvE-K3}f8pR-yZZedi&s6&qUDz<05MB(nJ`%)$^#fQCd>C>Y| zuS&Ne6Y)B5$!yy3IFhs~_q3WGjtE$M=aKEl0M+z5_ea6M4engP^-d2Y~nJwb{31!GPxxqq^qVM{;4LlkJ|q3hNt|Z4tXMDf<}R4w~P*ponEe1`|%h$YG8_p=hi^LxB8L7Gn-} zJINX6CiN-YQ-Gr=*cCs&+?C;geoAE6yfO+V-=; zon*TV5mba4QQjt=th>psB|RPGM@;G3X%^~kG<+Q8q<@LhyMOfLQbV*A4xBl?hgh?fN9W7dlY;R??8#=ffc^`=zXl zX6)59&y&imvYtzjO)ghj0w3_UGXc1 zd6V>Mm&VCh`jAUT2rpNW%mLd^s|Sac)AlbV&$Dp$i&w5mFNJ@+QiDy--LUwZsKZVp z^WRjVcYhi6GMz8#?3eNuD=^uh!xB!>#P6z*Sqvg2P`%=40;_isVE5LtDQ#k$i^b6k zffgxSri$0QF&Z!r>G-o}uQ5e+!m|FbH{e>a{XexzrjJV)kWn0;R!x;9Sov$#Tet^6 zzhQ>DFeNS3%#E^sD~^T}j%_u7b@3VJ@+E#kVWWJs2mYgQGR{%G@$(1Pv{G+tLq0B& zw^!N-lxcE~VpW+273IO@5m8A(hQB@A5nISfyHox4`G6POFAmH$-hb(O!GLFFEh@yv zgV3t1$xJ{cB9G1LZ=7q3tk=RdUD^7Sc=H#uZB)64woS|1YI^Cn`Fb>7Lc}h(k*s7j z_jONIdQHC_7GT?@t!Q&zg|y-!rJuOhbXPBOd;F3i&ry-dXeW0BJBgmSM+Iny_WXUa zRn=>P`yqZ&xtOz42PQY&VoHuA3nXqt%|BgoahHPqhW{|ReY@v43{g~?p@Cx~+dmg! zEXcN;S)Z50l0nfe?w%jk?`cXn!g6XmB9Dfb+^g8tfOpAEw>&^CKHs@ojo-iJV)&Ry z>=*Qn%ZAK$F`9Rrs>7mE`>fc`2iEbSSRrRg$yfo;;+I;=05_2*Q3%jz=8(1f^!8h| z;NS0>+iR!*n~^W?nCsobrgc(Jo8l}i)UvNm`vBj-3Aa`poW<`n&gDCDYOk+x7x&C( zuyU+->RF_0b$WL}uzAAFl#Q8#G0Kzc@D|8fx{o;KHO*UHUht{{sH|kp^Otpxsr&dC zARzN+&0@Hr?0YnKgy&1^Qm3B_dM*AVE}hH9!@)=c;JfzI5z(kFz>A;H;0*?=;tY^g zqL5~|acB>OFJ`haKDVA~HkoL;voSvjr5pw?00qpa-}F=asMZ>-t2#!bpAu3uCI=jo zb`>Y5NWLJ4ZUo=v5zqE3g!Rkb#{D{SLJ|C}Ibp;(7|&G<-(AhZfB znj#xXAcSEcjQJThDf$-_t#bQBeeXMP*T<0b6CrfnN%u(OT^*M`BThxeUkrpj>P z86N9@-$mBY9LSc}O+_85wbAU6*D?(%Yxby2^KDSxMuDZlpynG3^?bRHiJbGerq7UH zWT&yjBEW#UQ@hBf@A*iRQFW3h$LU(JrRnzhXNtAinVHZjky# z7HW9JF|ko+&KLd{r12>;k%n(1X7%rLj!asQb>@&#L_hdTkfrgdZX$L3!jopnjhiTz z5q`B(Uw@Cd@EiBR(v5c+G7o0`syL(O1e5P;LvDqpW~H~Cy6dm?7owb( z@zwDFSocNV?BiY%i947NU3?xoZ=S zS}JtA=;j~okH8D@Zx?m@ZKQ~U)xJk+t@?`0-#s-(kuXc?1sJAaw-lx65WiaNMT9-H}IuIY+I^?=G|0Abw2 zMBO1-OgR0EkU2}1NC2W}y%r(xmH2#2t;;At!6vy^B=DYJ?2c=c74I94rAn&X+sv}v zn5o#}D67(~>z{E7pf2?H=WyY7H)D=E3DWsT{3PDiuW0qYP#UJE`I#Q-5mG$)!mIPl zQpYgm;sy~aU`=ZzX*!0ZEp*Q4csxPXsjFncL=c>3;FNI|!HXXc$Ya6j&=%%K{hGcS zdfz@qw<6HUhC_!JnD@Dar*hW(3tP2i*-5Blmc;lcD$7T!u;|y%{uVprd3R-EN{5Oh z6uV2`-cjN)pd)M6t2FCIcOIN-@6hixyz&?^#ZKm~yrjp;{Z0)qrIVFa_$Ws`jo$ng zS(Cmh6TS1uCsiYi9_jh!t{kL-E9aTzusyK$+IVSBppU4 z!jeLMEzhYoyRjeZN$P|loAwM^Tq%&4^jJ=FtF~_bdd_~%hrILGsIEt&=4j<}f8%5G zzJ=Ez=mNha1;Ry6kIsu1SxY0D(adju7?+Ah`i0lDa4xxZ@GrRq5sPeCOi{-yuqS{J2-bNVc zu8Az`)gE~Y$ZbJ_*!-O(9P37HpJfvw}7nd9Bt2P@G)fC@t=NJ(8GM zuWflfyI5Ri&HHi=)RMaO1J=KHulod5Wl&??RQ$E(t41Vs8qjQ%5sAyfv4}oxZDiN6 zkLpSl%6z`4?_xQTSHCCT(I8Zx72chnEee#-NnFI1OZaHoW@IBZ>96&`-qf1sYD{lS zE72Q9EIF(N9b@G_F!;qwCE-pn>uY6np=f(~W@_^mb4at^dVuT6J&E=h-CK9344SlT zbj*b~wl71=+(-`S4(iXeHOd6_b2d4*C-w8**Yvs?yy41_a-Zg&9)~4)e96v)@^oEu`#y8J>(-;0 z?hR&)PRw|EdWj{8=D%40o+p-XrI&6d>3iF?V%;)JVO6r!QL?H1j8(R2N=f{y2LWi` z3InD5I(DM-LRnU?uiEY9%3TAl4(BMM#&DYjlyOJNen$!+2(O~0(ciq6=EJo%U1D(K zV-FQadG`KR#a|QLS{R@89*&pTDb?qH6ULP!nTlkkxGbB{pEH}t(e6b=4+P&P9lEKn zd#KKxezwX|{rLRBfHa{ZUM@9YkJX3RYcY-Ov!TqVW6E3DM#ay2DcaY!Q&%^Gv-|Eg?Ey`t_GaC3Qx zwE(!Cl^yR1rPdhvZzC=(Mc; zF(|)KFc2kysqUsLzKhu9KR!J2*s!7dT=Uz}1LymgZekhCfH=9R^_S$z4rvCcUrtyR+K*l!-+C4LZc+ukPBd7ui82 zKv`XXGBQRjOO0$KO|JiU%;3~tJg>^jHVeXog?IDmjCSXZ|?zZ=;BBWBmTt8A~6OznO-+D zJ3h&K7HbC9HNTl>zddeJJuAQx8fc7br<*MZp=})IYEXk`z>RtwQOk9NK~>>d(H_mm z$-km}%UqD`J^XZJwucB3mD}fq4q;d)XLXq*k(hqH>3B&jp>8nq2AJ&R=ODBDO^D6e z^o$;Qtl!3`adqaZ;rF5z_pM8gDvUcgpoFYw#FAut(`2y|zNmU@TiC&RlEgF;f3?uk zX>}xhLVt^YW)=8clTBy>settT&RK@Li5<~>>g*J<#f4IN7J_yNgZhMZxC;5lZm-9! zrg($#tdgi2=+Ia7&v@1|;aGMb=iXyo1fhc2} zG=}D#)VaVujKLIg|fR~ z)!H&(o4VsCXAM;^1>#oDW#CmP;rXt;HfZ%m&?Qw{;xM2uV(!L68K!~3)#euw1RRUinRRK=9{Ua0=% z+ej3w%2^l=?%sm#?XJsk)($tp#)pFa2A;5bS87}d4v=kkc%F4#!l;e?yICQNF3sOH zsVqe>u{Q;n>BevEbgbmQx71qm@$j9nrxPw7fMJ?DBiH(%*^n8q+)1P{A~i;!M46!0(F%Y8J+C{-CNNzxNR9=) zjL+Q~vSS45vjg-Dt%h2c2ZQe)>mX|p)#&h0LVu?6-gT8ZTLJ(anDx;az5d06QXas%D`S+m)7nLPe7pRt zo+(H0akU*-R4#w&Xf1od#O+229j*L>X@hEr1t}|}S4ubw0u0wBNJT+fR@GVv{x4bI zlQi!nc~k>Eo|w6%4>O*0^Tm`V?y%}c0j4_c*E&F8-I_$5l_S$8#ui!n6e#H_P5_me zPGTi5)@1G$6(zAF;PF&hwLq!_{36K@Q=+Nm`-$lUl^hDj9cTptpCW(3s!MzjjHdCi z;*DAMmcJ7!@9HhCl75vHE_L~<^syz%z#`W=22-WK^sy=kfSv@Vg?Gw#^a6X}iKKs} z29OI6)0Fzid8$McF}vD?rvfahoQm2^I-WPYWaCRaW`xrwVM;^qmu1QcnhQ^q^myDN z^YSni96ne5jSFzib@KSog<7Z%>*5ehh!QF)e>imrf7TG8i(XP4qv5~^1~S~CD_B7LP#xjb&k8=aol)*REX<3*t>3Gk(T8#JEJUfthq>J+Bf{aT=qJz*L4= z)+`oj@~&2ik^R&8hvGSzFLPr-_h*-K2{#^1FDp z+Z8&H?lnon#E&HJ7@7sO`GiFX9VoH9MejurAzMKo0RYPF=m@g8V$jD5?D#y~Ygg!# z=Rm+2S?4)Kl3RPM^`{K7^yT9gA;8K@fed+t!$p}NIIfb;q(VabB##E`XwiyeR0RZ?Y+^1*=f=(F!=vQmV$j5j@ zU;83K1ylc?r3id49^}FLn9rzZRO{CljwL4>@5;Avps}p`bHLWzllc#c8h%Z_MHZ1B zm*-iq+}d- z38MPI*ir@13)6=wNb7A>^4m+*=aWCUFX?+;a7QzI$+77(#a`>Y{T3mXam4>jMb_qU z(1QlLHSCfBuIXWyOVh7GUl&o}u*XCAH66ctcUbWQ_AXG|uN%Wt`Z`Q?rjCj1BA zpJ_dSzROadY4;`mN!O9ilC3$@FfFzs^#b_FiD1)v!or;d!G7_qgTfyKzr^j#j|HW7 z(9Omx#in3(|0cYK7vnU)KJ7I5>k#I%T9iwAYm+MaW~F*vUkEJKKePZfFr z^wdgId4%WD3DCHF9=);FD0%_m|fG`U|wjM^kBq9(D3Lq*pYho=Cyfo^VDV~b8zUagSFT%G4nX=T;g!#H8uDgl*Z(1fd+dRF zGNxO@IE35fU^?nKuFzLTu09^_X`Bc(-I`e6%r2Yr-dRfGtu@IjrS!UmK6T)u^_N#Z zNKPTa3Yf=7xE$XpWw?eh<)-DeiPiq7V;dc5W9$RYxf=Tb&*#9W^u3Lhdw*nUys0}2 zfM283gaoOfR-(is!ujXhMwuDd=CTrB8bkYw9RU;ZiqR3;vvY;EYyse3_i#K!9s=|R zTF;c?!s=9es(rz~3|SMYp!xnZok2VAv~jnO2v%L;3wcI*6X;wNHFUwJ+9&K18`WF6 zL=mbK_BOK0UJ?Yv8xdP**~!bpLv9XUM3UWxnFe5=N>B3vIyP7Z76XG_cyFaKjpo-= zKwz-(yRwvrB})+lzKbS_l#~pP)}e>9_W&$ffvc^>XAr7wKc^??p~DgYfG?kDJ=)fk7dgXVTJv_tc4r-ZcbWe3`D$-z zZWGj4=AU}3?Qzqli@Ec;B*q-(RhgI&4Qa1CHY#eos&#-4^*Ony*~TfQm%OGS)p(`^ zb4z!Zi9$Z78xf{OUKJsQt8v%WP`07%zW0|KRJS1Xgnefsv>|rQM#eM7KCC$H*sP~- zWan;H@l_d`Itf9429NYg=eP02f&p^Vynu<=4Xu8)`oShlGdasLH=t#&bEsMCYu9%^ zQ(-*T08EcWQC6kV>w0xW4&MDDn!8pO-Z8AN%qaf!I{`2^SSr)lc*8%@Nd8c(s1!XK z(>VbwVj-8tT=!z7PD8;swP2)4cJ|I)e}x>aG(w{6x3#1OLjNTUBv)I?f})~yQRPlO zK7&Wqf5wCEN5EQD=oG;D09!OF9Ta4Zxd-Rha^$7OMaA zX$$jRNOE_l@=9LBTckFSF;hU91^Z4KPg1W3gWuR=BCNbYyYu`zA$`;l5_j4*{*8&g z@XGJzJLnuTG5(7ACPDBXCb}BRbgl!RwW|pY1rOdu7TWiv5$R!d`P+Bm^Oi^>3?G^b zCs2Sm_3tF#1fJZrsB|>`Gs&JNVod6SBoQNd_K2Os{FY}Fwv8szgfSaSz^D99E311y zG%Ci~ZJT7HU+8lHCPw5{r?4~_ub)>q>|o5(dBvLiro;n1*O$zHNvHfia(7`24u3Gpqw&C=r(YhXMm6FGq}qE|Man>g#C2 zNANHKDMR!+w{VlhKjRmeaZy&}I7|`a1y4Q>WyMm?Eo=*$bEBAW2qu!E?*bo}wW-l# z2^Q0)3BvCKh?}7I8pC+}0IHwGXX~BC$jQ`Vi(UFXs%uHs zK$&L$sO8c35Y_s7|4#s%0Av5EossoD_&&+FCh%^@`b0!(y%Si&xy~RXYk({9{gkQu77&~O0000 Date: Fri, 1 May 2026 14:05:48 -0700 Subject: [PATCH 13/58] redis replica init now queries sentinel for master on startup (#250) * init 3.4.0 * public access now uses internal replica routing instead of external through domain * sentinel properly announces right hostname * extended sentinel timeouts, switched sentinel announce-ip to use headless * switched announce ip on sentinel to replica direct naming, changed domains to only use single port * improved ha proxy health check and patroni resiliancy with dcs (#245) * init 2.3.1 * switched ha proxy to use http over tcp check, added 2 health endpoints for ha proxy, increased dcs retry limit * lowered proxy rise to 1 (#246) * added secondary index value, added publishNotReadyAddresses tag, moved searcbd log dir (#247) * init 2.0.1 * added secondary index value * added secondary indexes * moved searchd log directory to prevent loading wrong config from donor * added publish not ready addresses tag * defaulted secondary indexes to false * lowered proxy rise to 1, added liveness probe, readiness probe and prestop hook (#248) * lowered proxy rise to 1 * added prestop hook, liveness probe and readiness probe * added secondary index value, added publishNotReadyAddresses tag, moved searcbd log dir (#247) * init 2.0.1 * added secondary index value * added secondary indexes * moved searchd log directory to prevent loading wrong config from donor * added publish not ready addresses tag * defaulted secondary indexes to false * readded to template (#249) * readded to template * added cpln-common tags * updated chart file * added retry for redis getting master from sentinel, updated non publicaccess mode to query sentinel hostname not replica specific sentinel --- redis/versions/3.3.0/Chart.yaml | 4 +- .../3.3.0/templates/workload-redis.yaml | 76 ++++++++++--------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/redis/versions/3.3.0/Chart.yaml b/redis/versions/3.3.0/Chart.yaml index 4b1619a1..7f420e97 100644 --- a/redis/versions/3.3.0/Chart.yaml +++ b/redis/versions/3.3.0/Chart.yaml @@ -3,7 +3,7 @@ name: redis description: A master-replica Redis configuration with Redis Sentinel type: application version: 3.3.0 -appVersion: "custom" +appVersion: "7.4" dependencies: - name: cpln-common @@ -12,6 +12,6 @@ dependencies: annotations: created: "2026-01-29" - lastModified: "2026-04-21" + lastModified: "2026-04-30" category: "cache" createsGvc: false \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/workload-redis.yaml b/redis/versions/3.3.0/templates/workload-redis.yaml index ef123a45..a27df867 100644 --- a/redis/versions/3.3.0/templates/workload-redis.yaml +++ b/redis/versions/3.3.0/templates/workload-redis.yaml @@ -72,45 +72,51 @@ spec: else {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} SENTINEL_HOST="{{ include "redis.sentinel.name" . }}-0.{{ include "redis.sentinel.name" . }}" - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - fi - MASTER_HOST=$(echo "$MASTER_INFO" | head -1) - MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - if [ -z "$MASTER_HOST" ]; then - MASTER_HOST="{{ .Values.redis.publicAccess.address }}" - MASTER_PORT=6380 - fi + MASTER_HOST="" + MASTER_PORT="" + until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$MASTER_INFO" | head -1) + MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) + echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } + done + echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} - SENTINEL_HOST="replica-0.{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - fi - MASTER_HOST=$(echo "$MASTER_INFO" | head -1) - MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - if [ -z "$MASTER_HOST" ]; then - MASTER_HOST="replica-0.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" - MASTER_PORT=6379 - fi + SENTINEL_HOST="{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" + MASTER_HOST="" + MASTER_PORT="" + until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$MASTER_INFO" | head -1) + MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) + echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } + done + echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT {{- else }} - SENTINEL_HOST="{{ include "redis.sentinel.name" . }}-0.{{ include "redis.sentinel.name" . }}" - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - fi - MASTER_HOST=$(echo "$MASTER_INFO" | head -1) - MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - if [ -z "$MASTER_HOST" ]; then - MASTER_HOST="{{ include "redis.name" . }}-0.{{ include "redis.name" . }}" - MASTER_PORT=6379 - fi + SENTINEL_HOST="{{ include "redis.sentinel.name" . }}" + MASTER_HOST="" + MASTER_PORT="" + until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$MASTER_INFO" | head -1) + MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) + echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } + done + echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT {{- end }} fi From 1059473b9271b907e4e53484317f36f55e3f47c5 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Fri, 1 May 2026 15:28:12 -0700 Subject: [PATCH 14/58] updated readme and releases files (#251) --- redis/RELEASES.md | 17 +++++++++++++++++ redis/versions/3.3.0/Chart.yaml | 2 +- redis/versions/3.3.0/README.md | 4 +++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/redis/RELEASES.md b/redis/RELEASES.md index 42a55397..8f090d23 100644 --- a/redis/RELEASES.md +++ b/redis/RELEASES.md @@ -1,3 +1,11 @@ +# Release Notes - Version 3.3.0 + +## What's New + +- **Smart Master Discovery**: Non-primary replicas now query Sentinel at startup to find the current master rather than hardcoding replica-0, ensuring correct replication after any failover. +- **Resilient Sentinel Targeting**: Standard and replicaDirect modes query the Sentinel service endpoint so any healthy Sentinel instance can respond, rather than always targeting replica-0. + + # Release Notes - Version 3.2.0 ## What's New @@ -5,6 +13,15 @@ - **Backup Support**: Added optional scheduled backup to AWS S3 or GCS via a dedicated cron workload. Configure with `backup.enabled`, `backup.provider`, and your cloud provider settings. Supports Redis password authentication (inline or from secret). See the README for full setup instructions. +# Release Notes - Version 3.1.1 + +## What's New + +- **Template Refactoring**: Centralized all resource naming into helper functions, improving consistency across templates. +- **Password Quoting Fix**: Secret password values are now properly quoted, preventing YAML parsing issues with special characters. +- **README Rewrite**: Documentation updated with clearer configuration examples and internal endpoint reference. + + # Release Notes - Version 3.1.0 ## What's New diff --git a/redis/versions/3.3.0/Chart.yaml b/redis/versions/3.3.0/Chart.yaml index 7f420e97..3eb414a2 100644 --- a/redis/versions/3.3.0/Chart.yaml +++ b/redis/versions/3.3.0/Chart.yaml @@ -12,6 +12,6 @@ dependencies: annotations: created: "2026-01-29" - lastModified: "2026-04-30" + lastModified: "2026-05-01" category: "cache" createsGvc: false \ No newline at end of file diff --git a/redis/versions/3.3.0/README.md b/redis/versions/3.3.0/README.md index 71acce83..fcb9b08f 100644 --- a/redis/versions/3.3.0/README.md +++ b/redis/versions/3.3.0/README.md @@ -100,9 +100,11 @@ dig .your-domain.com CNAME # should return .cpln.app dig .cpln.app # should return an IP address ``` -#### Connecting Externally +#### Connecting Externally (Public Access Enabled) ```bash +# add -a if auth is enabled + # Redis replica 0 redis-cli -h redis.your-domain.com -p 6380 ping From d060ebacaa6c372c5eb158bc7cafc9683a5dfc98 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Fri, 1 May 2026 16:21:26 -0700 Subject: [PATCH 15/58] updated public access sentinel querying logic (#252) --- redis/RELEASES.md | 2 +- .../versions/3.3.0/templates/workload-redis.yaml | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/redis/RELEASES.md b/redis/RELEASES.md index 8f090d23..4d92b4d6 100644 --- a/redis/RELEASES.md +++ b/redis/RELEASES.md @@ -3,7 +3,7 @@ ## What's New - **Smart Master Discovery**: Non-primary replicas now query Sentinel at startup to find the current master rather than hardcoding replica-0, ensuring correct replication after any failover. -- **Resilient Sentinel Targeting**: Standard and replicaDirect modes query the Sentinel service endpoint so any healthy Sentinel instance can respond, rather than always targeting replica-0. +- **Resilient Sentinel Targeting**: All modes query the Sentinel service endpoint so any healthy Sentinel instance can respond, rather than always targeting replica-0. # Release Notes - Version 3.2.0 diff --git a/redis/versions/3.3.0/templates/workload-redis.yaml b/redis/versions/3.3.0/templates/workload-redis.yaml index a27df867..fd3549aa 100644 --- a/redis/versions/3.3.0/templates/workload-redis.yaml +++ b/redis/versions/3.3.0/templates/workload-redis.yaml @@ -71,18 +71,26 @@ spec: {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} else {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} - SENTINEL_HOST="{{ include "redis.sentinel.name" . }}-0.{{ include "redis.sentinel.name" . }}" + SENTINEL_BASE="{{ include "redis.sentinel.name" . }}" + SENTINEL_REPLICA_COUNT={{ .Values.sentinel.replicas }} + SENTINEL_INDEX=0 MASTER_HOST="" MASTER_PORT="" until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do + SENTINEL_HOST="${SENTINEL_BASE}-${SENTINEL_INDEX}.${SENTINEL_BASE}" + SENTINEL_PORT=$((26380 + SENTINEL_INDEX)) if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p $SENTINEL_PORT --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26380 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p $SENTINEL_PORT SENTINEL get-master-addr-by-name mymaster 2>/dev/null) fi MASTER_HOST=$(echo "$MASTER_INFO" | head -1) MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } + if ! echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; then + echo "Waiting for sentinel at $SENTINEL_HOST:$SENTINEL_PORT to return master, trying next..." + SENTINEL_INDEX=$(( (SENTINEL_INDEX + 1) % SENTINEL_REPLICA_COUNT )) + [ $SENTINEL_INDEX -eq 0 ] && sleep 5 + fi done echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT From 03860b7d65f5f971829cad1fd17c8e82def05e4f Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Tue, 5 May 2026 08:39:18 -0700 Subject: [PATCH 16/58] added 1password connect provider (#253) * init 1.4.0 * added 1password connect provider * added cpln-common tagging --- ess/versions/1.4.0/Chart.yaml | 17 ++ ess/versions/1.4.0/README.md | 199 +++++++++++++++++++++ ess/versions/1.4.0/templates/_helpers.tpl | 39 ++++ ess/versions/1.4.0/templates/identity.yaml | 5 + ess/versions/1.4.0/templates/policy.yaml | 10 ++ ess/versions/1.4.0/templates/secret.yaml | 9 + ess/versions/1.4.0/templates/workload.yaml | 61 +++++++ ess/versions/1.4.0/values.yaml | 82 +++++++++ 8 files changed, 422 insertions(+) create mode 100644 ess/versions/1.4.0/Chart.yaml create mode 100644 ess/versions/1.4.0/README.md create mode 100644 ess/versions/1.4.0/templates/_helpers.tpl create mode 100644 ess/versions/1.4.0/templates/identity.yaml create mode 100644 ess/versions/1.4.0/templates/policy.yaml create mode 100644 ess/versions/1.4.0/templates/secret.yaml create mode 100644 ess/versions/1.4.0/templates/workload.yaml create mode 100644 ess/versions/1.4.0/values.yaml diff --git a/ess/versions/1.4.0/Chart.yaml b/ess/versions/1.4.0/Chart.yaml new file mode 100644 index 00000000..0daa0c04 --- /dev/null +++ b/ess/versions/1.4.0/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: ess +description: External Secret Syncer for Control Plane +type: application +version: 1.4.0 +appVersion: "v1.3.4" + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2025-03-12" + lastModified: "2026-05-05" + category: "secrets" + createsGvc: false \ No newline at end of file diff --git a/ess/versions/1.4.0/README.md b/ess/versions/1.4.0/README.md new file mode 100644 index 00000000..ff3e3da5 --- /dev/null +++ b/ess/versions/1.4.0/README.md @@ -0,0 +1,199 @@ +## External Secret Syncer (ESS) + +### Overview + +Creates an application that continuously syncs secrets from external providers into Control Plane secrets on a configurable schedule. Supported providers: **HashiCorp Vault**, **AWS Secrets Manager**, **AWS Parameter Store**, **Doppler**, **GCP Secret Manager**, and **1Password**. + +--- + +### How It Works + +ESS runs as a workload on Control Plane. Your provider configuration and secrets list are stored in a Control Plane secret and mounted into the workload as `sync.yaml`. On startup, ESS schedules a polling loop for each configured secret. At each interval, it fetches the latest value from the external provider and creates or updates the corresponding Control Plane secret via the API. + +ESS tags every secret it manages with `syncer.cpln.io/source` (set to the workload path). This prevents two ESS instances from accidentally overwriting each other's secrets. An hourly cleanup job also deletes any Control Plane secrets that ESS owns but that have been removed from your `sync.yaml` config. + +--- + +### Configuring `values.yaml` + +#### Top-level fields + +| Field | Description | +|---|---| +| `image` | The ESS container image. Do not change unless upgrading. | +| `resources.cpu` / `resources.memory` | Resource limits for the workload container. | +| `port` | Port for the ESS HTTP admin API (default: `3004`). Used for health checks and manual sync triggers. | +| `allowedIp` | List of CIDRs allowed to reach the ESS admin API externally. Replace the placeholder with your IP, or use `0.0.0.0/0` to allow all. | +| `essConfig` | The full sync configuration — providers and secrets (see below). | + +--- + +#### `essConfig.providers` + +Each provider entry requires a unique `name` and exactly one provider block. An optional `syncInterval` sets the default interval for all secrets using that provider. + +**Vault** +```yaml +- name: my-vault + vault: + address: https://my-vault.com:8200 # required + token: # required + syncInterval: 1m # optional — overrides global default +``` + +**AWS Parameter Store** +```yaml +- name: my-aws-ssm + awsParameterStore: + region: us-east-1 + accessKeyId: # optional if using an IAM-linked identity + secretAccessKey: # optional if using an IAM-linked identity +``` + +**AWS Secrets Manager** +```yaml +- name: my-aws-secrets-manager + awsSecretsManager: + region: us-east-1 + accessKeyId: + secretAccessKey: +``` + +**Doppler** +```yaml +- name: my-doppler + doppler: + accessToken: # use a Doppler service token (dp.st....) +``` + +**GCP Secret Manager** +```yaml +- name: my-gcp + gcpSecretManager: + projectId: 123456789876 + credentials: # optional — omit to use Application Default Credentials + clientEmail: + privateKey: +``` + +**1Password** +```yaml +- name: my-1password + onePassword: + serviceAccountToken: + integrationName: my-ess # optional + integrationVersion: 1.0.0 # optional +``` + +--- + +#### `essConfig.secrets` + +Each secret entry syncs one value (or a set of values) from a provider into a Control Plane secret. + +| Field | Description | +|---|---| +| `name` | Name of the Control Plane secret to create or update. | +| `provider` | Must match a provider `name` defined above. | +| `syncInterval` | Optional. Overrides the provider-level and global default for this specific secret. | + +Each secret must use exactly one of the following sync types: + +--- + +##### `opaque` — Single value (stored as a Control Plane `opaque` secret) + +Shorthand (path only, no fallback): +```yaml +- name: my-secret + provider: my-vault + opaque: /v1/secret/data/myapp +``` + +With options: +```yaml +- name: my-secret + provider: my-vault + opaque: + path: /v1/secret/data/myapp # path to fetch + parse: data.password # optional — extract a key from a JSON/YAML response + default: fallback-value # optional — used if fetch fails + encoding: base64 # optional — base64-decode the fetched value +``` + +> **Note:** If you use the shorthand form (`opaque: /some/path`) with no `default`, a fetch failure causes the sync to fail with no fallback. + +--- + +##### `dictionary` — Multiple values (stored as a Control Plane `dictionary` secret) + +Each key in the dictionary is fetched independently: +```yaml +- name: my-secret + provider: my-vault + dictionary: + PORT: + path: /v1/secret/data/app + parse: data.port + default: 5432 + PASSWORD: + path: /v1/secret/data/app + parse: data.password + USERNAME: + path: /v1/secret/data/app + parse: data.username + default: "no username" +``` + +Each key supports `path`, `parse`, `default`, and `encoding` — the same options as `opaque`. A failure on one key does not block others. + +--- + +##### `dictionaryFromProject` — Entire Doppler project (Doppler only) + +Syncs all secrets from a Doppler project+config in one operation, stored as a Control Plane `dictionary` secret: +```yaml +- name: my-doppler-config + provider: my-doppler + dictionaryFromProject: + path: my-project/dev # format: "project/config" — exactly two segments +``` + +> **Note:** `dictionaryFromProject` is only valid with a Doppler provider. Using it with any other provider causes ESS to exit at startup. + +--- + +#### Doppler Path Formats + +| Sync type | Path format | Example | +|---|---|---| +| `opaque` or `dictionary` key | `project/config/SECRET_NAME` | `my-app/production/DATABASE_URL` | +| `dictionaryFromProject` | `project/config` | `my-app/production` | + +--- + +#### Sync Interval Format + +Intervals use the format `hms`. All parts are optional but at least one is required. + +Examples: `10s`, `5m`, `1h`, `1h30m`, `1h30m10s` + +Priority (highest wins): +1. Secret-level `syncInterval` +2. Provider-level `syncInterval` +3. Global default (`300s`) + +--- + +### Important Notes + +- **Conflict protection:** If a Control Plane secret already exists and is managed by a different ESS instance, the sync for that secret will fail. Two ESS instances cannot manage the same secret. +- **Secret type changes:** Changing a secret from `opaque` to `dictionary` (or vice versa) causes ESS to delete the existing secret and recreate it. There is a brief window where the secret does not exist. +- **Cleanup:** ESS runs an hourly job that deletes Control Plane secrets it owns but that no longer appear in `sync.yaml`. Removing a secret from the config will eventually result in its deletion from Control Plane. +- **Doppler `parse`:** The `parse` field only works when the Doppler secret's value is JSON or YAML. Using `parse` on a plain string secret throws an error. +- **`sync.yaml` hot reload:** ESS watches its config file and automatically restarts when changes are detected (every ~5 seconds). No workload restart is needed after updating the config secret. + +### Resources + +- [ESS Documentation](https://docs.controlplane.com/template-catalog/templates/external-secret-syncer) +- [Image Source Code](https://github.com/controlplane-com/external-secret-syncer) \ No newline at end of file diff --git a/ess/versions/1.4.0/templates/_helpers.tpl b/ess/versions/1.4.0/templates/_helpers.tpl new file mode 100644 index 00000000..95668c35 --- /dev/null +++ b/ess/versions/1.4.0/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* Resource Naming */}} + +{{/* +ESS Workload Name +*/}} +{{- define "ess.name" -}} +{{- printf "%s-ess" .Release.Name }} +{{- end }} + +{{/* +ESS Identity Name +*/}} +{{- define "ess.identity.name" -}} +{{- printf "%s-ess-identity" .Release.Name }} +{{- end }} + +{{/* +ESS Policy Name +*/}} +{{- define "ess.policy.name" -}} +{{- printf "%s-ess-policy" .Release.Name }} +{{- end }} + +{{/* +ESS Secret Config Name +*/}} +{{- define "ess.secret.name" -}} +{{- printf "%s-ess-config" .Release.Name }} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels +*/}} +{{- define "ess.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/ess/versions/1.4.0/templates/identity.yaml b/ess/versions/1.4.0/templates/identity.yaml new file mode 100644 index 00000000..e7176ee7 --- /dev/null +++ b/ess/versions/1.4.0/templates/identity.yaml @@ -0,0 +1,5 @@ +kind: identity +gvc: {{ .Values.global.cpln.gvc }} +name: {{ include "ess.identity.name" . }} +description: ESS identity +tags: {{- include "ess.tags" . | nindent 4 }} diff --git a/ess/versions/1.4.0/templates/policy.yaml b/ess/versions/1.4.0/templates/policy.yaml new file mode 100644 index 00000000..cba2f1dd --- /dev/null +++ b/ess/versions/1.4.0/templates/policy.yaml @@ -0,0 +1,10 @@ +kind: policy +name: {{ include "ess.policy.name" . }} +description: ESS policy +bindings: + - permissions: + - manage + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "ess.identity.name" . }} +target: all +targetKind: secret diff --git a/ess/versions/1.4.0/templates/secret.yaml b/ess/versions/1.4.0/templates/secret.yaml new file mode 100644 index 00000000..764bc110 --- /dev/null +++ b/ess/versions/1.4.0/templates/secret.yaml @@ -0,0 +1,9 @@ +kind: secret +name: {{ include "ess.secret.name" . }} +description: ESS config +tags: {{- include "ess.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: | +{{- toYaml .Values.essConfig | nindent 4 }} \ No newline at end of file diff --git a/ess/versions/1.4.0/templates/workload.yaml b/ess/versions/1.4.0/templates/workload.yaml new file mode 100644 index 00000000..a4106066 --- /dev/null +++ b/ess/versions/1.4.0/templates/workload.yaml @@ -0,0 +1,61 @@ +kind: workload +name: {{ include "ess.name" . }} +description: External Secret Syncer +tags: {{- include "ess.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: ess + cpu: {{ .Values.resources.cpu | quote }} + image: {{ .Values.image }} + inheritEnv: false + memory: {{ .Values.resources.memory | quote }} + ports: + - number: {{ .Values.port }} + protocol: http + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + path: /about + port: {{ .Values.port }} + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumes: + - path: /usr/src/app/sync.yaml + recoveryPolicy: retain + uri: cpln://secret/{{ include "ess.secret.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 3 + metric: cpu + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: + {{- toYaml .Values.allowedIp | nindent 8 }} + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: none + inboundAllowWorkload: [] + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "ess.identity.name" . }} + loadBalancer: + direct: + enabled: false + ports: [] + supportDynamicTags: false diff --git a/ess/versions/1.4.0/values.yaml b/ess/versions/1.4.0/values.yaml new file mode 100644 index 00000000..df98db40 --- /dev/null +++ b/ess/versions/1.4.0/values.yaml @@ -0,0 +1,82 @@ +image: ghcr.io/controlplane-com/cpln-build/external-secret-syncer:v1.3.4 + +resources: + cpu: 200m + memory: 256Mi + +port: 3004 + +allowedIp: + - 1.2.3.4 # Replace with your IP + +essConfig: + providers: + - name: my-vault + vault: + address: https://my-vault.com:8200 + token: + syncInterval: 1m + - name: my-aws-ssm + awsParameterStore: + region: us-east-1 + accessKeyId: # alternatively configure identity to natively use AWS permissions + secretAccessKey: # alternatively configure identity to natively use AWS permissions + # - name: my-aws-secrets-manager + # awsSecretsManager: + # region: us-east-1 + # accessKeyId: + # secretAccessKey: + # - name: my-1password + # onePassword: + # serviceAccountToken: + # integrationName: my-ess + # integrationVersion: 1.0.0 + # - name: my-1password-connect + # onePasswordConnect: + # serverURL: https://my-connect-server.example.com + # token: + # - name: my-doppler + # doppler: + # accessToken: + # - name: my-gcp + # gcpSecretManager: + # projectId: 123456789876 + # credentials: + # clientEmail: + # privateKey: + secrets: + - name: auth + provider: my-vault + syncInterval: 20s + dictionary: + PORT: + path: /v1/secret/data/app + parse: data.port + default: 5432 + PASSWORD: + path: /v1/secret/data/app + parse: data.password + USERNAME: + default: "no username" + path: /v1/secret/data/app + parse: data.username + - name: ssm + provider: my-aws + syncInterval: 20s + opaque: /example/app + # - name: secrets-manager + # provider: my-aws-secrets-manager + # dictionary: + # PASSWORD: + # path: /example/app + # parse: password + # - name: doppler-secret + # provider: my-doppler + # opaque: /project/config/SECRET_NAME + # - name: doppler-project + # provider: my-doppler + # dictionaryFromProject: + # path: project/config # syncs all secrets from a Doppler project+config + # - name: gcp + # provider: my-gcp + # opaque: database-password From cb2b7c75f8ea8950e7040673b05a3d8bfaad30a8 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Tue, 5 May 2026 08:45:28 -0700 Subject: [PATCH 17/58] added 1password connect (#254) --- ess/versions/1.4.0/Chart.yaml | 2 +- ess/versions/1.4.0/README.md | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ess/versions/1.4.0/Chart.yaml b/ess/versions/1.4.0/Chart.yaml index 0daa0c04..b177ec9c 100644 --- a/ess/versions/1.4.0/Chart.yaml +++ b/ess/versions/1.4.0/Chart.yaml @@ -3,7 +3,7 @@ name: ess description: External Secret Syncer for Control Plane type: application version: 1.4.0 -appVersion: "v1.3.4" +appVersion: "1.3.4" dependencies: - name: cpln-common diff --git a/ess/versions/1.4.0/README.md b/ess/versions/1.4.0/README.md index ff3e3da5..1b963f0c 100644 --- a/ess/versions/1.4.0/README.md +++ b/ess/versions/1.4.0/README.md @@ -2,7 +2,7 @@ ### Overview -Creates an application that continuously syncs secrets from external providers into Control Plane secrets on a configurable schedule. Supported providers: **HashiCorp Vault**, **AWS Secrets Manager**, **AWS Parameter Store**, **Doppler**, **GCP Secret Manager**, and **1Password**. +Creates an application that continuously syncs secrets from external providers into Control Plane secrets on a configurable schedule. Supported providers: **HashiCorp Vault**, **AWS Secrets Manager**, **AWS Parameter Store**, **Doppler**, **GCP Secret Manager**, **1Password**, and **1Password Connect**. --- @@ -85,6 +85,14 @@ Each provider entry requires a unique `name` and exactly one provider block. An integrationVersion: 1.0.0 # optional ``` +**1Password Connect** +```yaml +- name: my-1password-connect + onePasswordConnect: + serverURL: https://my-connect-server.example.com # required + token: # required +``` + --- #### `essConfig.secrets` From 7691b8f489ace5d5a1cc5fe1a0f5d7ac0f596066 Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Tue, 5 May 2026 14:39:45 -0400 Subject: [PATCH 18/58] redis v3.2.0 - add knobs for repl-backlog-size and repl-timeout. Important for clusters with very high throughput and/or large data sets - add knob for client-output-buffer-limit to avoid problems during long full resyncs. - add cpln/publishNotReadyAddresses: "true" for both redis and sentinel so bootstrapping is always possible even when replicas aren't yet ready - add a more sophisticated readiness probe such that replicas are only ready once they're caught up to the master. - add a knob for requestRetryPolicy --- .../3.2.0/templates/secret-redis-config.yaml | 5 ++- .../3.2.0/templates/workload-redis.yaml | 24 ++++++++++--- .../3.2.0/templates/workload-sentinel.yaml | 11 ++++++ redis/versions/3.2.0/values.yaml | 34 +++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/redis/versions/3.2.0/templates/secret-redis-config.yaml b/redis/versions/3.2.0/templates/secret-redis-config.yaml index 7b35f60d..6f894191 100644 --- a/redis/versions/3.2.0/templates/secret-redis-config.yaml +++ b/redis/versions/3.2.0/templates/secret-redis-config.yaml @@ -9,4 +9,7 @@ data: save 900 1 save 300 10 save 60 10000 - appendonly yes \ No newline at end of file + appendonly yes + repl-backlog-size {{ .Values.redis.replication.backlogSize }} + repl-timeout {{ .Values.redis.replication.timeout }} + client-output-buffer-limit slave {{ .Values.redis.replication.slaveOutputBufferLimit }} \ No newline at end of file diff --git a/redis/versions/3.2.0/templates/workload-redis.yaml b/redis/versions/3.2.0/templates/workload-redis.yaml index 91cb9e8d..ec85d083 100644 --- a/redis/versions/3.2.0/templates/workload-redis.yaml +++ b/redis/versions/3.2.0/templates/workload-redis.yaml @@ -6,6 +6,11 @@ tags: {{- if .Values.redis.tags }} {{ toYaml .Values.redis.tags | indent 2 }} {{- end }} + # Sentinel discovery and replica startup must resolve `.:6379` even + # while the pod is still resyncing or marked NotReady by the replication-aware + # readiness probe. This tag exposes not-yet-Ready pods on the headless service so + # peers can reach them for replication and Sentinel can monitor them. + cpln/publishNotReadyAddresses: "true" {{- include "redis.tags" . | nindent 2 }} spec: type: stateful @@ -88,10 +93,17 @@ spec: {{- else }} PORT=6379 {{- end }} - if [ ! -z "$CUSTOM_REDIS_PASSWORD" ]; then - redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping; - else - redis-cli -p $PORT ping; + rcli() { + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" "$@" + else + redis-cli -p $PORT "$@" + fi + } + rcli ping >/dev/null && \ + if [ "$(rcli role | head -1)" = "slave" ]; then + [ "$(rcli info replication | awk -F: '/master_link_status/{print $2}' | tr -d '\r')" = "up" ] && \ + [ "$(rcli info replication | awk -F: '/master_sync_in_progress/{print $2}' | tr -d '\r')" = "0" ] fi failureThreshold: 10 initialDelaySeconds: 10 @@ -139,6 +151,10 @@ spec: multiZone: enabled: false {{- end }} +{{- if .Values.redis.requestRetryPolicy }} + requestRetryPolicy: +{{ toYaml .Values.redis.requestRetryPolicy | indent 4 }} +{{- end }} {{- if .Values.redis.firewall }} firewallConfig: {{- if or (hasKey .Values.redis.firewall "external_inboundAllowCIDR") (hasKey .Values.redis.firewall "external_outboundAllowCIDR") }} diff --git a/redis/versions/3.2.0/templates/workload-sentinel.yaml b/redis/versions/3.2.0/templates/workload-sentinel.yaml index 6ef60f7c..b2282b2d 100644 --- a/redis/versions/3.2.0/templates/workload-sentinel.yaml +++ b/redis/versions/3.2.0/templates/workload-sentinel.yaml @@ -6,6 +6,13 @@ tags: {{- if .Values.sentinel.tags }} {{ toYaml .Values.sentinel.tags | indent 2 }} {{- end }} + # Currently a no-op: the sentinel readiness probe is a plain `redis-cli ping`, so + # pods become Ready as soon as the port answers and the headless service exposes + # them anyway. Set here as a hedge — if the probe is ever tightened (e.g. to + # require quorum visibility or a known master), peers must still resolve each + # other via `.` during cold start to form the quorum, otherwise + # they deadlock: NotReady because no peers, no peers because NotReady. + cpln/publishNotReadyAddresses: "true" {{- include "redis.tags" . | nindent 2 }} spec: type: stateful @@ -151,6 +158,10 @@ spec: multiZone: enabled: false {{- end }} +{{- if .Values.sentinel.requestRetryPolicy }} + requestRetryPolicy: +{{ toYaml .Values.sentinel.requestRetryPolicy | indent 4 }} +{{- end }} {{- if .Values.sentinel.firewall }} firewallConfig: {{- if or (hasKey .Values.sentinel.firewall "external_inboundAllowCIDR") (hasKey .Values.sentinel.firewall "external_outboundAllowCIDR") }} diff --git a/redis/versions/3.2.0/values.yaml b/redis/versions/3.2.0/values.yaml index 480f054e..737dda09 100644 --- a/redis/versions/3.2.0/values.yaml +++ b/redis/versions/3.2.0/values.yaml @@ -32,6 +32,30 @@ redis: # external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list env: [] tags: {} + # requestRetryPolicy: + # attempts: 2 + # retryOn: + # - connect-failure + # - refused-stream + # - unavailable + # - cancelled + # - resource-exhausted + # - retriable-status-codes + requestRetryPolicy: {} + # Replication tuning. See secret-redis-config.yaml for how these are rendered. + # backlogSize: sized for (peak write throughput × tolerable disconnect window). + # 1mb (Redis default) escalates any brief disconnect to a full RDB resync. 1gb + # covers ~5 minutes of disconnect at ~3MB/s of writes. + # timeout (seconds): bound on full-resync transfer + RDB load + heartbeat. + # 60s (Redis default) is too low for multi-GB datasets — master/slave drop the + # link mid-sync. 300s covers ~30GB at typical 1Gbps + load throughput. + # slaveOutputBufferLimit: " ". Default + # "256mb 64mb 60" can't sustain a full resync of a multi-GB dataset at high + # write rate — master kills the replica mid-stream. Bump for production loads. + replication: + backlogSize: 1gb + timeout: 300 + slaveOutputBufferLimit: "2gb 512mb 300" dataDir: /data persistence: enabled: false @@ -86,6 +110,16 @@ sentinel: # external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list env: [] tags: {} + # requestRetryPolicy: + # attempts: 2 + # retryOn: + # - connect-failure + # - refused-stream + # - unavailable + # - cancelled + # - resource-exhausted + # - retriable-status-codes + requestRetryPolicy: {} persistence: enabled: false volumes: From 9360a71ac35a3b978c36308bc16cfa0f1b9d0b13 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 28 Apr 2026 18:02:21 -0700 Subject: [PATCH 19/58] updated template config --- cassandra/versions/1.0.0/Chart.yaml | 2 +- cassandra/versions/1.0.0/templates/secret-init.yaml | 2 +- cassandra/versions/1.0.0/templates/volumeset.yaml | 4 ++-- .../versions/1.0.0/templates/workload-cassandra.yaml | 10 +++------- cassandra/versions/1.0.0/values.yaml | 4 +--- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/cassandra/versions/1.0.0/Chart.yaml b/cassandra/versions/1.0.0/Chart.yaml index b4b5fc7a..9073d1b9 100644 --- a/cassandra/versions/1.0.0/Chart.yaml +++ b/cassandra/versions/1.0.0/Chart.yaml @@ -7,7 +7,7 @@ appVersion: "5.0" annotations: created: "2026-04-21" - lastModified: "2026-04-23" + lastModified: "2026-04-28" category: "database" createsGvc: true diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index e3550db7..c2ca8d3c 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -69,7 +69,7 @@ data: # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) - cp /config-template/cassandra-template.yaml /etc/cassandra/cassandra.yaml + cp /cassandra-config/cassandra.yaml /etc/cassandra/cassandra.yaml sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${MY_CPLN_FQDN}|g" /etc/cassandra/cassandra.yaml sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml diff --git a/cassandra/versions/1.0.0/templates/volumeset.yaml b/cassandra/versions/1.0.0/templates/volumeset.yaml index de173320..4b414d00 100644 --- a/cassandra/versions/1.0.0/templates/volumeset.yaml +++ b/cassandra/versions/1.0.0/templates/volumeset.yaml @@ -5,8 +5,8 @@ gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} spec: initialCapacity: {{ .Values.cassandra.volumes.data.initialCapacity }} - performanceClass: {{ .Values.cassandra.volumes.data.performanceClass }} - fileSystemType: {{ .Values.cassandra.volumes.data.fileSystemType }} + performanceClass: general-purpose-ssd + fileSystemType: ext4 autoscaling: maxCapacity: {{ .Values.cassandra.volumes.data.autoscaling.maxCapacity }} minFreePercentage: {{ .Values.cassandra.volumes.data.autoscaling.minFreePercentage }} diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 2f3a67cd..16748124 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -30,8 +30,8 @@ spec: - '-c' - nodetool drain image: {{ .Values.cassandra.image }} - cpu: '{{ .Values.cassandra.cpu }}' - memory: {{ .Values.cassandra.memory }} + cpu: {{ .Values.cassandra.cpu | quote }} + memory: {{ .Values.cassandra.memory | quote }} inheritEnv: false ports: - number: 9042 @@ -58,7 +58,7 @@ spec: - path: /var/lib/cassandra recoveryPolicy: retain uri: 'cpln://volumeset/{{ include "cassandra.volumeset.name" . }}' - - path: /config-template/cassandra-template.yaml + - path: /cassandra-config/cassandra.yaml recoveryPolicy: retain uri: 'cpln://secret/{{ include "cassandra.secret.config.name" . }}' - path: /scripts/cassandra-init.sh @@ -77,9 +77,6 @@ spec: suspend: {{ .Values.cassandra.suspend }} timeoutSeconds: 60 firewallConfig: - external: - outboundAllowCIDR: - - 0.0.0.0/0 internal: inboundAllowType: {{ .Values.internal_access.type }} {{- if .Values.internal_access.workloads }} @@ -89,7 +86,6 @@ spec: loadBalancer: direct: enabled: false - ports: [] replicaDirect: true localOptions: {{- range $location := .Values.gvc.locations }} diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index f28d169d..75dfb885 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -10,7 +10,7 @@ gvc: cassandra: image: cassandra:5.0 - cpu: '2' + cpu: 2 memory: 8Gi # JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) # Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. @@ -21,8 +21,6 @@ cassandra: volumes: data: initialCapacity: 10 - performanceClass: general-purpose-ssd - fileSystemType: ext4 autoscaling: maxCapacity: 100 minFreePercentage: 20 From c2184a1af46cb06bd2a5cd4bf3ddf1b249dcf6bc Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 28 Apr 2026 18:08:25 -0700 Subject: [PATCH 20/58] updated values file --- cassandra/versions/1.0.0/templates/workload-cassandra.yaml | 2 +- cassandra/versions/1.0.0/values.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 16748124..92456365 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -74,7 +74,7 @@ spec: target: 95 capacityAI: false debug: false - suspend: {{ .Values.cassandra.suspend }} + suspend: false timeoutSeconds: 60 firewallConfig: internal: diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 75dfb885..6f1afe37 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -15,9 +15,7 @@ cassandra: # JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) # Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. jvmHeapSize: 2G - clusterName: cp-cassandra - suspend: false - + clusterName: my-cassandra volumes: data: initialCapacity: 10 From 944c13aaa541365142bd8edc9bd7f7bb132a8975 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 28 Apr 2026 18:42:29 -0700 Subject: [PATCH 21/58] updated startup script --- cassandra/versions/1.0.0/templates/secret-init.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index c2ca8d3c..49c4ab66 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -64,8 +64,8 @@ data: # Write cassandra-rackdc.properties for DC/rack awareness. # DC = location name (e.g. aws-us-east-2), rack = rack1. mkdir -p /etc/cassandra - printf 'dc=%s\nrack=rack1\n' "${LOCATION}" > /etc/cassandra/cassandra-rackdc.properties - echo "cassandra-rackdc.properties written: dc=${LOCATION} rack=rack1" + printf 'dc=%s\nrack=rack1\nprefer_local=true\n' "${LOCATION}" > /etc/cassandra/cassandra-rackdc.properties + echo "cassandra-rackdc.properties written: dc=${LOCATION} rack=rack1 prefer_local=true" # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) From ac868ae425a315c2aad6cac3cc4aa7095d8f6aad Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Wed, 29 Apr 2026 10:48:00 -0700 Subject: [PATCH 22/58] updated script so all replicas are seeds --- .../versions/1.0.0/templates/secret-init.yaml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 49c4ab66..d9909afe 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -41,17 +41,19 @@ data: echo "LOCATION: ${LOCATION}" echo "GVC: ${GVC}" - # Build seed list: 2 replicas from each location using cpln.local addresses. - # This ensures cross-DC gossip bootstrap and keeps seed count stable. - SEED_COUNT=2 + # Build seed list: ALL replicas from all locations using cpln.local addresses. + # Every node must find its own broadcast_address in the seeds list so that + # Cassandra's shouldBootstrap() returns false and skips the shadow round. + # The shadow round requires peers to connect back to this node's broadcast_address + # (the cpln.local VIP), which is not routable before the readiness probe passes. + # Skipping bootstrap is safe here: persistent volumes retain data across restarts, + # and fresh clusters have no data to stream. SEEDS="" - LOCATIONS="{{ range .Values.gvc.locations }}{{ .name }} {{ end }}" - for loc in $LOCATIONS; do - for i in $(seq 0 $((SEED_COUNT - 1))); do - SEED_FQDN="replica-${i}.${WORKLOAD}.${loc}.${GVC}.cpln.local" - SEEDS="${SEEDS}${SEED_FQDN}," - done + {{- range $loc := .Values.gvc.locations }} + for i in $(seq 0 $(({{ $loc.replicas | int }} - 1))); do + SEEDS="${SEEDS}replica-${i}.${WORKLOAD}.{{ $loc.name }}.${GVC}.cpln.local," done + {{- end }} SEEDS="${SEEDS%,}" echo "SEEDS: ${SEEDS}" From 7774307da8da135d9eb76bac9186fbc341e3bbd9 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Wed, 29 Apr 2026 17:51:57 -0700 Subject: [PATCH 23/58] reverted to single location --- .../versions/1.0.0/templates/_helpers.tpl | 26 ----------------- cassandra/versions/1.0.0/templates/gvc.yaml | 11 -------- .../versions/1.0.0/templates/secret-init.yaml | 27 ++++-------------- .../1.0.0/templates/workload-cassandra.yaml | 28 +++---------------- cassandra/versions/1.0.0/values.yaml | 11 +++----- 5 files changed, 14 insertions(+), 89 deletions(-) delete mode 100644 cassandra/versions/1.0.0/templates/gvc.yaml diff --git a/cassandra/versions/1.0.0/templates/_helpers.tpl b/cassandra/versions/1.0.0/templates/_helpers.tpl index 450d9046..65d8c8a0 100644 --- a/cassandra/versions/1.0.0/templates/_helpers.tpl +++ b/cassandra/versions/1.0.0/templates/_helpers.tpl @@ -25,32 +25,6 @@ {{- end }} -{{/* Validation */}} - -{{/* -Validate locations: requires at least 1. If more than 1, requires at least 3. -Each location must have odd replicas >= 3. -*/}} -{{- define "cassandra.validateLocations" -}} -{{- $locations := .Values.gvc.locations -}} -{{- if lt (len $locations) 1 -}} - {{- fail "gvc.locations must contain at least one location." -}} -{{- end -}} -{{- if and (gt (len $locations) 1) (lt (len $locations) 3) -}} - {{- fail "Multi-location Cassandra requires at least 3 locations for cross-DC quorum." -}} -{{- end -}} -{{- range $locations -}} - {{- $r := .replicas | int -}} - {{- if lt $r 3 -}} - {{- fail (printf "Location %s: replicas must be at least 3." .name) -}} - {{- end -}} - {{- if eq (mod $r 2) 0 -}} - {{- fail (printf "Location %s: replicas must be an odd number (3, 5, 7, ...) for quorum." .name) -}} - {{- end -}} -{{- end -}} -{{- end }} - - {{/* Labeling */}} {{- define "cassandra.tags" -}} diff --git a/cassandra/versions/1.0.0/templates/gvc.yaml b/cassandra/versions/1.0.0/templates/gvc.yaml deleted file mode 100644 index 744088fb..00000000 --- a/cassandra/versions/1.0.0/templates/gvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: gvc -name: {{ .Values.gvc.name }} -description: {{ .Values.gvc.name }} -tags: {{- include "cassandra.tags" . | nindent 2 }} -spec: - endpointNamingFormat: org - staticPlacement: - locationLinks: - {{- range .Values.gvc.locations }} - - //location/{{ .name }} - {{- end }} diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index d9909afe..282fc6cd 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -1,4 +1,3 @@ -{{ include "cassandra.validateLocations" . -}} kind: secret name: {{ include "cassandra.secret.init.name" . }} type: opaque @@ -24,36 +23,22 @@ data: SERVICE=$(echo "${MY_FQDN}" | cut -d'.' -f2) REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') - # Location and GVC from environment (injected by Control Plane / workload env). + # Location from environment (injected by Control Plane). LOCATION=$(basename "${CPLN_LOCATION}") - GVC="${CASSANDRA_GVC}" - WORKLOAD="${CASSANDRA_WORKLOAD}" - - # cpln.local address — reachable cross-location, used as broadcast_address. - MY_CPLN_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD}.${LOCATION}.${GVC}.cpln.local" echo "HOSTNAME: ${HOSTNAME}" echo "MY_FQDN: ${MY_FQDN}" - echo "MY_CPLN_FQDN: ${MY_CPLN_FQDN}" echo "NAMESPACE_HASH: ${NAMESPACE_HASH}" echo "SERVICE: ${SERVICE}" echo "REPLICA_INDEX: ${REPLICA_INDEX}" echo "LOCATION: ${LOCATION}" - echo "GVC: ${GVC}" - # Build seed list: ALL replicas from all locations using cpln.local addresses. - # Every node must find its own broadcast_address in the seeds list so that - # Cassandra's shouldBootstrap() returns false and skips the shadow round. - # The shadow round requires peers to connect back to this node's broadcast_address - # (the cpln.local VIP), which is not routable before the readiness probe passes. - # Skipping bootstrap is safe here: persistent volumes retain data across restarts, - # and fresh clusters have no data to stream. + # Seeds: all replicas' internal FQDNs within this location. + # These resolve directly to pod IPs (10.x.x.x) via cluster DNS — no Envoy involved. SEEDS="" - {{- range $loc := .Values.gvc.locations }} - for i in $(seq 0 $(({{ $loc.replicas | int }} - 1))); do - SEEDS="${SEEDS}replica-${i}.${WORKLOAD}.{{ $loc.name }}.${GVC}.cpln.local," + for i in $(seq 0 $(( ${CASSANDRA_REPLICAS} - 1 ))); do + SEEDS="${SEEDS}replica-${i}.${SERVICE}.${NAMESPACE_HASH}.svc.cluster.local," done - {{- end }} SEEDS="${SEEDS%,}" echo "SEEDS: ${SEEDS}" @@ -73,7 +58,7 @@ data: # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) cp /cassandra-config/cassandra.yaml /etc/cassandra/cassandra.yaml sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml - sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${MY_CPLN_FQDN}|g" /etc/cassandra/cassandra.yaml + sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml echo "cassandra.yaml written. Starting Cassandra..." diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 92456365..d7630b36 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -1,4 +1,3 @@ -{{- include "cassandra.validateLocations" . -}} kind: workload name: {{ include "cassandra.workload.name" . }} description: Cassandra cluster @@ -18,10 +17,8 @@ spec: env: - name: MAX_HEAP_SIZE value: {{ .Values.cassandra.jvmHeapSize | quote }} - - name: CASSANDRA_GVC - value: {{ .Values.gvc.name | quote }} - - name: CASSANDRA_WORKLOAD - value: {{ include "cassandra.workload.name" . | quote }} + - name: CASSANDRA_REPLICAS + value: {{ .Values.cassandra.replicas | quote }} lifecycle: preStop: exec: @@ -36,8 +33,6 @@ spec: ports: - number: 9042 protocol: tcp - - number: 7000 - protocol: tcp livenessProbe: failureThreshold: 5 initialDelaySeconds: 120 @@ -67,9 +62,9 @@ spec: defaultOptions: autoscaling: maxConcurrency: 0 - maxScale: {{ (index .Values.gvc.locations 0).replicas | int }} + maxScale: {{ .Values.cassandra.replicas | int }} metric: disabled - minScale: {{ (index .Values.gvc.locations 0).replicas | int }} + minScale: {{ .Values.cassandra.replicas | int }} scaleToZeroDelay: 300 target: 95 capacityAI: false @@ -87,21 +82,6 @@ spec: direct: enabled: false replicaDirect: true - localOptions: - {{- range $location := .Values.gvc.locations }} - - autoscaling: - maxConcurrency: 0 - maxScale: {{ if eq ($location.replicas | int) 0 }}1{{ else }}{{ $location.replicas | int }}{{ end }} - metric: disabled - minScale: {{ if eq ($location.replicas | int) 0 }}0{{ else }}{{ $location.replicas | int }}{{ end }} - scaleToZeroDelay: 300 - target: 95 - capacityAI: false - debug: false - location: //location/{{ $location.name }} - suspend: {{ if eq ($location.replicas | int) 0 }}true{{ else }}false{{ end }} - timeoutSeconds: 60 - {{- end }} rolloutOptions: maxSurgeReplicas: 0% maxUnavailableReplicas: '1' diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 6f1afe37..104668b2 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,14 +1,11 @@ gvc: name: cassandra-gvc - locations: - - name: aws-us-east-2 - replicas: 3 - - name: aws-us-west-2 - replicas: 3 - - name: aws-eu-central-1 - replicas: 3 cassandra: + replicas: 3 + # Set to true when deploying across multiple availability zones. + # Ensures Cassandra's rack awareness is enabled (dc=location, rack=rack1). + multiZone: false image: cassandra:5.0 cpu: 2 memory: 8Gi From 426d7b4757a6c2ac2870ef8f6775b53e3126b840 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 5 May 2026 17:25:25 -0700 Subject: [PATCH 24/58] updated node communication port, reverted to multi location setup --- cassandra/versions/1.0.0/templates/gvc.yaml | 11 ++++++ .../1.0.0/templates/secret-config.yaml | 10 +++-- .../versions/1.0.0/templates/secret-init.yaml | 32 ++++++++-------- .../versions/1.0.0/templates/volumeset.yaml | 8 ++-- .../1.0.0/templates/workload-cassandra.yaml | 35 +++++++++++++---- cassandra/versions/1.0.0/values.yaml | 38 +++++++++---------- 6 files changed, 84 insertions(+), 50 deletions(-) create mode 100644 cassandra/versions/1.0.0/templates/gvc.yaml diff --git a/cassandra/versions/1.0.0/templates/gvc.yaml b/cassandra/versions/1.0.0/templates/gvc.yaml new file mode 100644 index 00000000..744088fb --- /dev/null +++ b/cassandra/versions/1.0.0/templates/gvc.yaml @@ -0,0 +1,11 @@ +kind: gvc +name: {{ .Values.gvc.name }} +description: {{ .Values.gvc.name }} +tags: {{- include "cassandra.tags" . | nindent 2 }} +spec: + endpointNamingFormat: org + staticPlacement: + locationLinks: + {{- range .Values.gvc.locations }} + - //location/{{ .name }} + {{- end }} diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml index b344f3ee..74534a34 100644 --- a/cassandra/versions/1.0.0/templates/secret-config.yaml +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -4,11 +4,15 @@ type: opaque data: encoding: plain payload: | - cluster_name: '{{ .Values.cassandra.clusterName }}' + cluster_name: '{{ .Values.clusterName }}' - # Networking — replaced at pod startup by the init script - listen_address: LISTEN_ADDRESS_PLACEHOLDER + # Networking — broadcast_address replaced at pod startup by the init script. + # listen_interface: lo binds gossip port 7000 to loopback so Envoy's inbound + # proxy (which forwards to 127.0.0.1) can reach Cassandra the same way it does + # for rpc_address 0.0.0.0 on port 9042. + listen_interface: lo broadcast_address: BROADCAST_ADDRESS_PLACEHOLDER + storage_port: 9043 rpc_address: 0.0.0.0 broadcast_rpc_address: BROADCAST_ADDRESS_PLACEHOLDER diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 282fc6cd..1d2fedf7 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -7,7 +7,7 @@ data: #!/bin/bash set -euo pipefail - # Derive own internal FQDN from /etc/hosts. + # Derive own internal FQDN and pod IP from /etc/hosts. # Control Plane inserts a line like: # 10.x.x.x cassandra-0.cassandra..svc.cluster.local cassandra-0 MY_FQDN=$(grep -E "^[0-9]" /etc/hosts | grep "${HOSTNAME}" | awk '{print $2}') @@ -18,30 +18,31 @@ data: exit 1 fi - # Parse components from the FQDN. - NAMESPACE_HASH=$(echo "${MY_FQDN}" | cut -d'.' -f3) - SERVICE=$(echo "${MY_FQDN}" | cut -d'.' -f2) + MY_IP=$(grep -E "^[0-9]" /etc/hosts | grep "${HOSTNAME}" | awk '{print $1}') REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') - - # Location from environment (injected by Control Plane). LOCATION=$(basename "${CPLN_LOCATION}") echo "HOSTNAME: ${HOSTNAME}" echo "MY_FQDN: ${MY_FQDN}" - echo "NAMESPACE_HASH: ${NAMESPACE_HASH}" - echo "SERVICE: ${SERVICE}" + echo "MY_IP: ${MY_IP}" echo "REPLICA_INDEX: ${REPLICA_INDEX}" echo "LOCATION: ${LOCATION}" - # Seeds: all replicas' internal FQDNs within this location. - # These resolve directly to pod IPs (10.x.x.x) via cluster DNS — no Envoy involved. + # Broadcast address: replicaDirect cpln.local FQDN for this replica. + # Resolves to a stable VIP that Envoy routes cross-location on port 9043. + BROADCAST_ADDR="replica-${REPLICA_INDEX}.${CASSANDRA_WORKLOAD}.${LOCATION}.${CASSANDRA_GVC}.cpln.local" + echo "BROADCAST_ADDR: ${BROADCAST_ADDR}" + + # Seeds: replicaDirect FQDNs for all replicas in all locations. + # Stable across redeploys — no pod IPs needed. SEEDS="" - for i in $(seq 0 $(( ${CASSANDRA_REPLICAS} - 1 ))); do - SEEDS="${SEEDS}replica-${i}.${SERVICE}.${NAMESPACE_HASH}.svc.cluster.local," + {{- range .Values.gvc.locations }} + for i in $(seq 0 $(( {{ .replicas | int }} - 1 ))); do + SEEDS="${SEEDS}replica-${i}.${CASSANDRA_WORKLOAD}.{{ .name }}.${CASSANDRA_GVC}.cpln.local," done + {{- end }} SEEDS="${SEEDS%,}" - - echo "SEEDS: ${SEEDS}" + echo "SEEDS: ${SEEDS}" # Cassandra data dir ownership (cassandra runs as uid 999 in the official image). mkdir -p /var/lib/cassandra @@ -57,8 +58,7 @@ data: # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) cp /cassandra-config/cassandra.yaml /etc/cassandra/cassandra.yaml - sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml - sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml + sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${BROADCAST_ADDR}|g" /etc/cassandra/cassandra.yaml sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml echo "cassandra.yaml written. Starting Cassandra..." diff --git a/cassandra/versions/1.0.0/templates/volumeset.yaml b/cassandra/versions/1.0.0/templates/volumeset.yaml index 4b414d00..12af3e3f 100644 --- a/cassandra/versions/1.0.0/templates/volumeset.yaml +++ b/cassandra/versions/1.0.0/templates/volumeset.yaml @@ -4,10 +4,10 @@ description: {{ include "cassandra.workload.name" . }} data gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} spec: - initialCapacity: {{ .Values.cassandra.volumes.data.initialCapacity }} + initialCapacity: {{ .Values.volumes.data.initialCapacity }} performanceClass: general-purpose-ssd fileSystemType: ext4 autoscaling: - maxCapacity: {{ .Values.cassandra.volumes.data.autoscaling.maxCapacity }} - minFreePercentage: {{ .Values.cassandra.volumes.data.autoscaling.minFreePercentage }} - scalingFactor: {{ .Values.cassandra.volumes.data.autoscaling.scalingFactor }} + maxCapacity: {{ .Values.volumes.data.autoscaling.maxCapacity }} + minFreePercentage: {{ .Values.volumes.data.autoscaling.minFreePercentage }} + scalingFactor: {{ .Values.volumes.data.autoscaling.scalingFactor }} diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index d7630b36..7a92cbe9 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -16,9 +16,11 @@ spec: /tmp/cassandra-init.sh env: - name: MAX_HEAP_SIZE - value: {{ .Values.cassandra.jvmHeapSize | quote }} - - name: CASSANDRA_REPLICAS - value: {{ .Values.cassandra.replicas | quote }} + value: {{ .Values.jvmHeapSize | quote }} + - name: CASSANDRA_WORKLOAD + value: {{ include "cassandra.workload.name" . | quote }} + - name: CASSANDRA_GVC + value: {{ .Values.gvc.name | quote }} lifecycle: preStop: exec: @@ -26,13 +28,15 @@ spec: - bash - '-c' - nodetool drain - image: {{ .Values.cassandra.image }} - cpu: {{ .Values.cassandra.cpu | quote }} - memory: {{ .Values.cassandra.memory | quote }} + image: {{ .Values.image }} + cpu: {{ .Values.cpu | quote }} + memory: {{ .Values.memory | quote }} inheritEnv: false ports: - number: 9042 protocol: tcp + - number: 9043 + protocol: tcp livenessProbe: failureThreshold: 5 initialDelaySeconds: 120 @@ -62,15 +66,30 @@ spec: defaultOptions: autoscaling: maxConcurrency: 0 - maxScale: {{ .Values.cassandra.replicas | int }} + maxScale: {{ (index .Values.gvc.locations 0).replicas | int }} metric: disabled - minScale: {{ .Values.cassandra.replicas | int }} + minScale: {{ (index .Values.gvc.locations 0).replicas | int }} scaleToZeroDelay: 300 target: 95 capacityAI: false debug: false suspend: false timeoutSeconds: 60 + localOptions: + {{- range .Values.gvc.locations }} + - location: //location/{{ .name }} + autoscaling: + maxConcurrency: 0 + maxScale: {{ .replicas | int }} + metric: disabled + minScale: {{ .replicas | int }} + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 60 + {{- end }} firewallConfig: internal: inboundAllowType: {{ .Values.internal_access.type }} diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 104668b2..754fcf1c 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,25 +1,25 @@ gvc: name: cassandra-gvc + locations: + - name: aws-us-east-2 + replicas: 1 + - name: aws-us-west-2 + replicas: 1 -cassandra: - replicas: 3 - # Set to true when deploying across multiple availability zones. - # Ensures Cassandra's rack awareness is enabled (dc=location, rack=rack1). - multiZone: false - image: cassandra:5.0 - cpu: 2 - memory: 8Gi - # JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) - # Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. - jvmHeapSize: 2G - clusterName: my-cassandra - volumes: - data: - initialCapacity: 10 - autoscaling: - maxCapacity: 100 - minFreePercentage: 20 - scalingFactor: 1.5 +image: cassandra:5.0 +cpu: 2 +memory: 8Gi +# JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) +# Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. +jvmHeapSize: 2G +clusterName: my-cassandra +volumes: + data: + initialCapacity: 10 + autoscaling: + maxCapacity: 100 + minFreePercentage: 20 + scalingFactor: 1.5 internal_access: type: same-gvc From b54c509c0fca00516276a88d878c8d137b95adfd Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Wed, 6 May 2026 12:06:50 -0700 Subject: [PATCH 25/58] updated listen address and broadcast address --- cassandra/versions/1.0.0/templates/secret-config.yaml | 9 ++++----- cassandra/versions/1.0.0/templates/secret-init.yaml | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml index 74534a34..2034f1f4 100644 --- a/cassandra/versions/1.0.0/templates/secret-config.yaml +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -6,11 +6,10 @@ data: payload: | cluster_name: '{{ .Values.clusterName }}' - # Networking — broadcast_address replaced at pod startup by the init script. - # listen_interface: lo binds gossip port 7000 to loopback so Envoy's inbound - # proxy (which forwards to 127.0.0.1) can reach Cassandra the same way it does - # for rpc_address 0.0.0.0 on port 9042. - listen_interface: lo + # Networking — replaced at pod startup by the init script. + # listen_address binds the gossip port to the pod IP; Envoy's inbound proxy + # forwards to the original destination (pod IP) via ORIGINAL_DST. + listen_address: LISTEN_ADDRESS_PLACEHOLDER broadcast_address: BROADCAST_ADDRESS_PLACEHOLDER storage_port: 9043 rpc_address: 0.0.0.0 diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 1d2fedf7..f23e6c8f 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -58,6 +58,7 @@ data: # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) cp /cassandra-config/cassandra.yaml /etc/cassandra/cassandra.yaml + sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${BROADCAST_ADDR}|g" /etc/cassandra/cassandra.yaml sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml From df0651b733d0f61ec2c2f8d4b4527859ef212565 Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Tue, 5 May 2026 15:25:15 -0400 Subject: [PATCH 26/58] debezium + cdc pipeline templates --- cdc-pipeline/versions/1.0.0/Chart.yaml | 26 ++ cdc-pipeline/versions/1.0.0/README.md | 86 ++++ .../versions/1.0.0/templates/_helpers.tpl | 80 ++++ .../versions/1.0.0/templates/validation.yaml | 3 + cdc-pipeline/versions/1.0.0/values.yaml | 379 ++++++++++++++++++ debezium-server/deploy-aws-us-east-2.yaml | 37 ++ debezium-server/icon.png | Bin 0 -> 2367 bytes debezium-server/versions/1.0.0/Chart.yaml | 12 + debezium-server/versions/1.0.0/README.md | 347 ++++++++++++++++ .../versions/1.0.0/templates/_helpers.tpl | 264 ++++++++++++ .../versions/1.0.0/templates/identity.yaml | 19 + .../versions/1.0.0/templates/policy.yaml | 17 + .../1.0.0/templates/secret-config.yaml | 260 ++++++++++++ .../1.0.0/templates/secret-credentials.yaml | 99 +++++ .../1.0.0/templates/secret-entrypoint.yaml | 46 +++ .../versions/1.0.0/templates/volumeset.yaml | 15 + .../1.0.0/templates/workload-debezium.yaml | 221 ++++++++++ debezium-server/versions/1.0.0/values.yaml | 219 ++++++++++ debezium-server/versions/1.1.0/Chart.yaml | 12 + debezium-server/versions/1.1.0/README.md | 347 ++++++++++++++++ .../versions/1.1.0/templates/_helpers.tpl | 293 ++++++++++++++ .../versions/1.1.0/templates/identity.yaml | 19 + .../versions/1.1.0/templates/policy.yaml | 17 + .../1.1.0/templates/secret-config.yaml | 260 ++++++++++++ .../1.1.0/templates/secret-credentials.yaml | 99 +++++ .../1.1.0/templates/secret-entrypoint.yaml | 46 +++ .../versions/1.1.0/templates/volumeset.yaml | 15 + .../1.1.0/templates/workload-debezium.yaml | 221 ++++++++++ debezium-server/versions/1.1.0/values.yaml | 219 ++++++++++ etcd/versions/1.4.0/templates/_helpers.tpl | 32 +- etcd/versions/1.4.0/templates/secret.yaml | 24 +- 31 files changed, 3720 insertions(+), 14 deletions(-) create mode 100644 cdc-pipeline/versions/1.0.0/Chart.yaml create mode 100644 cdc-pipeline/versions/1.0.0/README.md create mode 100644 cdc-pipeline/versions/1.0.0/templates/_helpers.tpl create mode 100644 cdc-pipeline/versions/1.0.0/templates/validation.yaml create mode 100644 cdc-pipeline/versions/1.0.0/values.yaml create mode 100644 debezium-server/deploy-aws-us-east-2.yaml create mode 100644 debezium-server/icon.png create mode 100644 debezium-server/versions/1.0.0/Chart.yaml create mode 100644 debezium-server/versions/1.0.0/README.md create mode 100644 debezium-server/versions/1.0.0/templates/_helpers.tpl create mode 100644 debezium-server/versions/1.0.0/templates/identity.yaml create mode 100644 debezium-server/versions/1.0.0/templates/policy.yaml create mode 100644 debezium-server/versions/1.0.0/templates/secret-config.yaml create mode 100644 debezium-server/versions/1.0.0/templates/secret-credentials.yaml create mode 100644 debezium-server/versions/1.0.0/templates/secret-entrypoint.yaml create mode 100644 debezium-server/versions/1.0.0/templates/volumeset.yaml create mode 100644 debezium-server/versions/1.0.0/templates/workload-debezium.yaml create mode 100644 debezium-server/versions/1.0.0/values.yaml create mode 100644 debezium-server/versions/1.1.0/Chart.yaml create mode 100644 debezium-server/versions/1.1.0/README.md create mode 100644 debezium-server/versions/1.1.0/templates/_helpers.tpl create mode 100644 debezium-server/versions/1.1.0/templates/identity.yaml create mode 100644 debezium-server/versions/1.1.0/templates/policy.yaml create mode 100644 debezium-server/versions/1.1.0/templates/secret-config.yaml create mode 100644 debezium-server/versions/1.1.0/templates/secret-credentials.yaml create mode 100644 debezium-server/versions/1.1.0/templates/secret-entrypoint.yaml create mode 100644 debezium-server/versions/1.1.0/templates/volumeset.yaml create mode 100644 debezium-server/versions/1.1.0/templates/workload-debezium.yaml create mode 100644 debezium-server/versions/1.1.0/values.yaml diff --git a/cdc-pipeline/versions/1.0.0/Chart.yaml b/cdc-pipeline/versions/1.0.0/Chart.yaml new file mode 100644 index 00000000..8c1625e7 --- /dev/null +++ b/cdc-pipeline/versions/1.0.0/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: cdc-pipeline +description: >- + Change Data Capture pipeline with PostgreSQL HA, Kafka, and Debezium Server. + Automatically coordinates database WAL settings, Kafka SASL credentials, + and cross-service DNS for a production-ready CDC streaming setup. +type: application +version: 1.0.0 +appVersion: "1.0" + +dependencies: + - name: postgres-highly-available + version: 2.2.0 + repository: "oci://ghcr.io/controlplane-com/templates" + - name: kafka + version: 3.4.0 + repository: "oci://ghcr.io/controlplane-com/templates" + - name: debezium-server + version: 1.1.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2026-04-13" + lastModified: "2026-04-13" + category: "event-streaming" + createsGvc: false diff --git a/cdc-pipeline/versions/1.0.0/README.md b/cdc-pipeline/versions/1.0.0/README.md new file mode 100644 index 00000000..44d7108b --- /dev/null +++ b/cdc-pipeline/versions/1.0.0/README.md @@ -0,0 +1,86 @@ +# CDC Pipeline + +A meta-template that deploys a complete Change Data Capture (CDC) pipeline on Control Plane, bundling: + +- **PostgreSQL HA** (Patroni + etcd + HAProxy) as the source database +- **Apache Kafka** (KRaft mode + Kafbat UI) as the event streaming platform +- **Debezium Server** as the CDC connector (PostgreSQL -> Kafka) + +## Why Use This Template? + +When deploying these three components individually, you must manually coordinate: + +- PostgreSQL WAL level (`logical` is required for CDC) +- Database credentials between PostgreSQL and Debezium +- Kafka SASL credentials between Kafka and Debezium +- Internal DNS hostnames for cross-service communication + +This meta-template handles all of that automatically. Shared values are defined once and validated at deploy time. + +## Quick Start + +1. Install the template and customize `values.yaml`: + - Set real passwords (replace all `changeme-*` values) + - Configure `source.tableIncludeList` to specify which tables to capture + - Adjust resource sizes and replica counts as needed + +2. Internal DNS names are computed automatically from the release name: + - PostgreSQL: `-postgres-ha-proxy..cpln.local:5432` + - Kafka: `-cluster..cpln.local:9092` + - Debezium: `-debezium..cpln.local` + +## Configuration + +### Shared Values + +These values must match between components. The default `values.yaml` pre-coordinates them: + +| Value | PostgreSQL Path | Debezium Path | +|-------|----------------|---------------| +| DB Username | `postgres-highly-available.postgres.username` | `debezium-server.source.database.user` | +| DB Password | `postgres-highly-available.postgres.password` | `debezium-server.source.database.password` | +| DB Name | `postgres-highly-available.postgres.database` | `debezium-server.source.database.name` | + +| Value | Kafka Path | Debezium Path | +|-------|-----------|---------------| +| SASL Username | `kafka.kafka.listeners.client.sasl.users` | `debezium-server.sink.kafka.saslUsername` | +| SASL Password | `kafka.kafka.listeners.client.sasl.passwords` | `debezium-server.sink.kafka.saslPassword` | + +### Cross-Component Validation + +The template validates at deploy time that: + +- `postgres-highly-available.postgres.walLevel` is `logical` +- Database credentials match between PostgreSQL and Debezium +- Debezium's Kafka SASL username exists in Kafka's configured users + +### Connecting to External Instances + +To use an external PostgreSQL or Kafka instead of the bundled one, set the hostname/bootstrap servers explicitly: + +```yaml +debezium-server: + source: + database: + hostname: "my-external-postgres.example.com" + sink: + kafka: + bootstrapServers: "my-external-kafka.example.com:9092" +``` + +### Debezium Heartbeat (Recommended for HA) + +The default configuration enables Debezium heartbeats (every 5 seconds) to prevent WAL accumulation during low-traffic periods. You must create the heartbeat table in PostgreSQL after deployment: + +```sql +CREATE TABLE IF NOT EXISTS debezium_heartbeat (id INT PRIMARY KEY, ts TIMESTAMPTZ); +INSERT INTO debezium_heartbeat VALUES (1, now()); +``` + +## Component Versions + +| Component | Version | +|-----------|---------| +| PostgreSQL HA | 2.2.0 (Patroni, PostgreSQL 17) | +| Kafka | 3.4.0 (Apache Kafka 3.9.1, KRaft) | +| Debezium Server | 1.1.0 (Debezium 3.0) | diff --git a/cdc-pipeline/versions/1.0.0/templates/_helpers.tpl b/cdc-pipeline/versions/1.0.0/templates/_helpers.tpl new file mode 100644 index 00000000..b2472041 --- /dev/null +++ b/cdc-pipeline/versions/1.0.0/templates/_helpers.tpl @@ -0,0 +1,80 @@ +{{/* +================================================================================ +CDC Pipeline - Cross-Component Validation +================================================================================ +*/}} + +{{/* +Validate that PostgreSQL WAL level is set to "logical" (required for CDC) +*/}} +{{- define "cdc.validateWalLevel" -}} +{{- $walLevel := index .Values "postgres-highly-available" "postgres" "walLevel" -}} +{{- if ne $walLevel "logical" -}} +{{- fail (printf "postgres-highly-available.postgres.walLevel must be 'logical' for CDC, got '%s'" $walLevel) -}} +{{- end -}} +{{- end -}} + +{{/* +Validate that database credentials match between PostgreSQL and Debezium +*/}} +{{- define "cdc.validateCredentials" -}} +{{- $pgUser := index .Values "postgres-highly-available" "postgres" "username" -}} +{{- $pgPass := index .Values "postgres-highly-available" "postgres" "password" -}} +{{- $pgDb := index .Values "postgres-highly-available" "postgres" "database" -}} +{{- $dbzUser := index .Values "debezium-server" "source" "database" "user" -}} +{{- $dbzPass := index .Values "debezium-server" "source" "database" "password" -}} +{{- $dbzDb := index .Values "debezium-server" "source" "database" "name" -}} +{{- if ne $pgUser $dbzUser -}} +{{- fail (printf "Credential mismatch: postgres-highly-available.postgres.username ('%s') != debezium-server.source.database.user ('%s')" $pgUser $dbzUser) -}} +{{- end -}} +{{- if ne $pgPass $dbzPass -}} +{{- fail "Credential mismatch: postgres-highly-available.postgres.password != debezium-server.source.database.password" -}} +{{- end -}} +{{- if ne $pgDb $dbzDb -}} +{{- fail (printf "Database mismatch: postgres-highly-available.postgres.database ('%s') != debezium-server.source.database.name ('%s')" $pgDb $dbzDb) -}} +{{- end -}} +{{- end -}} + +{{/* +Validate that Kafka SASL credentials match between Kafka and Debezium +*/}} +{{- define "cdc.validateKafkaCredentials" -}} +{{- $dbzSinkType := index .Values "debezium-server" "sink" "type" -}} +{{- if eq $dbzSinkType "kafka" -}} +{{- $kafkaUsers := index .Values "kafka" "kafka" "listeners" "client" "sasl" "users" -}} +{{- $dbzUser := index .Values "debezium-server" "sink" "kafka" "saslUsername" -}} +{{- if not (contains $dbzUser $kafkaUsers) -}} +{{- fail (printf "Kafka SASL mismatch: debezium saslUsername ('%s') not found in kafka listeners.client.sasl.users ('%s')" $dbzUser $kafkaUsers) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +================================================================================ +Labeling +================================================================================ +*/}} + +{{/* +Create chart name and version as used by the chart label +*/}} +{{- define "cdc.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Marketplace tags for the meta-template +*/}} +{{- define "cdc.tags" -}} +helm.sh/chart: {{ include "cdc.chart" . }} +app.cpln.io/name: {{ .Release.Name }} +app.cpln.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.cpln.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.cpln.io/managed-by: {{ .Release.Service }} +cpln/marketplace: "true" +cpln/marketplace-template: cdc-pipeline +cpln/marketplace-template-version: {{ .Chart.Version }} +cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} +{{- end }} diff --git a/cdc-pipeline/versions/1.0.0/templates/validation.yaml b/cdc-pipeline/versions/1.0.0/templates/validation.yaml new file mode 100644 index 00000000..ec2e6138 --- /dev/null +++ b/cdc-pipeline/versions/1.0.0/templates/validation.yaml @@ -0,0 +1,3 @@ +{{- include "cdc.validateWalLevel" . -}} +{{- include "cdc.validateCredentials" . -}} +{{- include "cdc.validateKafkaCredentials" . -}} diff --git a/cdc-pipeline/versions/1.0.0/values.yaml b/cdc-pipeline/versions/1.0.0/values.yaml new file mode 100644 index 00000000..339ea5e3 --- /dev/null +++ b/cdc-pipeline/versions/1.0.0/values.yaml @@ -0,0 +1,379 @@ +# ============================================================================= +# CDC Pipeline Meta-Template +# ============================================================================= +# Deploys a complete Change Data Capture pipeline: +# - PostgreSQL HA (Patroni + etcd + HAProxy) +# - Kafka (KRaft cluster + Kafbat UI) +# - Debezium Server (CDC connector: postgres -> kafka) +# +# Shared credentials are defined once here and wired to each component. +# Internal DNS names are computed automatically from the release name: +# PostgreSQL: -postgres-ha-proxy..cpln.local:5432 +# Kafka: -cluster..cpln.local:9092 +# Debezium: -debezium..cpln.local +# ============================================================================= + +# ============================================================================= +# PostgreSQL HA +# ============================================================================= +postgres-highly-available: + replicas: 3 + + resources: + minCpu: 500m + minMemory: 1Gi + maxCpu: 1 + maxMemory: 2Gi + + image: controlplanecorporation/patroni-postgres:0.7 + + postgres: + username: cdc_user + password: "changeme-postgres-password" + database: cdcdb + walLevel: logical # Required for CDC -- do not change + + multiZone: false + + volumeset: + capacity: 10 + autoscaling: + enabled: false + maxCapacity: 100 + minFreePercentage: 10 + scalingFactor: 1.2 + + internal_access: + type: same-gvc + + etcd: + replicas: 3 + resources: + cpu: 500m + memory: 512Mi + multiZone: false + volumeset: + capacity: 10 + internal_access: + type: same-gvc + + pgbouncer: + enabled: false + + proxy: + enabled: true + image: haproxy:2.9 + resources: + cpu: 100m + memory: 128Mi + minReplicas: 2 + maxReplicas: 2 + + backup: + enabled: false + +# ============================================================================= +# Kafka +# ============================================================================= +kafka: + kafka: + name: cluster + image: apache/kafka:3.9.1 + suspend: false + deletionProtection: false + replicas: 3 + minReadySeconds: 0 + debug: false + multiZone: false + logDirs: /opt/kafka/logs-0,/opt/kafka/logs-1 + env: [] + volumes: + logs: + initialCapacity: 10 + performanceClass: general-purpose-ssd + fileSystemType: ext4 + snapshots: + createFinalSnapshot: true + retentionDuration: 7d + schedule: 0 0 * * * + autoscaling: + maxCapacity: 1000 + minFreePercentage: 20 + scalingFactor: 1.2 + cpu: 1000m + memory: 2000Mi + minCpu: 250m + minMemory: 2000Mi + firewall: + internal_inboundAllowType: "same-gvc" + listeners: + client: + protocol: SASL_PLAINTEXT + name: CLIENT + containerPort: 9092 + sasl: + admin: + username: admin + password: "changeme-kafka-admin-password" + users: "debezium" + passwords: "changeme-kafka-debezium-password" + acl: + superUsers: "User:admin" + allowEveryoneIfNoAclFound: false + secrets: + kraft_cluster_id: "changeme-kraft-cluster-id" + inter_broker_password: "changeme-inter-broker-password" + controller_password: "changeme-controller-password" + extra_configurations: + default.replication.factor: 3 + auto.create.topics.enable: true + log.retention.hours: 168 + + kafka_exporter: + name: exporter + image: danielqsj/kafka-exporter:v1.9.0 + debug: false + cpu: 50m + memory: 128Mi + listener: client + env: [] + dropMetrics: [] + + jmx_exporter: + name: jmx-exporter + image: ghcr.io/controlplane-com/bitnami/jmx-exporter + kafkaJmxPort: 5557 + exporterPort: 5556 + debug: false + cpu: 250m + memory: 256Mi + minCpu: 80m + minMemory: 125Mi + listener: client + dropMetrics: [] + config: + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5557/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false + whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* + - kafka.producer:* + - kafka.consumer:* + rules: + - labels: + request: "$3" + name: kafka_request_count + pattern: kafka.network<>(Count) + - labels: + request: "$3" + stat: "$4" + name: kafka_request_metrics_totaltimems + pattern: kafka.network<>(.+) + - labels: + request: "$3" + component: "$2" + stat: "$4" + name: kafka_request_latency_ms + pattern: kafka.network<>(.+) + - labels: + client_type: "$3" + metric: "$2" + stat: "$4" + name: kafka_client_metrics + pattern: kafka.network<>(.+) + - labels: + client_id: "$1" + metric: "$2" + name: kafka_consumer_metrics + pattern: kafka.consumer<>(.+) + - labels: + client_id: "$1" + metric: "$2" + name: kafka_producer_metrics + pattern: kafka.producer<>(.+) + - name: kafka_server_$1_$2_$3 + pattern: kafka.server<>(Count|Value) + - name: java_lang_$1_$2 + pattern: java.lang<>(.+) + + kafbat_ui: + enabled: true + deletionProtection: false + name: kafbat-ui + image: ghcr.io/kafbat/kafka-ui + cpu: 300m + memory: 1000Mi + minCpu: 100m + minMemory: 400Mi + replicas: 1 + timeoutSeconds: 30 + configuration_secret: kafka-kafbat-ui-config + firewall: + external_inboundAllowCIDR: "0.0.0.0/0" + external_outboundAllowCIDR: "0.0.0.0/0" + + kafka_rest_proxy: + enabled: false + + kafka_client: + name: client + image: apache/kafka:3.9.1 + cpu: 500m + memory: 1000Mi + firewall: + external_outboundAllowCIDR: "0.0.0.0/0" + + kafka_ui: + enabled: false + +# ============================================================================= +# Debezium Server +# ============================================================================= +# Database hostname and Kafka bootstrap servers are auto-computed from +# the release name when left empty. Override only if connecting to +# external instances not managed by this meta-template. +debezium-server: + image: quay.io/debezium/server:3.0 + + resources: + cpu: 500m + memory: 512Mi + + source: + type: postgres + + database: + hostname: "" # Auto-computed: -postgres-ha-proxy..cpln.local + port: 5432 + name: cdcdb # Must match postgres-highly-available.postgres.database + user: cdc_user # Must match postgres-highly-available.postgres.username + password: "changeme-postgres-password" # Must match postgres-highly-available.postgres.password + + serverName: "dbserver1" + tableIncludeList: "" + tableExcludeList: "" + + postgres: + slotName: "debezium" + publicationName: "dbz_publication" + pluginName: "pgoutput" + slotDropOnStop: false + heartbeatIntervalMs: 5000 + heartbeatActionQuery: "UPDATE debezium_heartbeat SET ts = now() WHERE id = 1" + + offset: + storage: file + flushIntervalMs: 10000 + flushTimeoutMs: 60000 + file: + filename: "/debezium/data/offsets.dat" + redis: + address: "" + key: "debezium:offsets" + password: "" + ssl: false + jdbc: + url: "" + user: "" + password: "" + tableName: "debezium_offsets" + + schemaHistory: + storage: file + file: + filename: "/debezium/data/schema-history.dat" + redis: + address: "" + key: "debezium:schema-history" + password: "" + ssl: false + jdbc: + url: "" + user: "" + password: "" + tableName: "debezium_schema_history" + + errors: + retryDelayInitialMs: 300 + retryDelayMaxMs: 10000 + maxRetries: -1 + + sink: + type: kafka + kafka: + bootstrapServers: "" # Auto-computed: -cluster..cpln.local:9092 + topic: "" + securityProtocol: "SASL_PLAINTEXT" + saslMechanism: "PLAIN" + saslUsername: "debezium" # Must match kafka.kafka.listeners.client.sasl.users + saslPassword: "changeme-kafka-debezium-password" # Must match kafka.kafka.listeners.client.sasl.passwords + + redis: + address: "" + password: "" + ssl: false + streamName: "" + + nats: + url: "" + subject: "" + username: "" + password: "" + + http: + url: "" + headers: {} + authType: "" + username: "" + password: "" + bearerToken: "" + + kinesis: + region: "" + streamName: "" + credentialsProvider: "default" + cloudAccount: + enabled: false + name: "" + + pubsub: + projectId: "" + topic: "" + cloudAccount: + enabled: false + name: "" + + pulsar: + serviceUrl: "" + topic: "" + authPluginClassName: "" + authToken: "" + + eventhubs: + connectionString: "" + hubName: "" + + format: + key: json + value: json + schemaRegistry: + url: "" + username: "" + password: "" + + volumeset: + capacity: 10 + performanceClass: general-purpose-ssd + + firewall: + internal: + inboundAllowType: same-gvc + workloads: [] + external: + outboundAllowCIDR: + - 0.0.0.0/0 diff --git a/debezium-server/deploy-aws-us-east-2.yaml b/debezium-server/deploy-aws-us-east-2.yaml new file mode 100644 index 00000000..cc7b8eef --- /dev/null +++ b/debezium-server/deploy-aws-us-east-2.yaml @@ -0,0 +1,37 @@ +source: + type: postgres + database: + hostname: data-postgres-ha-proxy.aws-us-east-2.cpln.local + port: 5432 + name: test + user: username + password: password + serverName: dbserver1 + postgres: + slotName: debezium + publicationName: dbz_publication + pluginName: pgoutput + slotDropOnStop: false + heartbeatIntervalMs: 5000 + heartbeatActionQuery: "UPDATE debezium_heartbeat SET ts = now() WHERE id = 1" + offset: + storage: file + flushIntervalMs: 5000 + flushTimeoutMs: 10000 + errors: + retryDelayInitialMs: 300 + retryDelayMaxMs: 10000 + maxRetries: 10 + +sink: + type: kafka + kafka: + bootstrapServers: "replica-0.etl-cluster.aws-us-east-2.aws-us-east-2.cpln.local:9092,replica-1.etl-cluster.aws-us-east-2.aws-us-east-2.cpln.local:9092,replica-2.etl-cluster.aws-us-east-2.aws-us-east-2.cpln.local:9092" + securityProtocol: SASL_PLAINTEXT + saslMechanism: PLAIN + saslUsername: admin + saslPassword: your-admin-password + +format: + key: json + value: json diff --git a/debezium-server/icon.png b/debezium-server/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..f89204d10abd7066d4b20fd44858daee51af33d9 GIT binary patch literal 2367 zcmV-F3BdM=P)op68k0oE10Te89*yTmTjTYjoRur`HQ--+agn0)qeu zFnfQ;j|Bh%vv3(?UZ_}Bueg_2jVgn_V?PEUHsZd0EPTu$7a(({?~_<;LiSMsFblxB z`O|)_*9$WO0D~$FdNAc4cc$DETJN}-005v0Lp(UYa?LT8XEa#|!6*T+dx6LZp!+oy zX0j)pQ(AJ|%mBP&5SXdL5cb%zsi}Yf00jW{jh}u~Ony2SfZ#O+2DKW_*;s4|0Kxw) zFvwB`0Sr+9$V_}LKY-wMB`_I@aPtH3k45F|(rsNDLf40>Z9XS#8UWxO>uGT%00c1% zDF~4dQ@H+SsQT`lkQo3NRJyoYFQ+HQ+yDTG%ITMPi=+TRSI}NLAs2w)!Y>?i zsdCR@P?bSz5&*!Uk^0JXXf~Mrbqk z4^@hW_JzX`8kig=ZRfknJzPg#h}52DzLyH{qvn?+W&J1!I*MSOfXX;r3y3|V&c>7Gr3OKSe~&M zVkDsotOLSiChN|wleuAt*&h#7GbBOh&%gY8s$rgVP8kfE?NJ3>2DKREFhnPk1f6R; zE~myGTQ)5Qjr6Dj3xKd0wT+2K}mJ0>ZKeYN6PaqYRc9)L>A1$~_TpXPIUqh zgC&M&Fes+Q{We_BTLu|a0>mscIY$?tXfX&mt#E;vzMFJTh5X{=_M06BO-O3-QdElr zfR_Y?nFBzmy=Msf$l}v(({bJG)MH{3MtcK#;Uc459bP37L&hEn)rIDz$hd zpWBiEf@32kC*)FMg^R28Y6L=CkrghSetB1m0hR{qkyIN5s11Mndz)8bm%Ep!!9G;Xog0ALm% zM68U26fLIRfl^?FwMbk?6##%h%zn9%x5DLw}_RQ+#uxJMr_XZF`?y}aJ3 zmw&A$0H`vA2GDSiKA`z<_fSvM{JUrWs%{bhm?f0^_^T^LUyB>wwSTM%%rQv6*+1IJ z8vBng{!wHI!=I6bgeuSklksTJfesW5Y1>?1nJzMf=}FL78FU$Be7^IHohP>b;&Bn4 zpczgmgDykpU+&rO{NE^dV|hkrhzBWEpvg?;on0q$o$%4cCk8{zrc{9@KoeiRzc;^f zzw~;nnxzUv`+$fR*BSJnM-^D}jZ@oeeM3pGbARAaDx@QePYp25_NW4`=NUvB=xVFI z|J`Zrt)#{l7hj*}995v{YjGz*i+lG6@|D4LzZ0B@YHBiWyIT6v~9Y9P-YViU?JU+7cGz8+yJ^MS%Waa?SeOzFW{h|Gzh;o;+!i{rF zw?bXw?VW$Jf*UZBR@e%(c-Qx4N&wiH6@ai0+H1v-ZI+axNz0<8K#La`BD914tM~VI z*y};^0`QMT0f5m3LD~uro>}^fwnbVzN#{F~0MNBpt2;4DOVPM;S7U_>A3gg%VwQxh z0001#4%%x)+CnYfR%nHb@x9H`0D_^>R%!9Ozy3p8r4^0|Tr~g?%LyvAxN0jLQ~8Pj zgmQvbyb_C6Yo1H(&F9Y@!svWDgeQ9XsOyQo}}DWY4L>Et_lEt z$4Hi{-QuYYlL9TCu+&Ty004ky`aZE4G^b*>cyw3;zqOcYK^8#LQu})F7nZ*pP71)r zPXd`qX9!nTi>LfHvc~?&%4qTQ*uhn0J^&jK%>H@@o^qd)(c($t;qIZ1uf-Q+w0P2Z zGWfH$cb`(Yc>#oe9D1YQ=-lidtubhycMmbU4@eu2b|36K+H;`F5XXY0m-TWuZG2OG lul-H6tN}O%U@i}Y{{p_koVI=bQ&9i_002ovPDHLkV1jDIN=5(x literal 0 HcmV?d00001 diff --git a/debezium-server/versions/1.0.0/Chart.yaml b/debezium-server/versions/1.0.0/Chart.yaml new file mode 100644 index 00000000..de9327f6 --- /dev/null +++ b/debezium-server/versions/1.0.0/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: debezium-server +description: Debezium Server CDC app for Control Plane (standalone mode) +type: application +version: 1.0.0 +appVersion: "3.0" + +annotations: + created: "2026-04-03" + lastModified: "2026-04-03" + category: "event-streaming" + createsGvc: false diff --git a/debezium-server/versions/1.0.0/README.md b/debezium-server/versions/1.0.0/README.md new file mode 100644 index 00000000..02c3ac45 --- /dev/null +++ b/debezium-server/versions/1.0.0/README.md @@ -0,0 +1,347 @@ +# Debezium Server Template + +Debezium Server is a standalone Change Data Capture (CDC) application that streams database changes to various messaging systems. Unlike Debezium connectors that run on Kafka Connect, Debezium Server runs as a standalone application and can send events directly to Kafka, Redis, NATS, HTTP endpoints, cloud services, and more. + +## Overview + +This template deploys Debezium Server on Control Plane with: + +- Configurable source database connectors (PostgreSQL, MySQL, MongoDB, SQL Server, Oracle) +- Multiple sink options (Kafka, Redis, NATS JetStream, HTTP, AWS Kinesis, GCP Pub/Sub, Pulsar, Event Hubs) +- Flexible offset storage (file, Redis, JDBC) +- Universal Cloud Identity integration for AWS and GCP sinks +- Automatic secret management for credentials + +## Quick Start + +### PostgreSQL to Kafka + +```yaml +source: + type: postgres + database: + hostname: postgres.mygvc.cpln.local + port: 5432 + name: mydb + user: debezium + password: secret123 + serverName: myserver + tableIncludeList: "public.users,public.orders" + postgres: + slotName: debezium_slot + publicationName: dbz_publication + +sink: + type: kafka + kafka: + bootstrapServers: kafka.mygvc.cpln.local:9092 + topic: cdc-events + +format: + key: json + value: json +``` + +### MySQL to Redis Streams + +```yaml +source: + type: mysql + database: + hostname: mysql.mygvc.cpln.local + port: 3306 + name: mydb + user: debezium + password: secret123 + serverName: myserver + mysql: + serverId: 85744 + includeSchemaChanges: true + +sink: + type: redis + redis: + address: redis.mygvc.cpln.local:6379 + streamName: cdc-stream +``` + +### PostgreSQL to AWS Kinesis (Universal Cloud Identity) + +```yaml +source: + type: postgres + database: + hostname: my-rds-instance.us-east-1.rds.amazonaws.com + port: 5432 + name: mydb + user: debezium + password: secret123 + serverName: myserver + +sink: + type: kinesis + kinesis: + region: us-east-1 + streamName: cdc-events + credentialsProvider: default + cloudAccount: + enabled: true + name: my-aws-account +``` + +## Supported Sources + +| Database | Connector | Default Port | Key Configuration | +|----------|-----------|--------------|-------------------| +| PostgreSQL | PostgresConnector | 5432 | `slotName`, `publicationName`, `pluginName` | +| MySQL | MySqlConnector | 3306 | `serverId`, `includeSchemaChanges` | +| MongoDB | MongoDbConnector | 27017 | `connectionString`, `replicaSet` | +| SQL Server | SqlServerConnector | 1433 | `databaseNames`, `snapshotMode` | +| Oracle | OracleConnector | 1521 | `pdbName`, `logMiningStrategy` | + +### PostgreSQL Prerequisites + +1. Enable logical replication in `postgresql.conf`: + ``` + wal_level = logical + max_replication_slots = 4 + max_wal_senders = 4 + ``` + +2. Create a publication and replication slot: + ```sql + CREATE PUBLICATION dbz_publication FOR ALL TABLES; + -- Slot is created automatically by Debezium + ``` + +3. Grant permissions: + ```sql + GRANT USAGE ON SCHEMA public TO debezium; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO debezium; + ALTER USER debezium REPLICATION; + ``` + +### MySQL Prerequisites + +1. Enable binary logging in `my.cnf`: + ``` + server-id = 1 + log_bin = mysql-bin + binlog_format = ROW + binlog_row_image = FULL + ``` + +2. Grant permissions: + ```sql + GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'debezium'@'%'; + ``` + +## Supported Sinks + +| Sink | Required Configuration | Notes | +|------|------------------------|-------| +| Kafka | `bootstrapServers` | Simple Kafka producer (no Kafka Connect required) | +| Redis | `address` | Redis Streams for real-time event streaming | +| NATS JetStream | `url` | Cloud-native messaging with persistence | +| HTTP | `url` | Webhooks and custom HTTP endpoints | +| Kinesis | `region`, `streamName` | AWS Kinesis (uses Universal Cloud Identity) | +| Pub/Sub | `projectId` | GCP Pub/Sub (uses Universal Cloud Identity) | +| Pulsar | `serviceUrl` | Apache Pulsar with optional authentication | +| Event Hubs | `connectionString`, `hubName` | Azure Event Hubs | + +## Offset Storage + +Debezium tracks the position of captured changes using offset storage. Three options are available: + +### File Storage (Default) + +Stores offsets in a local file. Requires a volumeset for persistence. + +```yaml +source: + offset: + storage: file + file: + filename: /debezium/data/offsets.dat + +volumeset: + capacity: 10 + performanceClass: general-purpose-ssd +``` + +### Redis Storage + +Stores offsets in Redis. No volumeset required. + +```yaml +source: + offset: + storage: redis + redis: + address: redis.mygvc.cpln.local:6379 + key: debezium:offsets + password: "" + ssl: false +``` + +### JDBC Storage + +Stores offsets in a relational database. No volumeset required. + +```yaml +source: + offset: + storage: jdbc + jdbc: + url: jdbc:postgresql://postgres.mygvc.cpln.local:5432/offsets + user: debezium + password: secret123 + tableName: debezium_offsets +``` + +## Schema History (MySQL/SQL Server Only) + +MySQL and SQL Server connectors require schema history storage to track DDL changes: + +```yaml +source: + type: mysql + schemaHistory: + storage: file # or: redis, jdbc + file: + filename: /debezium/data/schema-history.dat +``` + +## Serialization Formats + +Supports JSON, Avro, and Protobuf serialization: + +```yaml +format: + key: json + value: json + + # For Avro/Protobuf, configure schema registry: + schemaRegistry: + url: http://schema-registry.mygvc.cpln.local:8081 + username: "" + password: "" +``` + +## Universal Cloud Identity + +For AWS Kinesis and GCP Pub/Sub sinks, this template integrates with Control Plane's Universal Cloud Identity for credential-less authentication. + +### AWS Kinesis + +1. Create an AWS cloud account in Control Plane +2. Configure the identity with appropriate IAM policies +3. Enable the cloud account in your values: + +```yaml +sink: + type: kinesis + kinesis: + region: us-east-1 + streamName: my-stream + credentialsProvider: default + cloudAccount: + enabled: true + name: my-aws-account +``` + +### GCP Pub/Sub + +```yaml +sink: + type: pubsub + pubsub: + projectId: my-gcp-project + cloudAccount: + enabled: true + name: my-gcp-account +``` + +## Resource Configuration + +```yaml +resources: + cpu: 500m # CPU allocation + memory: 512Mi # Memory allocation + +volumeset: + capacity: 10 # GiB (only used with file storage) + performanceClass: general-purpose-ssd +``` + +## Firewall Configuration + +```yaml +firewall: + internal: + inboundAllowType: same-gvc # none, same-gvc, same-org, workload-list + workloads: [] # For workload-list type + external: + outboundAllowCIDR: + - 0.0.0.0/0 # Required for external database connectivity +``` + +## Health Checks + +Debezium Server exposes Quarkus health endpoints: + +- **Readiness**: `/q/health/ready` - Checks if the connector is ready +- **Liveness**: `/q/health/live` - Checks if the server is alive + +## Installation + +```bash +cpln helm install debezium ./debezium-server/versions/1.0.0 \ + --gvc my-gvc \ + -f my-values.yaml +``` + +## Verification + +1. Check workload status: + ```bash + cpln workload get debezium--debezium --gvc my-gvc + ``` + +2. Check health endpoint: + ```bash + curl http://debezium--debezium.my-gvc.cpln.local:8080/q/health + ``` + +3. View logs: + ```bash + cpln workload logs debezium--debezium --gvc my-gvc + ``` + +4. Test CDC by making changes in the source database and verifying events appear in the configured sink. + +## Troubleshooting + +### Connector Not Starting + +- Check database connectivity and credentials +- Verify replication permissions are granted +- Review logs for specific error messages + +### Offset Storage Issues + +- For file storage: ensure volumeset is properly mounted +- For Redis/JDBC: verify connectivity and credentials +- Check that the storage backend is accessible from the GVC + +### Sink Delivery Failures + +- Verify sink connectivity and authentication +- For cloud sinks (Kinesis/Pub/Sub): ensure cloud account is properly configured +- Check firewall rules allow outbound traffic to the sink + +## Resources + +- [Debezium Documentation](https://debezium.io/documentation/) +- [Debezium Server Documentation](https://debezium.io/documentation/reference/stable/operations/debezium-server.html) +- [Control Plane Documentation](https://docs.controlplane.com/) diff --git a/debezium-server/versions/1.0.0/templates/_helpers.tpl b/debezium-server/versions/1.0.0/templates/_helpers.tpl new file mode 100644 index 00000000..d99d2eb9 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/_helpers.tpl @@ -0,0 +1,264 @@ +{{/* +================================================================================ +Resource Naming +================================================================================ +*/}} + +{{/* +Debezium Server Workload Name +*/}} +{{- define "debezium.name" -}} +{{- printf "%s-debezium" .Release.Name }} +{{- end }} + +{{/* +Debezium Identity Name +*/}} +{{- define "debezium.identity.name" -}} +{{- printf "%s-debezium-identity" .Release.Name }} +{{- end }} + +{{/* +Debezium Policy Name +*/}} +{{- define "debezium.policy.name" -}} +{{- printf "%s-debezium-policy" .Release.Name }} +{{- end }} + +{{/* +Debezium Config Secret Name (opaque - application.properties) +*/}} +{{- define "debezium.config.name" -}} +{{- printf "%s-debezium-config" .Release.Name }} +{{- end }} + +{{/* +Debezium Credentials Secret Name (dictionary) +*/}} +{{- define "debezium.credentials.name" -}} +{{- printf "%s-debezium-credentials" .Release.Name }} +{{- end }} + +{{/* +Debezium Volumeset Name +*/}} +{{- define "debezium.volumeset.name" -}} +{{- printf "%s-debezium-data" .Release.Name }} +{{- end }} + +{{/* +Debezium Entrypoint Secret Name +*/}} +{{- define "debezium.entrypoint.name" -}} +{{- printf "%s-debezium-entrypoint" .Release.Name }} +{{- end }} + +{{/* +================================================================================ +Validation Helpers +================================================================================ +*/}} + +{{/* +Validate source configuration +*/}} +{{- define "debezium.validateSource" -}} +{{- $validTypes := list "postgres" "mysql" "mongodb" "sqlserver" "oracle" -}} +{{- if not (has .Values.source.type $validTypes) -}} +{{- fail (printf "Invalid source.type '%s'. Must be one of: %s" .Values.source.type (join ", " $validTypes)) -}} +{{- end -}} +{{- if not .Values.source.database.hostname -}} +{{- fail "source.database.hostname is required" -}} +{{- end -}} +{{- if not .Values.source.database.name -}} +{{- fail "source.database.name is required" -}} +{{- end -}} +{{- if not .Values.source.database.user -}} +{{- fail "source.database.user is required" -}} +{{- end -}} +{{- if not .Values.source.database.password -}} +{{- fail "source.database.password is required" -}} +{{- end -}} +{{- end -}} + +{{/* +Validate sink configuration +*/}} +{{- define "debezium.validateSink" -}} +{{- $validTypes := list "kafka" "redis" "nats-jetstream" "http" "kinesis" "pubsub" "pulsar" "eventhubs" -}} +{{- if not (has .Values.sink.type $validTypes) -}} +{{- fail (printf "Invalid sink.type '%s'. Must be one of: %s" .Values.sink.type (join ", " $validTypes)) -}} +{{- end -}} +{{- if eq .Values.sink.type "kafka" -}} + {{- if not .Values.sink.kafka.bootstrapServers -}} + {{- fail "sink.kafka.bootstrapServers is required when sink.type is 'kafka'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "redis" -}} + {{- if not .Values.sink.redis.address -}} + {{- fail "sink.redis.address is required when sink.type is 'redis'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "nats-jetstream" -}} + {{- if not .Values.sink.nats.url -}} + {{- fail "sink.nats.url is required when sink.type is 'nats-jetstream'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "http" -}} + {{- if not .Values.sink.http.url -}} + {{- fail "sink.http.url is required when sink.type is 'http'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "kinesis" -}} + {{- if not .Values.sink.kinesis.region -}} + {{- fail "sink.kinesis.region is required when sink.type is 'kinesis'" -}} + {{- end -}} + {{- if not .Values.sink.kinesis.streamName -}} + {{- fail "sink.kinesis.streamName is required when sink.type is 'kinesis'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "pubsub" -}} + {{- if not .Values.sink.pubsub.projectId -}} + {{- fail "sink.pubsub.projectId is required when sink.type is 'pubsub'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "pulsar" -}} + {{- if not .Values.sink.pulsar.serviceUrl -}} + {{- fail "sink.pulsar.serviceUrl is required when sink.type is 'pulsar'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "eventhubs" -}} + {{- if not .Values.sink.eventhubs.connectionString -}} + {{- fail "sink.eventhubs.connectionString is required when sink.type is 'eventhubs'" -}} + {{- end -}} + {{- if not .Values.sink.eventhubs.hubName -}} + {{- fail "sink.eventhubs.hubName is required when sink.type is 'eventhubs'" -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Validate offset storage configuration +*/}} +{{- define "debezium.validateOffsetStorage" -}} +{{- $validTypes := list "file" "redis" "jdbc" -}} +{{- if not (has .Values.source.offset.storage $validTypes) -}} +{{- fail (printf "Invalid source.offset.storage '%s'. Must be one of: %s" .Values.source.offset.storage (join ", " $validTypes)) -}} +{{- end -}} +{{- if eq .Values.source.offset.storage "redis" -}} + {{- if not .Values.source.offset.redis.address -}} + {{- fail "source.offset.redis.address is required when offset storage is 'redis'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.source.offset.storage "jdbc" -}} + {{- if not .Values.source.offset.jdbc.url -}} + {{- fail "source.offset.jdbc.url is required when offset storage is 'jdbc'" -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +================================================================================ +Connector Class Mapping +================================================================================ +*/}} + +{{/* +Get the Debezium connector class for the source type +*/}} +{{- define "debezium.connectorClass" -}} +{{- $connectorMap := dict + "postgres" "io.debezium.connector.postgresql.PostgresConnector" + "mysql" "io.debezium.connector.mysql.MySqlConnector" + "mongodb" "io.debezium.connector.mongodb.MongoDbConnector" + "sqlserver" "io.debezium.connector.sqlserver.SqlServerConnector" + "oracle" "io.debezium.connector.oracle.OracleConnector" +-}} +{{- get $connectorMap .Values.source.type -}} +{{- end -}} + +{{/* +Get the default port for the source type +*/}} +{{- define "debezium.defaultPort" -}} +{{- $portMap := dict + "postgres" 5432 + "mysql" 3306 + "mongodb" 27017 + "sqlserver" 1433 + "oracle" 1521 +-}} +{{- get $portMap .Values.source.type -}} +{{- end -}} + +{{/* +Get the effective database port +*/}} +{{- define "debezium.databasePort" -}} +{{- if .Values.source.database.port -}} +{{- .Values.source.database.port -}} +{{- else -}} +{{- include "debezium.defaultPort" . -}} +{{- end -}} +{{- end -}} + +{{/* +Check if schema history is required (MySQL and SQL Server need it) +*/}} +{{- define "debezium.requiresSchemaHistory" -}} +{{- if or (eq .Values.source.type "mysql") (eq .Values.source.type "sqlserver") -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} + +{{/* +Check if file-based storage is used (requires volumeset) +*/}} +{{- define "debezium.requiresVolumeset" -}} +{{- if eq .Values.source.offset.storage "file" -}} +true +{{- else if and (eq (include "debezium.requiresSchemaHistory" .) "true") (eq .Values.source.schemaHistory.storage "file") -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} + +{{/* +================================================================================ +Labeling +================================================================================ +*/}} + +{{/* +Create chart name and version as used by the chart label +*/}} +{{- define "debezium.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels/tags +*/}} +{{- define "debezium.tags" -}} +helm.sh/chart: {{ include "debezium.chart" . }} +{{ include "debezium.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.cpln.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.cpln.io/managed-by: {{ .Release.Service }} +cpln/marketplace: "true" +cpln/marketplace-template: debezium-server +cpln/marketplace-template-version: {{ .Chart.Version }} +cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "debezium.selectorLabels" -}} +app.cpln.io/name: {{ .Release.Name }} +app.cpln.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/debezium-server/versions/1.0.0/templates/identity.yaml b/debezium-server/versions/1.0.0/templates/identity.yaml new file mode 100644 index 00000000..ac53fdd2 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/identity.yaml @@ -0,0 +1,19 @@ +{{- include "debezium.validateSource" . -}} +{{- include "debezium.validateSink" . -}} +{{- include "debezium.validateOffsetStorage" . -}} +kind: identity +name: {{ include "debezium.identity.name" . }} +description: Debezium Server identity for secret access and cloud integration +gvc: {{ .Values.global.cpln.gvc }} +tags: + {{- include "debezium.tags" . | nindent 2 }} +{{- if and (eq .Values.sink.type "kinesis") .Values.sink.kinesis.cloudAccount.enabled }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.sink.kinesis.cloudAccount.name }} +{{- end }} +{{- if and (eq .Values.sink.type "pubsub") .Values.sink.pubsub.cloudAccount.enabled }} +gcp: + cloudAccountLink: //cloudaccount/{{ .Values.sink.pubsub.cloudAccount.name }} + scopes: + - https://www.googleapis.com/auth/pubsub +{{- end }} diff --git a/debezium-server/versions/1.0.0/templates/policy.yaml b/debezium-server/versions/1.0.0/templates/policy.yaml new file mode 100644 index 00000000..b9e40ef1 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/policy.yaml @@ -0,0 +1,17 @@ +kind: policy +name: {{ include "debezium.policy.name" . }} +description: Debezium Server policy for secret access +tags: + {{- include "debezium.tags" . | nindent 2 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "debezium.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "debezium.config.name" . }} + - //secret/{{ include "debezium.credentials.name" . }} + {{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} + - //secret/{{ include "debezium.entrypoint.name" . }} + {{- end }} diff --git a/debezium-server/versions/1.0.0/templates/secret-config.yaml b/debezium-server/versions/1.0.0/templates/secret-config.yaml new file mode 100644 index 00000000..b32381b5 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/secret-config.yaml @@ -0,0 +1,260 @@ +kind: secret +name: {{ include "debezium.config.name" . }} +description: Debezium Server application.properties configuration +tags: + {{- include "debezium.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: |- + # ============================================================================= + # Debezium Server Configuration + # Generated by Control Plane Debezium Server Template + # ============================================================================= + + # ----------------------------------------------------------------------------- + # Quarkus Settings + # ----------------------------------------------------------------------------- + quarkus.http.port=8080 + quarkus.log.console.json=false + + # ----------------------------------------------------------------------------- + # Source Connector Configuration + # ----------------------------------------------------------------------------- + debezium.source.connector.class={{ include "debezium.connectorClass" . }} + debezium.source.topic.prefix={{ .Values.source.serverName }} + + # Database connection + debezium.source.database.hostname=${DB_HOSTNAME} + debezium.source.database.port={{ include "debezium.databasePort" . }} + debezium.source.database.user=${DB_USER} + debezium.source.database.password=${DB_PASSWORD} + {{- if or (eq .Values.source.type "oracle") (eq .Values.source.type "postgres") }} + debezium.source.database.dbname={{ .Values.source.database.name }} + {{- else }} + debezium.source.database.name={{ .Values.source.database.name }} + {{- end }} + + {{- if .Values.source.tableIncludeList }} + debezium.source.table.include.list={{ .Values.source.tableIncludeList }} + {{- end }} + {{- if .Values.source.tableExcludeList }} + debezium.source.table.exclude.list={{ .Values.source.tableExcludeList }} + {{- end }} + + {{- /* PostgreSQL-specific settings */}} + {{- if eq .Values.source.type "postgres" }} + debezium.source.plugin.name={{ .Values.source.postgres.pluginName }} + debezium.source.slot.name={{ .Values.source.postgres.slotName }} + debezium.source.publication.name={{ .Values.source.postgres.publicationName }} + debezium.source.slot.drop.on.stop={{ .Values.source.postgres.slotDropOnStop }} + {{- if gt (int .Values.source.postgres.heartbeatIntervalMs) 0 }} + debezium.source.heartbeat.interval.ms={{ .Values.source.postgres.heartbeatIntervalMs }} + {{- if .Values.source.postgres.heartbeatActionQuery }} + debezium.source.heartbeat.action.query={{ .Values.source.postgres.heartbeatActionQuery }} + {{- end }} + {{- end }} + {{- end }} + + {{- /* MySQL-specific settings */}} + {{- if eq .Values.source.type "mysql" }} + debezium.source.database.server.id={{ .Values.source.mysql.serverId }} + debezium.source.include.schema.changes={{ .Values.source.mysql.includeSchemaChanges }} + {{- end }} + + {{- /* MongoDB-specific settings */}} + {{- if eq .Values.source.type "mongodb" }} + {{- if .Values.source.mongodb.connectionString }} + debezium.source.mongodb.connection.string=${MONGODB_CONNECTION_STRING} + {{- end }} + {{- if .Values.source.mongodb.replicaSet }} + debezium.source.mongodb.replica.set={{ .Values.source.mongodb.replicaSet }} + {{- end }} + {{- end }} + + {{- /* SQL Server-specific settings */}} + {{- if eq .Values.source.type "sqlserver" }} + {{- if .Values.source.sqlserver.databaseNames }} + debezium.source.database.names={{ .Values.source.sqlserver.databaseNames }} + {{- end }} + debezium.source.snapshot.mode={{ .Values.source.sqlserver.snapshotMode }} + {{- end }} + + {{- /* Oracle-specific settings */}} + {{- if eq .Values.source.type "oracle" }} + {{- if .Values.source.oracle.pdbName }} + debezium.source.database.pdb.name={{ .Values.source.oracle.pdbName }} + {{- end }} + debezium.source.log.mining.strategy={{ .Values.source.oracle.logMiningStrategy }} + {{- end }} + + # ----------------------------------------------------------------------------- + # Offset Storage Configuration + # ----------------------------------------------------------------------------- + {{- if eq .Values.source.offset.storage "file" }} + debezium.source.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore + debezium.source.offset.storage.file.filename={{ .Values.source.offset.file.filename }} + {{- else if eq .Values.source.offset.storage "redis" }} + debezium.source.offset.storage=io.debezium.storage.redis.offset.RedisOffsetBackingStore + debezium.source.offset.storage.redis.address=${OFFSET_REDIS_ADDRESS} + debezium.source.offset.storage.redis.key={{ .Values.source.offset.redis.key }} + {{- if .Values.source.offset.redis.password }} + debezium.source.offset.storage.redis.password=${OFFSET_REDIS_PASSWORD} + {{- end }} + {{- if .Values.source.offset.redis.ssl }} + debezium.source.offset.storage.redis.ssl.enabled=true + {{- end }} + {{- else if eq .Values.source.offset.storage "jdbc" }} + debezium.source.offset.storage=io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore + debezium.source.offset.storage.jdbc.url=${OFFSET_JDBC_URL} + debezium.source.offset.storage.jdbc.user=${OFFSET_JDBC_USER} + debezium.source.offset.storage.jdbc.password=${OFFSET_JDBC_PASSWORD} + debezium.source.offset.storage.jdbc.offset.table.name={{ .Values.source.offset.jdbc.tableName }} + {{- end }} + debezium.source.offset.flush.interval.ms={{ .Values.source.offset.flushIntervalMs }} + debezium.source.offset.flush.timeout.ms={{ .Values.source.offset.flushTimeoutMs }} + + # ----------------------------------------------------------------------------- + # Error Retry Configuration + # ----------------------------------------------------------------------------- + debezium.source.errors.retry.delay.initial.ms={{ .Values.source.errors.retryDelayInitialMs }} + debezium.source.errors.retry.delay.max.ms={{ .Values.source.errors.retryDelayMaxMs }} + debezium.source.errors.max.retries={{ .Values.source.errors.maxRetries }} + + {{- /* Schema History Storage (MySQL and SQL Server only) */}} + {{- if eq (include "debezium.requiresSchemaHistory" .) "true" }} + + # ----------------------------------------------------------------------------- + # Schema History Storage Configuration + # ----------------------------------------------------------------------------- + {{- if eq .Values.source.schemaHistory.storage "file" }} + debezium.source.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory + debezium.source.schema.history.internal.file.filename={{ .Values.source.schemaHistory.file.filename }} + {{- else if eq .Values.source.schemaHistory.storage "redis" }} + debezium.source.schema.history.internal=io.debezium.storage.redis.history.RedisSchemaHistory + debezium.source.schema.history.internal.redis.address=${SCHEMA_HISTORY_REDIS_ADDRESS} + debezium.source.schema.history.internal.redis.key={{ .Values.source.schemaHistory.redis.key }} + {{- if .Values.source.schemaHistory.redis.password }} + debezium.source.schema.history.internal.redis.password=${SCHEMA_HISTORY_REDIS_PASSWORD} + {{- end }} + {{- if .Values.source.schemaHistory.redis.ssl }} + debezium.source.schema.history.internal.redis.ssl.enabled=true + {{- end }} + {{- else if eq .Values.source.schemaHistory.storage "jdbc" }} + debezium.source.schema.history.internal=io.debezium.storage.jdbc.history.JdbcSchemaHistory + debezium.source.schema.history.internal.jdbc.url=${SCHEMA_HISTORY_JDBC_URL} + debezium.source.schema.history.internal.jdbc.user=${SCHEMA_HISTORY_JDBC_USER} + debezium.source.schema.history.internal.jdbc.password=${SCHEMA_HISTORY_JDBC_PASSWORD} + debezium.source.schema.history.internal.jdbc.schema.history.table.name={{ .Values.source.schemaHistory.jdbc.tableName }} + {{- end }} + {{- end }} + + # ----------------------------------------------------------------------------- + # Sink Configuration + # ----------------------------------------------------------------------------- + {{- if eq .Values.sink.type "kafka" }} + debezium.sink.type=kafka + debezium.sink.kafka.producer.bootstrap.servers=${KAFKA_BOOTSTRAP_SERVERS} + debezium.sink.kafka.producer.key.serializer=org.apache.kafka.common.serialization.StringSerializer + debezium.sink.kafka.producer.value.serializer=org.apache.kafka.common.serialization.StringSerializer + {{- if .Values.sink.kafka.topic }} + debezium.sink.kafka.producer.topic.prefix={{ .Values.sink.kafka.topic }} + {{- end }} + debezium.sink.kafka.producer.security.protocol={{ .Values.sink.kafka.securityProtocol }} + {{- if .Values.sink.kafka.saslMechanism }} + debezium.sink.kafka.producer.sasl.mechanism={{ .Values.sink.kafka.saslMechanism }} + debezium.sink.kafka.producer.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="${KAFKA_SASL_USERNAME}" password="${KAFKA_SASL_PASSWORD}"; + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "redis" }} + debezium.sink.type=redis + debezium.sink.redis.address=${SINK_REDIS_ADDRESS} + {{- if .Values.sink.redis.password }} + debezium.sink.redis.password=${SINK_REDIS_PASSWORD} + {{- end }} + {{- if .Values.sink.redis.ssl }} + debezium.sink.redis.ssl.enabled=true + {{- end }} + {{- if .Values.sink.redis.streamName }} + debezium.sink.redis.stream.name={{ .Values.sink.redis.streamName }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "nats-jetstream" }} + debezium.sink.type=nats-jetstream + debezium.sink.nats-jetstream.url=${NATS_URL} + {{- if .Values.sink.nats.subject }} + debezium.sink.nats-jetstream.subject={{ .Values.sink.nats.subject }} + {{- end }} + {{- if .Values.sink.nats.username }} + debezium.sink.nats-jetstream.username=${NATS_USERNAME} + debezium.sink.nats-jetstream.password=${NATS_PASSWORD} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "http" }} + debezium.sink.type=http + debezium.sink.http.url=${HTTP_SINK_URL} + {{- if eq .Values.sink.http.authType "basic" }} + debezium.sink.http.authentication.type=basic + debezium.sink.http.authentication.username=${HTTP_SINK_USERNAME} + debezium.sink.http.authentication.password=${HTTP_SINK_PASSWORD} + {{- else if eq .Values.sink.http.authType "bearer" }} + debezium.sink.http.authentication.type=bearer + debezium.sink.http.authentication.bearer.token=${HTTP_SINK_BEARER_TOKEN} + {{- end }} + {{- range $key, $value := .Values.sink.http.headers }} + debezium.sink.http.headers.{{ $key }}={{ $value }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "kinesis" }} + debezium.sink.type=kinesis + debezium.sink.kinesis.region={{ .Values.sink.kinesis.region }} + debezium.sink.kinesis.stream={{ .Values.sink.kinesis.streamName }} + debezium.sink.kinesis.credentials.provider={{ .Values.sink.kinesis.credentialsProvider }} + {{- end }} + + {{- if eq .Values.sink.type "pubsub" }} + debezium.sink.type=pubsub + debezium.sink.pubsub.project.id={{ .Values.sink.pubsub.projectId }} + {{- if .Values.sink.pubsub.topic }} + debezium.sink.pubsub.topic.prefix={{ .Values.sink.pubsub.topic }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "pulsar" }} + debezium.sink.type=pulsar + debezium.sink.pulsar.client.serviceUrl=${PULSAR_SERVICE_URL} + {{- if .Values.sink.pulsar.topic }} + debezium.sink.pulsar.topic.prefix={{ .Values.sink.pulsar.topic }} + {{- end }} + {{- if .Values.sink.pulsar.authPluginClassName }} + debezium.sink.pulsar.client.authPluginClassName={{ .Values.sink.pulsar.authPluginClassName }} + debezium.sink.pulsar.client.authParams=token:${PULSAR_AUTH_TOKEN} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "eventhubs" }} + debezium.sink.type=eventhubs + debezium.sink.eventhubs.connection.string=${EVENTHUBS_CONNECTION_STRING} + debezium.sink.eventhubs.hub.name={{ .Values.sink.eventhubs.hubName }} + {{- end }} + + # ----------------------------------------------------------------------------- + # Serialization Format + # ----------------------------------------------------------------------------- + debezium.format.key={{ .Values.format.key }} + debezium.format.value={{ .Values.format.value }} + {{- if or (eq .Values.format.key "avro") (eq .Values.format.key "protobuf") (eq .Values.format.value "avro") (eq .Values.format.value "protobuf") }} + {{- if .Values.format.schemaRegistry.url }} + debezium.format.key.schemas.enable=true + debezium.format.value.schemas.enable=true + debezium.format.schema.registry.url=${SCHEMA_REGISTRY_URL} + {{- if .Values.format.schemaRegistry.username }} + debezium.format.schema.registry.basic.auth.credentials.source=USER_INFO + debezium.format.schema.registry.basic.auth.user.info=${SCHEMA_REGISTRY_USERNAME}:${SCHEMA_REGISTRY_PASSWORD} + {{- end }} + {{- end }} + {{- end }} diff --git a/debezium-server/versions/1.0.0/templates/secret-credentials.yaml b/debezium-server/versions/1.0.0/templates/secret-credentials.yaml new file mode 100644 index 00000000..c4008cf6 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/secret-credentials.yaml @@ -0,0 +1,99 @@ +kind: secret +name: {{ include "debezium.credentials.name" . }} +description: Debezium Server credentials +tags: + {{- include "debezium.tags" . | nindent 2 }} +type: dictionary +data: + # Database credentials + db-hostname: {{ .Values.source.database.hostname | quote }} + db-user: {{ .Values.source.database.user | quote }} + db-password: {{ .Values.source.database.password | quote }} + + {{- /* MongoDB connection string */}} + {{- if and (eq .Values.source.type "mongodb") .Values.source.mongodb.connectionString }} + mongodb-connection-string: {{ .Values.source.mongodb.connectionString | quote }} + {{- end }} + + {{- /* Offset storage credentials */}} + {{- if eq .Values.source.offset.storage "redis" }} + offset-redis-address: {{ .Values.source.offset.redis.address | quote }} + {{- if .Values.source.offset.redis.password }} + offset-redis-password: {{ .Values.source.offset.redis.password | quote }} + {{- end }} + {{- end }} + {{- if eq .Values.source.offset.storage "jdbc" }} + offset-jdbc-url: {{ .Values.source.offset.jdbc.url | quote }} + offset-jdbc-user: {{ .Values.source.offset.jdbc.user | quote }} + offset-jdbc-password: {{ .Values.source.offset.jdbc.password | quote }} + {{- end }} + + {{- /* Schema history storage credentials (MySQL/SQL Server only) */}} + {{- if eq (include "debezium.requiresSchemaHistory" .) "true" }} + {{- if eq .Values.source.schemaHistory.storage "redis" }} + schema-history-redis-address: {{ .Values.source.schemaHistory.redis.address | quote }} + {{- if .Values.source.schemaHistory.redis.password }} + schema-history-redis-password: {{ .Values.source.schemaHistory.redis.password | quote }} + {{- end }} + {{- end }} + {{- if eq .Values.source.schemaHistory.storage "jdbc" }} + schema-history-jdbc-url: {{ .Values.source.schemaHistory.jdbc.url | quote }} + schema-history-jdbc-user: {{ .Values.source.schemaHistory.jdbc.user | quote }} + schema-history-jdbc-password: {{ .Values.source.schemaHistory.jdbc.password | quote }} + {{- end }} + {{- end }} + + {{- /* Sink credentials */}} + {{- if eq .Values.sink.type "kafka" }} + kafka-bootstrap-servers: {{ .Values.sink.kafka.bootstrapServers | quote }} + {{- if .Values.sink.kafka.saslUsername }} + kafka-sasl-username: {{ .Values.sink.kafka.saslUsername | quote }} + kafka-sasl-password: {{ .Values.sink.kafka.saslPassword | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "redis" }} + sink-redis-address: {{ .Values.sink.redis.address | quote }} + {{- if .Values.sink.redis.password }} + sink-redis-password: {{ .Values.sink.redis.password | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "nats-jetstream" }} + nats-url: {{ .Values.sink.nats.url | quote }} + {{- if .Values.sink.nats.username }} + nats-username: {{ .Values.sink.nats.username | quote }} + nats-password: {{ .Values.sink.nats.password | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "http" }} + http-sink-url: {{ .Values.sink.http.url | quote }} + {{- if eq .Values.sink.http.authType "basic" }} + http-sink-username: {{ .Values.sink.http.username | quote }} + http-sink-password: {{ .Values.sink.http.password | quote }} + {{- end }} + {{- if eq .Values.sink.http.authType "bearer" }} + http-sink-bearer-token: {{ .Values.sink.http.bearerToken | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "pulsar" }} + pulsar-service-url: {{ .Values.sink.pulsar.serviceUrl | quote }} + {{- if .Values.sink.pulsar.authToken }} + pulsar-auth-token: {{ .Values.sink.pulsar.authToken | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "eventhubs" }} + eventhubs-connection-string: {{ .Values.sink.eventhubs.connectionString | quote }} + {{- end }} + + {{- /* Schema registry credentials */}} + {{- if .Values.format.schemaRegistry.url }} + schema-registry-url: {{ .Values.format.schemaRegistry.url | quote }} + {{- if .Values.format.schemaRegistry.username }} + schema-registry-username: {{ .Values.format.schemaRegistry.username | quote }} + schema-registry-password: {{ .Values.format.schemaRegistry.password | quote }} + {{- end }} + {{- end }} diff --git a/debezium-server/versions/1.0.0/templates/secret-entrypoint.yaml b/debezium-server/versions/1.0.0/templates/secret-entrypoint.yaml new file mode 100644 index 00000000..469c0503 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/secret-entrypoint.yaml @@ -0,0 +1,46 @@ +{{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} +kind: secret +name: {{ include "debezium.entrypoint.name" . }} +description: Debezium Server entrypoint script for PostgreSQL prerequisites +tags: + {{- include "debezium.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: | + #!/bin/bash + + set -o nounset + + echo "=== Debezium Server Entrypoint ===" + + # Find the PostgreSQL JDBC driver JAR (bundled with Debezium Server) + JDBC_JAR=$(find /debezium -name "postgresql-*.jar" -print -quit 2>/dev/null || true) + if [ -z "${JDBC_JAR}" ]; then + echo "WARNING: PostgreSQL JDBC driver not found. Skipping prerequisites." + exec /debezium/run.sh + fi + echo "Using JDBC driver: ${JDBC_JAR}" + + # Decode pre-compiled PgInit.class (Java 17+ compatible) + # Connects via JDBC, creates heartbeat table + failover replication slot + echo "yv66vgAAAD0AtQoAAgADBwAEDAAFAAYBABBqYXZhL2xhbmcvT2JqZWN0AQAGPGluaXQ+AQADKClWCQAIAAkHAAoMAAsADAEAEGphdmEvbGFuZy9TeXN0ZW0BAANlcnIBABVMamF2YS9pby9QcmludFN0cmVhbTsIAA4BAE5Vc2FnZTogUGdJbml0IDxob3N0PiA8cG9ydD4gPGRibmFtZT4gPHVzZXI+IDxwYXNzd29yZD4gPHNsb3ROYW1lPiA8cGx1Z2luTmFtZT4KABAAEQcAEgwAEwAUAQATamF2YS9pby9QcmludFN0cmVhbQEAB3ByaW50bG4BABUoTGphdmEvbGFuZy9TdHJpbmc7KVYKAAgAFgwAFwAYAQAEZXhpdAEABChJKVYSAAAAGgwAGwAcAQAXbWFrZUNvbmNhdFdpdGhDb25zdGFudHMBAEooTGphdmEvbGFuZy9TdHJpbmc7TGphdmEvbGFuZy9TdHJpbmc7TGphdmEvbGFuZy9TdHJpbmc7KUxqYXZhL2xhbmcvU3RyaW5nOwoAHgAfBwAgDAAhACIBABZqYXZhL3NxbC9Ecml2ZXJNYW5hZ2VyAQANZ2V0Q29ubmVjdGlvbgEATShMamF2YS9sYW5nL1N0cmluZztMamF2YS9sYW5nL1N0cmluZztMamF2YS9sYW5nL1N0cmluZzspTGphdmEvc3FsL0Nvbm5lY3Rpb247CQAIACQMACUADAEAA291dAgAJwEAGENvbm5lY3RlZCB0byBQb3N0Z3JlU1FMLgcAKQEAFWphdmEvc3FsL1NRTEV4Y2VwdGlvbgoAKAArDAAsAC0BAApnZXRNZXNzYWdlAQAUKClMamF2YS9sYW5nL1N0cmluZzsSAAEALwwAGwAwAQAnKElMamF2YS9sYW5nL1N0cmluZzspTGphdmEvbGFuZy9TdHJpbmc7EgACADIMABsAMwEAKChJSUxqYXZhL2xhbmcvU3RyaW5nOylMamF2YS9sYW5nL1N0cmluZzsFAAAAAAAAE4gKADcAOAcAOQwAOgA7AQAQamF2YS9sYW5nL1RocmVhZAEABXNsZWVwAQAEKEopVgcAPQEAHmphdmEvbGFuZy9JbnRlcnJ1cHRlZEV4Y2VwdGlvbgoANwA/DABAAEEBAA1jdXJyZW50VGhyZWFkAQAUKClMamF2YS9sYW5nL1RocmVhZDsKADcAQwwARAAGAQAJaW50ZXJydXB0CABGAQArRW5zdXJpbmcgZGViZXppdW1faGVhcnRiZWF0IHRhYmxlIGV4aXN0cy4uLgsASABJBwBKDABLAEwBABNqYXZhL3NxbC9Db25uZWN0aW9uAQAPY3JlYXRlU3RhdGVtZW50AQAWKClMamF2YS9zcWwvU3RhdGVtZW50OwgATgEAa0NSRUFURSBUQUJMRSBJRiBOT1QgRVhJU1RTIGRlYmV6aXVtX2hlYXJ0YmVhdCAoaWQgSU5URUdFUiBQUklNQVJZIEtFWSwgdHMgVElNRVNUQU1QIE5PVCBOVUxMIERFRkFVTFQgbm93KCkpCwBQAFEHAFIMAFMAVAEAEmphdmEvc3FsL1N0YXRlbWVudAEAB2V4ZWN1dGUBABUoTGphdmEvbGFuZy9TdHJpbmc7KVoIAFYBAFVJTlNFUlQgSU5UTyBkZWJleml1bV9oZWFydGJlYXQgKGlkLCB0cykgVkFMVUVTICgxLCBub3coKSkgT04gQ09ORkxJQ1QgKGlkKSBETyBOT1RISU5HCwBQAFgMAFkABgEABWNsb3NlBwBbAQATamF2YS9sYW5nL1Rocm93YWJsZQoAWgBdDABeAF8BAA1hZGRTdXBwcmVzc2VkAQAYKExqYXZhL2xhbmcvVGhyb3dhYmxlOylWCABhAQAWSGVhcnRiZWF0IHRhYmxlIHJlYWR5LhIAAwBjDAAbAGQBACYoTGphdmEvbGFuZy9TdHJpbmc7KUxqYXZhL2xhbmcvU3RyaW5nOwgAZgEAPVNFTEVDVCBjb3VudCgqKSBGUk9NIHBnX3JlcGxpY2F0aW9uX3Nsb3RzIFdIRVJFIHNsb3RfbmFtZSA9ID8LAEgAaAwAaQBqAQAQcHJlcGFyZVN0YXRlbWVudAEAMChMamF2YS9sYW5nL1N0cmluZzspTGphdmEvc3FsL1ByZXBhcmVkU3RhdGVtZW50OwsAbABtBwBuDABvAHABABpqYXZhL3NxbC9QcmVwYXJlZFN0YXRlbWVudAEACXNldFN0cmluZwEAFihJTGphdmEvbGFuZy9TdHJpbmc7KVYLAGwAcgwAcwB0AQAMZXhlY3V0ZVF1ZXJ5AQAWKClMamF2YS9zcWwvUmVzdWx0U2V0OwsAdgB3BwB4DAB5AHoBABJqYXZhL3NxbC9SZXN1bHRTZXQBAARuZXh0AQADKClaCwB2AHwMAH0AfgEABmdldEludAEABChJKUkSAAQAgAwAGwCBAQA4KExqYXZhL2xhbmcvU3RyaW5nO0xqYXZhL2xhbmcvU3RyaW5nOylMamF2YS9sYW5nL1N0cmluZzsSAAUAYxIABgBjCwBsAFgLAEgAWAgAhwEAF1ByZXJlcXVpc2l0ZXMgY29tcGxldGUuEgAHAGMIAIoBACJEZWJleml1bSBTZXJ2ZXIgd2lsbCBzdGFydCBhbnl3YXkuBwCMAQAGUGdJbml0AQAEQ29kZQEAD0xpbmVOdW1iZXJUYWJsZQEABG1haW4BABYoW0xqYXZhL2xhbmcvU3RyaW5nOylWAQANU3RhY2tNYXBUYWJsZQcAkwEAE1tMamF2YS9sYW5nL1N0cmluZzsHAJUBABBqYXZhL2xhbmcvU3RyaW5nAQAKU291cmNlRmlsZQEAC1BnSW5pdC5qYXZhAQAQQm9vdHN0cmFwTWV0aG9kcwgAmgEAF2pkYmM6cG9zdGdyZXNxbDovLwE6AS8BCACcAQAsRVJST1I6IENvdWxkIG5vdCBjb25uZWN0IGFmdGVyIAEgYXR0ZW1wdHM6IAEIAJ4BACUgIEF0dGVtcHQgAS8BIC0gcmV0cnlpbmcgaW4gNXMuLi4gKAEpCACgAQAwRW5zdXJpbmcgZmFpbG92ZXIgcmVwbGljYXRpb24gc2xvdCAnAScgZXhpc3RzLi4uCACiAQBAU0VMRUNUIHBnX2NyZWF0ZV9sb2dpY2FsX3JlcGxpY2F0aW9uX3Nsb3QoJwEnLCAnAScsIGZhbHNlLCB0cnVlKQgApAEAJkNyZWF0ZWQgZmFpbG92ZXIgcmVwbGljYXRpb24gc2xvdCAnAScuCACmAQAkUmVwbGljYXRpb24gc2xvdCAnAScgYWxyZWFkeSBleGlzdHMuCACoAQAlV0FSTklORzogUHJlcmVxdWlzaXRlIHNldHVwIGZhaWxlZDogAQ8GAKoKAKsArAcArQwAGwCuAQAkamF2YS9sYW5nL2ludm9rZS9TdHJpbmdDb25jYXRGYWN0b3J5AQCYKExqYXZhL2xhbmcvaW52b2tlL01ldGhvZEhhbmRsZXMkTG9va3VwO0xqYXZhL2xhbmcvU3RyaW5nO0xqYXZhL2xhbmcvaW52b2tlL01ldGhvZFR5cGU7TGphdmEvbGFuZy9TdHJpbmc7W0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL2ludm9rZS9DYWxsU2l0ZTsBAAxJbm5lckNsYXNzZXMHALEBACVqYXZhL2xhbmcvaW52b2tlL01ldGhvZEhhbmRsZXMkTG9va3VwBwCzAQAeamF2YS9sYW5nL2ludm9rZS9NZXRob2RIYW5kbGVzAQAGTG9va3VwACEAiwACAAAAAAACAAEABQAGAAEAjQAAAB0AAQABAAAABSq3AAGxAAAAAQCOAAAABgABAAAAAwAJAI8AkAABAI0AAARsAAQAEAAAAgIqvhAHogAPsgAHEg22AA8EuAAVKgMyTCoEMk0qBTJOKgYyOgQqBzI6BSoIMjoGKhAGMjoHKywtugAZAAA6CBAeNgkBOgoENgsVCxUJowBjGQgZBBkFuAAdOgqyACMSJrYAD6cATToMFQsVCaAAGbIABxUJGQy2ACq6AC4AALYADwO4ABWyACMVCxUJGQy2ACq6ADEAALYADxQANLgANqcACzoNuAA+tgBChAsBp/+csgAjEkW2AA8ZCrkARwEAOgsZCxJNuQBPAgBXGQsSVbkATwIAVxkLxgAqGQu5AFcBAKcAIDoMGQvGABYZC7kAVwEApwAMOg0ZDBkNtgBcGQy/sgAjEmC2AA+yACMZBroAYgAAtgAPGQoSZbkAZwIAOgsZCwQZBrkAawMAGQu5AHEBADoMGQy5AHUBAFcZDAS5AHsCAJoAWRkKuQBHAQA6DRkNGQYZB7oAfwAAuQBPAgBXGQ3GACoZDbkAVwEApwAgOg4ZDcYAFhkNuQBXAQCnAAw6DxkOGQ+2AFwZDr+yACMZBroAggAAtgAPpwAQsgAjGQa6AIMAALYADxkLxgAqGQu5AIQBAKcAIDoMGQvGABYZC7kAhAEApwAMOg0ZDBkNtgBcGQy/GQq5AIUBALIAIxKGtgAPpwAdOguyAAcZC7YAKroAiAAAtgAPsgAHEom2AA+xAAkATwBiAGUAKACYAJ4AoQA8AMAA1ADjAFoA6gDxAPQAWgFPAWABbwBaAXYBfQGAAFoBIAGpAbgAWgG/AcYByQBaAK8B5AHnACgAAgCOAAAAwgAwAAAABQAHAAYADwAHABMACQAfAAoAKQALADQADAA+AA4AQgAPAEUAEABPABIAWgATAGIAFABlABUAZwAWAG4AFwCAABgAhAAaAJgAGwCpABAArwAgALcAIQDAACIAygAjANQAJADjACEBAAAlAQgAJwEVACgBIAApASoAKgEzACsBOwAsAUYALQFPAC4BYAAvAW8ALQGMADABnAAyAakANAG4ACgB1QA2AdwANwHkADsB5wA4AekAOQH5ADoCAQA8AJEAAAFIABcT/wA0AAwHAJIHAJQHAJQHAJQHAJQHAJQHAJQHAJQHAJQBBwBIAQAAXAcAKPwAHgcAKFwHADz6AAf6AAX/ADMADAcAkgcAlAcAlAcAlAcAlAcAlAcAlAcAlAcAlAEHAEgHAFAAAQcAWv8AEAANBwCSBwCUBwCUBwCUBwCUBwCUBwCUBwCUBwCUAQcASAcAUAcAWgABBwBaCPkAAv8AbgAOBwCSBwCUBwCUBwCUBwCUBwCUBwCUBwCUBwCUAQcASAcAbAcAdgcAUAABBwBa/wAQAA8HAJIHAJQHAJQHAJQHAJQHAJQHAJQHAJQHAJQBBwBIBwBsBwB2BwBQBwBaAAEHAFoI+QACD/oADE4HAFr/ABAADQcAkgcAlAcAlAcAlAcAlAcAlAcAlAcAlAcAlAEHAEgHAGwHAFoAAQcAWgj5AAJRBwAoGQADAJYAAAACAJcAmAAAADIACACpAAEAmQCpAAEAmwCpAAEAnQCpAAEAnwCpAAEAoQCpAAEAowCpAAEApQCpAAEApwCvAAAACgABALAAsgC0ABk=" | base64 -d > /tmp/PgInit.class + + if [ $? -ne 0 ]; then + echo "WARNING: Failed to decode PgInit.class. Skipping prerequisites." + exec /debezium/run.sh + fi + + echo "Running PostgreSQL prerequisites..." + java -cp "${JDBC_JAR}:/tmp" PgInit \ + "${DB_HOSTNAME}" \ + "{{ include "debezium.databasePort" . }}" \ + "{{ .Values.source.database.name }}" \ + "${DB_USER}" \ + "${DB_PASSWORD}" \ + "{{ .Values.source.postgres.slotName }}" \ + "{{ .Values.source.postgres.pluginName }}" || echo "WARNING: Prerequisites script returned non-zero. Continuing anyway." + + echo "=== Starting Debezium Server ===" + exec /debezium/run.sh +{{- end }} diff --git a/debezium-server/versions/1.0.0/templates/volumeset.yaml b/debezium-server/versions/1.0.0/templates/volumeset.yaml new file mode 100644 index 00000000..b6c4a5c3 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/volumeset.yaml @@ -0,0 +1,15 @@ +{{- if eq (include "debezium.requiresVolumeset" .) "true" }} +kind: volumeset +name: {{ include "debezium.volumeset.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +description: Debezium Server data volumeset for offset and schema history storage +tags: + {{- include "debezium.tags" . | nindent 2 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.volumeset.capacity }} + performanceClass: {{ .Values.volumeset.performanceClass }} + snapshots: + createFinalSnapshot: true + retentionDuration: 7d +{{- end }} diff --git a/debezium-server/versions/1.0.0/templates/workload-debezium.yaml b/debezium-server/versions/1.0.0/templates/workload-debezium.yaml new file mode 100644 index 00000000..e71cc327 --- /dev/null +++ b/debezium-server/versions/1.0.0/templates/workload-debezium.yaml @@ -0,0 +1,221 @@ +kind: workload +name: {{ include "debezium.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +description: Debezium Server CDC workload +tags: + {{- include "debezium.tags" . | nindent 2 }} +spec: + {{- if eq (include "debezium.requiresVolumeset" .) "true" }} + type: stateful + {{- else }} + type: standard + {{- end }} + identityLink: //identity/{{ include "debezium.identity.name" . }} + containers: + - name: debezium-server + {{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} + command: /bin/bash + args: + - '-c' + - >- + cp /scripts/debezium-entrypoint.sh /tmp/ && chmod +x /tmp/debezium-entrypoint.sh && + /tmp/debezium-entrypoint.sh + {{- end }} + image: {{ .Values.image }} + inheritEnv: false + cpu: {{ .Values.resources.cpu | quote }} + memory: {{ .Values.resources.memory | quote }} + ports: + - number: 8080 + protocol: http + env: + # Database credentials from secret + - name: DB_HOSTNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.db-hostname' + - name: DB_USER + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.db-user' + - name: DB_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.db-password' + + {{- /* MongoDB connection string */}} + {{- if and (eq .Values.source.type "mongodb") .Values.source.mongodb.connectionString }} + - name: MONGODB_CONNECTION_STRING + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.mongodb-connection-string' + {{- end }} + + {{- /* Offset storage credentials */}} + {{- if eq .Values.source.offset.storage "redis" }} + - name: OFFSET_REDIS_ADDRESS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-redis-address' + {{- if .Values.source.offset.redis.password }} + - name: OFFSET_REDIS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-redis-password' + {{- end }} + {{- end }} + {{- if eq .Values.source.offset.storage "jdbc" }} + - name: OFFSET_JDBC_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-jdbc-url' + - name: OFFSET_JDBC_USER + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-jdbc-user' + - name: OFFSET_JDBC_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-jdbc-password' + {{- end }} + + {{- /* Schema history storage credentials */}} + {{- if eq (include "debezium.requiresSchemaHistory" .) "true" }} + {{- if eq .Values.source.schemaHistory.storage "redis" }} + - name: SCHEMA_HISTORY_REDIS_ADDRESS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-redis-address' + {{- if .Values.source.schemaHistory.redis.password }} + - name: SCHEMA_HISTORY_REDIS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-redis-password' + {{- end }} + {{- end }} + {{- if eq .Values.source.schemaHistory.storage "jdbc" }} + - name: SCHEMA_HISTORY_JDBC_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-jdbc-url' + - name: SCHEMA_HISTORY_JDBC_USER + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-jdbc-user' + - name: SCHEMA_HISTORY_JDBC_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-jdbc-password' + {{- end }} + {{- end }} + + {{- /* Sink credentials */}} + {{- if eq .Values.sink.type "kafka" }} + - name: KAFKA_BOOTSTRAP_SERVERS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.kafka-bootstrap-servers' + {{- if .Values.sink.kafka.saslUsername }} + - name: KAFKA_SASL_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.kafka-sasl-username' + - name: KAFKA_SASL_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.kafka-sasl-password' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "redis" }} + - name: SINK_REDIS_ADDRESS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.sink-redis-address' + {{- if .Values.sink.redis.password }} + - name: SINK_REDIS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.sink-redis-password' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "nats-jetstream" }} + - name: NATS_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.nats-url' + {{- if .Values.sink.nats.username }} + - name: NATS_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.nats-username' + - name: NATS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.nats-password' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "http" }} + - name: HTTP_SINK_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-url' + {{- if eq .Values.sink.http.authType "basic" }} + - name: HTTP_SINK_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-username' + - name: HTTP_SINK_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-password' + {{- end }} + {{- if eq .Values.sink.http.authType "bearer" }} + - name: HTTP_SINK_BEARER_TOKEN + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-bearer-token' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "pulsar" }} + - name: PULSAR_SERVICE_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.pulsar-service-url' + {{- if .Values.sink.pulsar.authToken }} + - name: PULSAR_AUTH_TOKEN + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.pulsar-auth-token' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "eventhubs" }} + - name: EVENTHUBS_CONNECTION_STRING + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.eventhubs-connection-string' + {{- end }} + + {{- /* Schema registry credentials */}} + {{- if .Values.format.schemaRegistry.url }} + - name: SCHEMA_REGISTRY_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-registry-url' + {{- if .Values.format.schemaRegistry.username }} + - name: SCHEMA_REGISTRY_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-registry-username' + - name: SCHEMA_REGISTRY_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-registry-password' + {{- end }} + {{- end }} + + volumes: + # Mount application.properties from opaque secret + - path: /debezium/config/application.properties + recoveryPolicy: retain + uri: cpln://secret/{{ include "debezium.config.name" . }}.payload + {{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} + # Mount entrypoint script for PostgreSQL prerequisites + - path: /scripts/debezium-entrypoint.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "debezium.entrypoint.name" . }}.payload + {{- end }} + {{- if eq (include "debezium.requiresVolumeset" .) "true" }} + # Mount data volume for offset and schema history storage + - path: /debezium/data + uri: cpln://volumeset/{{ include "debezium.volumeset.name" . }} + {{- end }} + + readinessProbe: + httpGet: + path: /q/health/ready + port: 8080 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + + livenessProbe: + httpGet: + path: /q/health/live + port: 8080 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + + defaultOptions: + # Debezium should run as a single instance for CDC consistency + autoscaling: + metric: disabled + minScale: 1 + maxScale: 1 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 60 + + {{- if eq (include "debezium.requiresVolumeset" .) "true" }} + securityOptions: + filesystemGroupId: 185 + {{- end }} + + firewallConfig: + external: + outboundAllowCIDR: + {{- toYaml .Values.firewall.external.outboundAllowCIDR | nindent 8 }} + internal: + inboundAllowType: {{ .Values.firewall.internal.inboundAllowType }} + {{- if .Values.firewall.internal.workloads }} + inboundAllowWorkload: + {{- toYaml .Values.firewall.internal.workloads | nindent 8 }} + {{- end }} diff --git a/debezium-server/versions/1.0.0/values.yaml b/debezium-server/versions/1.0.0/values.yaml new file mode 100644 index 00000000..9719f2d1 --- /dev/null +++ b/debezium-server/versions/1.0.0/values.yaml @@ -0,0 +1,219 @@ +# Debezium Server CDC Template +# Documentation: https://debezium.io/documentation/reference/stable/operations/debezium-server.html + +image: quay.io/debezium/server:3.0 + +resources: + cpu: 500m + memory: 512Mi + +# ============================================================================= +# Source Database Configuration +# ============================================================================= +source: + # Database type: postgres, mysql, mongodb, sqlserver, oracle + type: postgres + + # Database connection settings + database: + hostname: "" # Required: database hostname or IP + port: 5432 # Default port varies by type (postgres:5432, mysql:3306, mongodb:27017, sqlserver:1433, oracle:1521) + name: "" # Required: database name (or SID for Oracle) + user: "" # Required: database username + password: "" # Required: database password (stored in credentials secret) + + # Server name used as prefix for topic names + serverName: "dbserver1" + + # Tables to capture (comma-separated, e.g., "public.users,public.orders") + # Leave empty to capture all tables + tableIncludeList: "" + + # Tables to exclude (comma-separated) + tableExcludeList: "" + + # PostgreSQL-specific settings + postgres: + slotName: "debezium" # Replication slot name + publicationName: "dbz_publication" # Publication name + pluginName: "pgoutput" # Logical decoding plugin: pgoutput (default), decoderbufs + slotDropOnStop: false # Keep replication slot on stop (required for HA/failover) + heartbeatIntervalMs: 0 # Heartbeat interval in ms (0=disabled; set 5000 for HA) + heartbeatActionQuery: "" # SQL executed on heartbeat (e.g., "UPDATE debezium_heartbeat SET ts = now() WHERE id = 1") + + # MySQL-specific settings + mysql: + serverId: 85744 # Unique server ID for MySQL replication + includeSchemaChanges: true # Include DDL events + + # MongoDB-specific settings + mongodb: + connectionString: "" # Full connection string (overrides hostname/port) + replicaSet: "" # Replica set name + + # SQL Server-specific settings + sqlserver: + databaseNames: "" # Comma-separated database names to capture + snapshotMode: "initial" # Snapshot mode: initial, schema_only, initial_only + + # Oracle-specific settings + oracle: + pdbName: "" # Pluggable database name + logMiningStrategy: "online_catalog" # Log mining strategy: online_catalog, redo_log_catalog + + # Offset storage configuration + offset: + # Storage type: file, redis, jdbc + storage: file + + # Flush settings + flushIntervalMs: 10000 # How often offsets flush to storage (ms) + flushTimeoutMs: 60000 # Timeout for offset flush operations (ms) + + # File storage settings (requires volumeset) + file: + filename: "/debezium/data/offsets.dat" + + # Redis storage settings + redis: + address: "" # Redis address (e.g., redis.mygvc.cpln.local:6379) + key: "debezium:offsets" # Redis key for offsets + password: "" # Redis password (stored in credentials secret) + ssl: false # Enable SSL/TLS + + # JDBC storage settings + jdbc: + url: "" # JDBC URL (e.g., jdbc:postgresql://host:5432/dbname) + user: "" # JDBC username + password: "" # JDBC password (stored in credentials secret) + tableName: "debezium_offsets" # Table name for storing offsets + + # Schema history storage (required for MySQL and SQL Server) + schemaHistory: + # Storage type: file, redis, jdbc (only used for mysql/sqlserver) + storage: file + + # File storage settings + file: + filename: "/debezium/data/schema-history.dat" + + # Redis storage settings + redis: + address: "" + key: "debezium:schema-history" + password: "" + ssl: false + + # JDBC storage settings + jdbc: + url: "" + user: "" + password: "" + tableName: "debezium_schema_history" + + # Error retry configuration + errors: + retryDelayInitialMs: 300 # Initial retry delay (ms) + retryDelayMaxMs: 10000 # Max retry delay (ms) + maxRetries: -1 # Max retries (-1 = infinite) + +# ============================================================================= +# Sink Configuration +# ============================================================================= +sink: + # Sink type: kafka, redis, nats-jetstream, http, kinesis, pubsub, pulsar, eventhubs + type: kafka + + # Kafka sink settings + kafka: + bootstrapServers: "" # Required: Kafka bootstrap servers (e.g., kafka.mygvc.cpln.local:9092) + topic: "" # Topic prefix (events sent to {topic}.{table}) + securityProtocol: "PLAINTEXT" # Security protocol: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + saslMechanism: "" # SASL mechanism: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512 + saslUsername: "" # SASL username + saslPassword: "" # SASL password (stored in credentials secret) + + # Redis sink settings (Redis Streams) + redis: + address: "" # Required: Redis address (e.g., redis.mygvc.cpln.local:6379) + password: "" # Redis password (stored in credentials secret) + ssl: false # Enable SSL/TLS + streamName: "" # Stream name prefix (events sent to {streamName}.{table}) + + # NATS JetStream sink settings + nats: + url: "" # Required: NATS URL (e.g., nats://nats.mygvc.cpln.local:4222) + subject: "" # Subject prefix (events sent to {subject}.{table}) + username: "" # NATS username + password: "" # NATS password (stored in credentials secret) + + # HTTP sink settings (webhooks) + http: + url: "" # Required: HTTP endpoint URL + headers: {} # Additional headers (key-value pairs) + authType: "" # Auth type: none, basic, bearer + username: "" # Basic auth username + password: "" # Basic auth password (stored in credentials secret) + bearerToken: "" # Bearer token (stored in credentials secret) + + # AWS Kinesis sink settings (uses Universal Cloud Identity) + kinesis: + region: "" # Required: AWS region + streamName: "" # Required: Kinesis stream name + credentialsProvider: "default" # Use "default" for Universal Cloud Identity + # Cloud account for Universal Cloud Identity + cloudAccount: + enabled: false + name: "" # AWS cloud account name in Control Plane + + # GCP Pub/Sub sink settings (uses Universal Cloud Identity) + pubsub: + projectId: "" # Required: GCP project ID + topic: "" # Topic prefix (events sent to {topic}.{table}) + # Cloud account for Universal Cloud Identity + cloudAccount: + enabled: false + name: "" # GCP cloud account name in Control Plane + + # Apache Pulsar sink settings + pulsar: + serviceUrl: "" # Required: Pulsar service URL + topic: "" # Topic prefix + authPluginClassName: "" # Auth plugin class (e.g., org.apache.pulsar.client.impl.auth.AuthenticationToken) + authToken: "" # Auth token (stored in credentials secret) + + # Azure Event Hubs sink settings + eventhubs: + connectionString: "" # Required: Event Hubs connection string (stored in credentials secret) + hubName: "" # Required: Event Hub name + +# ============================================================================= +# Serialization Format +# ============================================================================= +format: + key: json # Key format: json, avro, protobuf + value: json # Value format: json, avro, protobuf + + # Schema registry settings (for avro/protobuf) + schemaRegistry: + url: "" # Schema registry URL + username: "" # Schema registry username + password: "" # Schema registry password (stored in credentials secret) + +# ============================================================================= +# Volumeset Configuration (for file-based offset/schema-history storage) +# ============================================================================= +volumeset: + capacity: 10 # Initial capacity in GiB (minimum 10) + performanceClass: general-purpose-ssd # Performance class: general-purpose-ssd, high-throughput-ssd + +# ============================================================================= +# Firewall Configuration +# ============================================================================= +firewall: + internal: + inboundAllowType: same-gvc # Options: none, same-gvc, same-org, workload-list + workloads: [] # Workload list for inbound access (when type is workload-list) + external: + outboundAllowCIDR: + - 0.0.0.0/0 # Allow all outbound by default (required for database connectivity) diff --git a/debezium-server/versions/1.1.0/Chart.yaml b/debezium-server/versions/1.1.0/Chart.yaml new file mode 100644 index 00000000..1fff4825 --- /dev/null +++ b/debezium-server/versions/1.1.0/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: debezium-server +description: Debezium Server CDC app for Control Plane (standalone mode) +type: application +version: 1.1.0 +appVersion: "3.0" + +annotations: + created: "2026-04-03" + lastModified: "2026-04-13" + category: "event-streaming" + createsGvc: false diff --git a/debezium-server/versions/1.1.0/README.md b/debezium-server/versions/1.1.0/README.md new file mode 100644 index 00000000..02c3ac45 --- /dev/null +++ b/debezium-server/versions/1.1.0/README.md @@ -0,0 +1,347 @@ +# Debezium Server Template + +Debezium Server is a standalone Change Data Capture (CDC) application that streams database changes to various messaging systems. Unlike Debezium connectors that run on Kafka Connect, Debezium Server runs as a standalone application and can send events directly to Kafka, Redis, NATS, HTTP endpoints, cloud services, and more. + +## Overview + +This template deploys Debezium Server on Control Plane with: + +- Configurable source database connectors (PostgreSQL, MySQL, MongoDB, SQL Server, Oracle) +- Multiple sink options (Kafka, Redis, NATS JetStream, HTTP, AWS Kinesis, GCP Pub/Sub, Pulsar, Event Hubs) +- Flexible offset storage (file, Redis, JDBC) +- Universal Cloud Identity integration for AWS and GCP sinks +- Automatic secret management for credentials + +## Quick Start + +### PostgreSQL to Kafka + +```yaml +source: + type: postgres + database: + hostname: postgres.mygvc.cpln.local + port: 5432 + name: mydb + user: debezium + password: secret123 + serverName: myserver + tableIncludeList: "public.users,public.orders" + postgres: + slotName: debezium_slot + publicationName: dbz_publication + +sink: + type: kafka + kafka: + bootstrapServers: kafka.mygvc.cpln.local:9092 + topic: cdc-events + +format: + key: json + value: json +``` + +### MySQL to Redis Streams + +```yaml +source: + type: mysql + database: + hostname: mysql.mygvc.cpln.local + port: 3306 + name: mydb + user: debezium + password: secret123 + serverName: myserver + mysql: + serverId: 85744 + includeSchemaChanges: true + +sink: + type: redis + redis: + address: redis.mygvc.cpln.local:6379 + streamName: cdc-stream +``` + +### PostgreSQL to AWS Kinesis (Universal Cloud Identity) + +```yaml +source: + type: postgres + database: + hostname: my-rds-instance.us-east-1.rds.amazonaws.com + port: 5432 + name: mydb + user: debezium + password: secret123 + serverName: myserver + +sink: + type: kinesis + kinesis: + region: us-east-1 + streamName: cdc-events + credentialsProvider: default + cloudAccount: + enabled: true + name: my-aws-account +``` + +## Supported Sources + +| Database | Connector | Default Port | Key Configuration | +|----------|-----------|--------------|-------------------| +| PostgreSQL | PostgresConnector | 5432 | `slotName`, `publicationName`, `pluginName` | +| MySQL | MySqlConnector | 3306 | `serverId`, `includeSchemaChanges` | +| MongoDB | MongoDbConnector | 27017 | `connectionString`, `replicaSet` | +| SQL Server | SqlServerConnector | 1433 | `databaseNames`, `snapshotMode` | +| Oracle | OracleConnector | 1521 | `pdbName`, `logMiningStrategy` | + +### PostgreSQL Prerequisites + +1. Enable logical replication in `postgresql.conf`: + ``` + wal_level = logical + max_replication_slots = 4 + max_wal_senders = 4 + ``` + +2. Create a publication and replication slot: + ```sql + CREATE PUBLICATION dbz_publication FOR ALL TABLES; + -- Slot is created automatically by Debezium + ``` + +3. Grant permissions: + ```sql + GRANT USAGE ON SCHEMA public TO debezium; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO debezium; + ALTER USER debezium REPLICATION; + ``` + +### MySQL Prerequisites + +1. Enable binary logging in `my.cnf`: + ``` + server-id = 1 + log_bin = mysql-bin + binlog_format = ROW + binlog_row_image = FULL + ``` + +2. Grant permissions: + ```sql + GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'debezium'@'%'; + ``` + +## Supported Sinks + +| Sink | Required Configuration | Notes | +|------|------------------------|-------| +| Kafka | `bootstrapServers` | Simple Kafka producer (no Kafka Connect required) | +| Redis | `address` | Redis Streams for real-time event streaming | +| NATS JetStream | `url` | Cloud-native messaging with persistence | +| HTTP | `url` | Webhooks and custom HTTP endpoints | +| Kinesis | `region`, `streamName` | AWS Kinesis (uses Universal Cloud Identity) | +| Pub/Sub | `projectId` | GCP Pub/Sub (uses Universal Cloud Identity) | +| Pulsar | `serviceUrl` | Apache Pulsar with optional authentication | +| Event Hubs | `connectionString`, `hubName` | Azure Event Hubs | + +## Offset Storage + +Debezium tracks the position of captured changes using offset storage. Three options are available: + +### File Storage (Default) + +Stores offsets in a local file. Requires a volumeset for persistence. + +```yaml +source: + offset: + storage: file + file: + filename: /debezium/data/offsets.dat + +volumeset: + capacity: 10 + performanceClass: general-purpose-ssd +``` + +### Redis Storage + +Stores offsets in Redis. No volumeset required. + +```yaml +source: + offset: + storage: redis + redis: + address: redis.mygvc.cpln.local:6379 + key: debezium:offsets + password: "" + ssl: false +``` + +### JDBC Storage + +Stores offsets in a relational database. No volumeset required. + +```yaml +source: + offset: + storage: jdbc + jdbc: + url: jdbc:postgresql://postgres.mygvc.cpln.local:5432/offsets + user: debezium + password: secret123 + tableName: debezium_offsets +``` + +## Schema History (MySQL/SQL Server Only) + +MySQL and SQL Server connectors require schema history storage to track DDL changes: + +```yaml +source: + type: mysql + schemaHistory: + storage: file # or: redis, jdbc + file: + filename: /debezium/data/schema-history.dat +``` + +## Serialization Formats + +Supports JSON, Avro, and Protobuf serialization: + +```yaml +format: + key: json + value: json + + # For Avro/Protobuf, configure schema registry: + schemaRegistry: + url: http://schema-registry.mygvc.cpln.local:8081 + username: "" + password: "" +``` + +## Universal Cloud Identity + +For AWS Kinesis and GCP Pub/Sub sinks, this template integrates with Control Plane's Universal Cloud Identity for credential-less authentication. + +### AWS Kinesis + +1. Create an AWS cloud account in Control Plane +2. Configure the identity with appropriate IAM policies +3. Enable the cloud account in your values: + +```yaml +sink: + type: kinesis + kinesis: + region: us-east-1 + streamName: my-stream + credentialsProvider: default + cloudAccount: + enabled: true + name: my-aws-account +``` + +### GCP Pub/Sub + +```yaml +sink: + type: pubsub + pubsub: + projectId: my-gcp-project + cloudAccount: + enabled: true + name: my-gcp-account +``` + +## Resource Configuration + +```yaml +resources: + cpu: 500m # CPU allocation + memory: 512Mi # Memory allocation + +volumeset: + capacity: 10 # GiB (only used with file storage) + performanceClass: general-purpose-ssd +``` + +## Firewall Configuration + +```yaml +firewall: + internal: + inboundAllowType: same-gvc # none, same-gvc, same-org, workload-list + workloads: [] # For workload-list type + external: + outboundAllowCIDR: + - 0.0.0.0/0 # Required for external database connectivity +``` + +## Health Checks + +Debezium Server exposes Quarkus health endpoints: + +- **Readiness**: `/q/health/ready` - Checks if the connector is ready +- **Liveness**: `/q/health/live` - Checks if the server is alive + +## Installation + +```bash +cpln helm install debezium ./debezium-server/versions/1.0.0 \ + --gvc my-gvc \ + -f my-values.yaml +``` + +## Verification + +1. Check workload status: + ```bash + cpln workload get debezium--debezium --gvc my-gvc + ``` + +2. Check health endpoint: + ```bash + curl http://debezium--debezium.my-gvc.cpln.local:8080/q/health + ``` + +3. View logs: + ```bash + cpln workload logs debezium--debezium --gvc my-gvc + ``` + +4. Test CDC by making changes in the source database and verifying events appear in the configured sink. + +## Troubleshooting + +### Connector Not Starting + +- Check database connectivity and credentials +- Verify replication permissions are granted +- Review logs for specific error messages + +### Offset Storage Issues + +- For file storage: ensure volumeset is properly mounted +- For Redis/JDBC: verify connectivity and credentials +- Check that the storage backend is accessible from the GVC + +### Sink Delivery Failures + +- Verify sink connectivity and authentication +- For cloud sinks (Kinesis/Pub/Sub): ensure cloud account is properly configured +- Check firewall rules allow outbound traffic to the sink + +## Resources + +- [Debezium Documentation](https://debezium.io/documentation/) +- [Debezium Server Documentation](https://debezium.io/documentation/reference/stable/operations/debezium-server.html) +- [Control Plane Documentation](https://docs.controlplane.com/) diff --git a/debezium-server/versions/1.1.0/templates/_helpers.tpl b/debezium-server/versions/1.1.0/templates/_helpers.tpl new file mode 100644 index 00000000..ddc0309a --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/_helpers.tpl @@ -0,0 +1,293 @@ +{{/* +================================================================================ +Resource Naming +================================================================================ +*/}} + +{{/* +Debezium Server Workload Name +*/}} +{{- define "debezium.name" -}} +{{- printf "%s-debezium" .Release.Name }} +{{- end }} + +{{/* +Debezium Identity Name +*/}} +{{- define "debezium.identity.name" -}} +{{- printf "%s-debezium-identity" .Release.Name }} +{{- end }} + +{{/* +Debezium Policy Name +*/}} +{{- define "debezium.policy.name" -}} +{{- printf "%s-debezium-policy" .Release.Name }} +{{- end }} + +{{/* +Debezium Config Secret Name (opaque - application.properties) +*/}} +{{- define "debezium.config.name" -}} +{{- printf "%s-debezium-config" .Release.Name }} +{{- end }} + +{{/* +Debezium Credentials Secret Name (dictionary) +*/}} +{{- define "debezium.credentials.name" -}} +{{- printf "%s-debezium-credentials" .Release.Name }} +{{- end }} + +{{/* +Debezium Volumeset Name +*/}} +{{- define "debezium.volumeset.name" -}} +{{- printf "%s-debezium-data" .Release.Name }} +{{- end }} + +{{/* +Debezium Entrypoint Secret Name +*/}} +{{- define "debezium.entrypoint.name" -}} +{{- printf "%s-debezium-entrypoint" .Release.Name }} +{{- end }} + +{{/* +================================================================================ +Auto-Computation Helpers (for meta-template / umbrella chart usage) +================================================================================ +*/}} + +{{/* +Resolve database hostname: use explicit value if set, otherwise compute from Release.Name. +When used standalone, hostname is always set. When used as a subchart in a meta-template, +hostname can be left empty and will auto-compute to the postgres-ha-proxy DNS name. +*/}} +{{- define "debezium.dbHostname" -}} +{{- if .Values.source.database.hostname -}} +{{- .Values.source.database.hostname -}} +{{- else -}} +{{- printf "%s-postgres-ha-proxy.%s.cpln.local" .Release.Name .Values.global.cpln.gvc -}} +{{- end -}} +{{- end -}} + +{{/* +Resolve Kafka bootstrap servers: use explicit value if set, otherwise compute from Release.Name. +When used standalone, bootstrapServers is always set. When used as a subchart in a meta-template, +it can be left empty and will auto-compute to the kafka cluster DNS name. +*/}} +{{- define "debezium.kafkaBootstrapServers" -}} +{{- if .Values.sink.kafka.bootstrapServers -}} +{{- .Values.sink.kafka.bootstrapServers -}} +{{- else -}} +{{- printf "%s-cluster.%s.cpln.local:9092" .Release.Name .Values.global.cpln.gvc -}} +{{- end -}} +{{- end -}} + +{{/* +================================================================================ +Validation Helpers +================================================================================ +*/}} + +{{/* +Validate source configuration +*/}} +{{- define "debezium.validateSource" -}} +{{- $validTypes := list "postgres" "mysql" "mongodb" "sqlserver" "oracle" -}} +{{- if not (has .Values.source.type $validTypes) -}} +{{- fail (printf "Invalid source.type '%s'. Must be one of: %s" .Values.source.type (join ", " $validTypes)) -}} +{{- end -}} +{{- if not .Values.source.database.name -}} +{{- fail "source.database.name is required" -}} +{{- end -}} +{{- if not .Values.source.database.user -}} +{{- fail "source.database.user is required" -}} +{{- end -}} +{{- if not .Values.source.database.password -}} +{{- fail "source.database.password is required" -}} +{{- end -}} +{{- end -}} + +{{/* +Validate sink configuration +*/}} +{{- define "debezium.validateSink" -}} +{{- $validTypes := list "kafka" "redis" "nats-jetstream" "http" "kinesis" "pubsub" "pulsar" "eventhubs" -}} +{{- if not (has .Values.sink.type $validTypes) -}} +{{- fail (printf "Invalid sink.type '%s'. Must be one of: %s" .Values.sink.type (join ", " $validTypes)) -}} +{{- end -}} +{{- if eq .Values.sink.type "kafka" -}} + {{- if not (include "debezium.kafkaBootstrapServers" .) -}} + {{- fail "sink.kafka.bootstrapServers is required when sink.type is 'kafka'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "redis" -}} + {{- if not .Values.sink.redis.address -}} + {{- fail "sink.redis.address is required when sink.type is 'redis'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "nats-jetstream" -}} + {{- if not .Values.sink.nats.url -}} + {{- fail "sink.nats.url is required when sink.type is 'nats-jetstream'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "http" -}} + {{- if not .Values.sink.http.url -}} + {{- fail "sink.http.url is required when sink.type is 'http'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "kinesis" -}} + {{- if not .Values.sink.kinesis.region -}} + {{- fail "sink.kinesis.region is required when sink.type is 'kinesis'" -}} + {{- end -}} + {{- if not .Values.sink.kinesis.streamName -}} + {{- fail "sink.kinesis.streamName is required when sink.type is 'kinesis'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "pubsub" -}} + {{- if not .Values.sink.pubsub.projectId -}} + {{- fail "sink.pubsub.projectId is required when sink.type is 'pubsub'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "pulsar" -}} + {{- if not .Values.sink.pulsar.serviceUrl -}} + {{- fail "sink.pulsar.serviceUrl is required when sink.type is 'pulsar'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.sink.type "eventhubs" -}} + {{- if not .Values.sink.eventhubs.connectionString -}} + {{- fail "sink.eventhubs.connectionString is required when sink.type is 'eventhubs'" -}} + {{- end -}} + {{- if not .Values.sink.eventhubs.hubName -}} + {{- fail "sink.eventhubs.hubName is required when sink.type is 'eventhubs'" -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Validate offset storage configuration +*/}} +{{- define "debezium.validateOffsetStorage" -}} +{{- $validTypes := list "file" "redis" "jdbc" -}} +{{- if not (has .Values.source.offset.storage $validTypes) -}} +{{- fail (printf "Invalid source.offset.storage '%s'. Must be one of: %s" .Values.source.offset.storage (join ", " $validTypes)) -}} +{{- end -}} +{{- if eq .Values.source.offset.storage "redis" -}} + {{- if not .Values.source.offset.redis.address -}} + {{- fail "source.offset.redis.address is required when offset storage is 'redis'" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.source.offset.storage "jdbc" -}} + {{- if not .Values.source.offset.jdbc.url -}} + {{- fail "source.offset.jdbc.url is required when offset storage is 'jdbc'" -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +================================================================================ +Connector Class Mapping +================================================================================ +*/}} + +{{/* +Get the Debezium connector class for the source type +*/}} +{{- define "debezium.connectorClass" -}} +{{- $connectorMap := dict + "postgres" "io.debezium.connector.postgresql.PostgresConnector" + "mysql" "io.debezium.connector.mysql.MySqlConnector" + "mongodb" "io.debezium.connector.mongodb.MongoDbConnector" + "sqlserver" "io.debezium.connector.sqlserver.SqlServerConnector" + "oracle" "io.debezium.connector.oracle.OracleConnector" +-}} +{{- get $connectorMap .Values.source.type -}} +{{- end -}} + +{{/* +Get the default port for the source type +*/}} +{{- define "debezium.defaultPort" -}} +{{- $portMap := dict + "postgres" 5432 + "mysql" 3306 + "mongodb" 27017 + "sqlserver" 1433 + "oracle" 1521 +-}} +{{- get $portMap .Values.source.type -}} +{{- end -}} + +{{/* +Get the effective database port +*/}} +{{- define "debezium.databasePort" -}} +{{- if .Values.source.database.port -}} +{{- .Values.source.database.port -}} +{{- else -}} +{{- include "debezium.defaultPort" . -}} +{{- end -}} +{{- end -}} + +{{/* +Check if schema history is required (MySQL and SQL Server need it) +*/}} +{{- define "debezium.requiresSchemaHistory" -}} +{{- if or (eq .Values.source.type "mysql") (eq .Values.source.type "sqlserver") -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} + +{{/* +Check if file-based storage is used (requires volumeset) +*/}} +{{- define "debezium.requiresVolumeset" -}} +{{- if eq .Values.source.offset.storage "file" -}} +true +{{- else if and (eq (include "debezium.requiresSchemaHistory" .) "true") (eq .Values.source.schemaHistory.storage "file") -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} + +{{/* +================================================================================ +Labeling +================================================================================ +*/}} + +{{/* +Create chart name and version as used by the chart label +*/}} +{{- define "debezium.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels/tags +*/}} +{{- define "debezium.tags" -}} +helm.sh/chart: {{ include "debezium.chart" . }} +{{ include "debezium.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.cpln.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.cpln.io/managed-by: {{ .Release.Service }} +cpln/marketplace: "true" +cpln/marketplace-template: debezium-server +cpln/marketplace-template-version: {{ .Chart.Version }} +cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "debezium.selectorLabels" -}} +app.cpln.io/name: {{ .Release.Name }} +app.cpln.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/debezium-server/versions/1.1.0/templates/identity.yaml b/debezium-server/versions/1.1.0/templates/identity.yaml new file mode 100644 index 00000000..ac53fdd2 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/identity.yaml @@ -0,0 +1,19 @@ +{{- include "debezium.validateSource" . -}} +{{- include "debezium.validateSink" . -}} +{{- include "debezium.validateOffsetStorage" . -}} +kind: identity +name: {{ include "debezium.identity.name" . }} +description: Debezium Server identity for secret access and cloud integration +gvc: {{ .Values.global.cpln.gvc }} +tags: + {{- include "debezium.tags" . | nindent 2 }} +{{- if and (eq .Values.sink.type "kinesis") .Values.sink.kinesis.cloudAccount.enabled }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.sink.kinesis.cloudAccount.name }} +{{- end }} +{{- if and (eq .Values.sink.type "pubsub") .Values.sink.pubsub.cloudAccount.enabled }} +gcp: + cloudAccountLink: //cloudaccount/{{ .Values.sink.pubsub.cloudAccount.name }} + scopes: + - https://www.googleapis.com/auth/pubsub +{{- end }} diff --git a/debezium-server/versions/1.1.0/templates/policy.yaml b/debezium-server/versions/1.1.0/templates/policy.yaml new file mode 100644 index 00000000..b9e40ef1 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/policy.yaml @@ -0,0 +1,17 @@ +kind: policy +name: {{ include "debezium.policy.name" . }} +description: Debezium Server policy for secret access +tags: + {{- include "debezium.tags" . | nindent 2 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "debezium.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "debezium.config.name" . }} + - //secret/{{ include "debezium.credentials.name" . }} + {{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} + - //secret/{{ include "debezium.entrypoint.name" . }} + {{- end }} diff --git a/debezium-server/versions/1.1.0/templates/secret-config.yaml b/debezium-server/versions/1.1.0/templates/secret-config.yaml new file mode 100644 index 00000000..b32381b5 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/secret-config.yaml @@ -0,0 +1,260 @@ +kind: secret +name: {{ include "debezium.config.name" . }} +description: Debezium Server application.properties configuration +tags: + {{- include "debezium.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: |- + # ============================================================================= + # Debezium Server Configuration + # Generated by Control Plane Debezium Server Template + # ============================================================================= + + # ----------------------------------------------------------------------------- + # Quarkus Settings + # ----------------------------------------------------------------------------- + quarkus.http.port=8080 + quarkus.log.console.json=false + + # ----------------------------------------------------------------------------- + # Source Connector Configuration + # ----------------------------------------------------------------------------- + debezium.source.connector.class={{ include "debezium.connectorClass" . }} + debezium.source.topic.prefix={{ .Values.source.serverName }} + + # Database connection + debezium.source.database.hostname=${DB_HOSTNAME} + debezium.source.database.port={{ include "debezium.databasePort" . }} + debezium.source.database.user=${DB_USER} + debezium.source.database.password=${DB_PASSWORD} + {{- if or (eq .Values.source.type "oracle") (eq .Values.source.type "postgres") }} + debezium.source.database.dbname={{ .Values.source.database.name }} + {{- else }} + debezium.source.database.name={{ .Values.source.database.name }} + {{- end }} + + {{- if .Values.source.tableIncludeList }} + debezium.source.table.include.list={{ .Values.source.tableIncludeList }} + {{- end }} + {{- if .Values.source.tableExcludeList }} + debezium.source.table.exclude.list={{ .Values.source.tableExcludeList }} + {{- end }} + + {{- /* PostgreSQL-specific settings */}} + {{- if eq .Values.source.type "postgres" }} + debezium.source.plugin.name={{ .Values.source.postgres.pluginName }} + debezium.source.slot.name={{ .Values.source.postgres.slotName }} + debezium.source.publication.name={{ .Values.source.postgres.publicationName }} + debezium.source.slot.drop.on.stop={{ .Values.source.postgres.slotDropOnStop }} + {{- if gt (int .Values.source.postgres.heartbeatIntervalMs) 0 }} + debezium.source.heartbeat.interval.ms={{ .Values.source.postgres.heartbeatIntervalMs }} + {{- if .Values.source.postgres.heartbeatActionQuery }} + debezium.source.heartbeat.action.query={{ .Values.source.postgres.heartbeatActionQuery }} + {{- end }} + {{- end }} + {{- end }} + + {{- /* MySQL-specific settings */}} + {{- if eq .Values.source.type "mysql" }} + debezium.source.database.server.id={{ .Values.source.mysql.serverId }} + debezium.source.include.schema.changes={{ .Values.source.mysql.includeSchemaChanges }} + {{- end }} + + {{- /* MongoDB-specific settings */}} + {{- if eq .Values.source.type "mongodb" }} + {{- if .Values.source.mongodb.connectionString }} + debezium.source.mongodb.connection.string=${MONGODB_CONNECTION_STRING} + {{- end }} + {{- if .Values.source.mongodb.replicaSet }} + debezium.source.mongodb.replica.set={{ .Values.source.mongodb.replicaSet }} + {{- end }} + {{- end }} + + {{- /* SQL Server-specific settings */}} + {{- if eq .Values.source.type "sqlserver" }} + {{- if .Values.source.sqlserver.databaseNames }} + debezium.source.database.names={{ .Values.source.sqlserver.databaseNames }} + {{- end }} + debezium.source.snapshot.mode={{ .Values.source.sqlserver.snapshotMode }} + {{- end }} + + {{- /* Oracle-specific settings */}} + {{- if eq .Values.source.type "oracle" }} + {{- if .Values.source.oracle.pdbName }} + debezium.source.database.pdb.name={{ .Values.source.oracle.pdbName }} + {{- end }} + debezium.source.log.mining.strategy={{ .Values.source.oracle.logMiningStrategy }} + {{- end }} + + # ----------------------------------------------------------------------------- + # Offset Storage Configuration + # ----------------------------------------------------------------------------- + {{- if eq .Values.source.offset.storage "file" }} + debezium.source.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore + debezium.source.offset.storage.file.filename={{ .Values.source.offset.file.filename }} + {{- else if eq .Values.source.offset.storage "redis" }} + debezium.source.offset.storage=io.debezium.storage.redis.offset.RedisOffsetBackingStore + debezium.source.offset.storage.redis.address=${OFFSET_REDIS_ADDRESS} + debezium.source.offset.storage.redis.key={{ .Values.source.offset.redis.key }} + {{- if .Values.source.offset.redis.password }} + debezium.source.offset.storage.redis.password=${OFFSET_REDIS_PASSWORD} + {{- end }} + {{- if .Values.source.offset.redis.ssl }} + debezium.source.offset.storage.redis.ssl.enabled=true + {{- end }} + {{- else if eq .Values.source.offset.storage "jdbc" }} + debezium.source.offset.storage=io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore + debezium.source.offset.storage.jdbc.url=${OFFSET_JDBC_URL} + debezium.source.offset.storage.jdbc.user=${OFFSET_JDBC_USER} + debezium.source.offset.storage.jdbc.password=${OFFSET_JDBC_PASSWORD} + debezium.source.offset.storage.jdbc.offset.table.name={{ .Values.source.offset.jdbc.tableName }} + {{- end }} + debezium.source.offset.flush.interval.ms={{ .Values.source.offset.flushIntervalMs }} + debezium.source.offset.flush.timeout.ms={{ .Values.source.offset.flushTimeoutMs }} + + # ----------------------------------------------------------------------------- + # Error Retry Configuration + # ----------------------------------------------------------------------------- + debezium.source.errors.retry.delay.initial.ms={{ .Values.source.errors.retryDelayInitialMs }} + debezium.source.errors.retry.delay.max.ms={{ .Values.source.errors.retryDelayMaxMs }} + debezium.source.errors.max.retries={{ .Values.source.errors.maxRetries }} + + {{- /* Schema History Storage (MySQL and SQL Server only) */}} + {{- if eq (include "debezium.requiresSchemaHistory" .) "true" }} + + # ----------------------------------------------------------------------------- + # Schema History Storage Configuration + # ----------------------------------------------------------------------------- + {{- if eq .Values.source.schemaHistory.storage "file" }} + debezium.source.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory + debezium.source.schema.history.internal.file.filename={{ .Values.source.schemaHistory.file.filename }} + {{- else if eq .Values.source.schemaHistory.storage "redis" }} + debezium.source.schema.history.internal=io.debezium.storage.redis.history.RedisSchemaHistory + debezium.source.schema.history.internal.redis.address=${SCHEMA_HISTORY_REDIS_ADDRESS} + debezium.source.schema.history.internal.redis.key={{ .Values.source.schemaHistory.redis.key }} + {{- if .Values.source.schemaHistory.redis.password }} + debezium.source.schema.history.internal.redis.password=${SCHEMA_HISTORY_REDIS_PASSWORD} + {{- end }} + {{- if .Values.source.schemaHistory.redis.ssl }} + debezium.source.schema.history.internal.redis.ssl.enabled=true + {{- end }} + {{- else if eq .Values.source.schemaHistory.storage "jdbc" }} + debezium.source.schema.history.internal=io.debezium.storage.jdbc.history.JdbcSchemaHistory + debezium.source.schema.history.internal.jdbc.url=${SCHEMA_HISTORY_JDBC_URL} + debezium.source.schema.history.internal.jdbc.user=${SCHEMA_HISTORY_JDBC_USER} + debezium.source.schema.history.internal.jdbc.password=${SCHEMA_HISTORY_JDBC_PASSWORD} + debezium.source.schema.history.internal.jdbc.schema.history.table.name={{ .Values.source.schemaHistory.jdbc.tableName }} + {{- end }} + {{- end }} + + # ----------------------------------------------------------------------------- + # Sink Configuration + # ----------------------------------------------------------------------------- + {{- if eq .Values.sink.type "kafka" }} + debezium.sink.type=kafka + debezium.sink.kafka.producer.bootstrap.servers=${KAFKA_BOOTSTRAP_SERVERS} + debezium.sink.kafka.producer.key.serializer=org.apache.kafka.common.serialization.StringSerializer + debezium.sink.kafka.producer.value.serializer=org.apache.kafka.common.serialization.StringSerializer + {{- if .Values.sink.kafka.topic }} + debezium.sink.kafka.producer.topic.prefix={{ .Values.sink.kafka.topic }} + {{- end }} + debezium.sink.kafka.producer.security.protocol={{ .Values.sink.kafka.securityProtocol }} + {{- if .Values.sink.kafka.saslMechanism }} + debezium.sink.kafka.producer.sasl.mechanism={{ .Values.sink.kafka.saslMechanism }} + debezium.sink.kafka.producer.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="${KAFKA_SASL_USERNAME}" password="${KAFKA_SASL_PASSWORD}"; + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "redis" }} + debezium.sink.type=redis + debezium.sink.redis.address=${SINK_REDIS_ADDRESS} + {{- if .Values.sink.redis.password }} + debezium.sink.redis.password=${SINK_REDIS_PASSWORD} + {{- end }} + {{- if .Values.sink.redis.ssl }} + debezium.sink.redis.ssl.enabled=true + {{- end }} + {{- if .Values.sink.redis.streamName }} + debezium.sink.redis.stream.name={{ .Values.sink.redis.streamName }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "nats-jetstream" }} + debezium.sink.type=nats-jetstream + debezium.sink.nats-jetstream.url=${NATS_URL} + {{- if .Values.sink.nats.subject }} + debezium.sink.nats-jetstream.subject={{ .Values.sink.nats.subject }} + {{- end }} + {{- if .Values.sink.nats.username }} + debezium.sink.nats-jetstream.username=${NATS_USERNAME} + debezium.sink.nats-jetstream.password=${NATS_PASSWORD} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "http" }} + debezium.sink.type=http + debezium.sink.http.url=${HTTP_SINK_URL} + {{- if eq .Values.sink.http.authType "basic" }} + debezium.sink.http.authentication.type=basic + debezium.sink.http.authentication.username=${HTTP_SINK_USERNAME} + debezium.sink.http.authentication.password=${HTTP_SINK_PASSWORD} + {{- else if eq .Values.sink.http.authType "bearer" }} + debezium.sink.http.authentication.type=bearer + debezium.sink.http.authentication.bearer.token=${HTTP_SINK_BEARER_TOKEN} + {{- end }} + {{- range $key, $value := .Values.sink.http.headers }} + debezium.sink.http.headers.{{ $key }}={{ $value }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "kinesis" }} + debezium.sink.type=kinesis + debezium.sink.kinesis.region={{ .Values.sink.kinesis.region }} + debezium.sink.kinesis.stream={{ .Values.sink.kinesis.streamName }} + debezium.sink.kinesis.credentials.provider={{ .Values.sink.kinesis.credentialsProvider }} + {{- end }} + + {{- if eq .Values.sink.type "pubsub" }} + debezium.sink.type=pubsub + debezium.sink.pubsub.project.id={{ .Values.sink.pubsub.projectId }} + {{- if .Values.sink.pubsub.topic }} + debezium.sink.pubsub.topic.prefix={{ .Values.sink.pubsub.topic }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "pulsar" }} + debezium.sink.type=pulsar + debezium.sink.pulsar.client.serviceUrl=${PULSAR_SERVICE_URL} + {{- if .Values.sink.pulsar.topic }} + debezium.sink.pulsar.topic.prefix={{ .Values.sink.pulsar.topic }} + {{- end }} + {{- if .Values.sink.pulsar.authPluginClassName }} + debezium.sink.pulsar.client.authPluginClassName={{ .Values.sink.pulsar.authPluginClassName }} + debezium.sink.pulsar.client.authParams=token:${PULSAR_AUTH_TOKEN} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "eventhubs" }} + debezium.sink.type=eventhubs + debezium.sink.eventhubs.connection.string=${EVENTHUBS_CONNECTION_STRING} + debezium.sink.eventhubs.hub.name={{ .Values.sink.eventhubs.hubName }} + {{- end }} + + # ----------------------------------------------------------------------------- + # Serialization Format + # ----------------------------------------------------------------------------- + debezium.format.key={{ .Values.format.key }} + debezium.format.value={{ .Values.format.value }} + {{- if or (eq .Values.format.key "avro") (eq .Values.format.key "protobuf") (eq .Values.format.value "avro") (eq .Values.format.value "protobuf") }} + {{- if .Values.format.schemaRegistry.url }} + debezium.format.key.schemas.enable=true + debezium.format.value.schemas.enable=true + debezium.format.schema.registry.url=${SCHEMA_REGISTRY_URL} + {{- if .Values.format.schemaRegistry.username }} + debezium.format.schema.registry.basic.auth.credentials.source=USER_INFO + debezium.format.schema.registry.basic.auth.user.info=${SCHEMA_REGISTRY_USERNAME}:${SCHEMA_REGISTRY_PASSWORD} + {{- end }} + {{- end }} + {{- end }} diff --git a/debezium-server/versions/1.1.0/templates/secret-credentials.yaml b/debezium-server/versions/1.1.0/templates/secret-credentials.yaml new file mode 100644 index 00000000..d516d6a0 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/secret-credentials.yaml @@ -0,0 +1,99 @@ +kind: secret +name: {{ include "debezium.credentials.name" . }} +description: Debezium Server credentials +tags: + {{- include "debezium.tags" . | nindent 2 }} +type: dictionary +data: + # Database credentials + db-hostname: {{ include "debezium.dbHostname" . | quote }} + db-user: {{ .Values.source.database.user | quote }} + db-password: {{ .Values.source.database.password | quote }} + + {{- /* MongoDB connection string */}} + {{- if and (eq .Values.source.type "mongodb") .Values.source.mongodb.connectionString }} + mongodb-connection-string: {{ .Values.source.mongodb.connectionString | quote }} + {{- end }} + + {{- /* Offset storage credentials */}} + {{- if eq .Values.source.offset.storage "redis" }} + offset-redis-address: {{ .Values.source.offset.redis.address | quote }} + {{- if .Values.source.offset.redis.password }} + offset-redis-password: {{ .Values.source.offset.redis.password | quote }} + {{- end }} + {{- end }} + {{- if eq .Values.source.offset.storage "jdbc" }} + offset-jdbc-url: {{ .Values.source.offset.jdbc.url | quote }} + offset-jdbc-user: {{ .Values.source.offset.jdbc.user | quote }} + offset-jdbc-password: {{ .Values.source.offset.jdbc.password | quote }} + {{- end }} + + {{- /* Schema history storage credentials (MySQL/SQL Server only) */}} + {{- if eq (include "debezium.requiresSchemaHistory" .) "true" }} + {{- if eq .Values.source.schemaHistory.storage "redis" }} + schema-history-redis-address: {{ .Values.source.schemaHistory.redis.address | quote }} + {{- if .Values.source.schemaHistory.redis.password }} + schema-history-redis-password: {{ .Values.source.schemaHistory.redis.password | quote }} + {{- end }} + {{- end }} + {{- if eq .Values.source.schemaHistory.storage "jdbc" }} + schema-history-jdbc-url: {{ .Values.source.schemaHistory.jdbc.url | quote }} + schema-history-jdbc-user: {{ .Values.source.schemaHistory.jdbc.user | quote }} + schema-history-jdbc-password: {{ .Values.source.schemaHistory.jdbc.password | quote }} + {{- end }} + {{- end }} + + {{- /* Sink credentials */}} + {{- if eq .Values.sink.type "kafka" }} + kafka-bootstrap-servers: {{ include "debezium.kafkaBootstrapServers" . | quote }} + {{- if .Values.sink.kafka.saslUsername }} + kafka-sasl-username: {{ .Values.sink.kafka.saslUsername | quote }} + kafka-sasl-password: {{ .Values.sink.kafka.saslPassword | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "redis" }} + sink-redis-address: {{ .Values.sink.redis.address | quote }} + {{- if .Values.sink.redis.password }} + sink-redis-password: {{ .Values.sink.redis.password | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "nats-jetstream" }} + nats-url: {{ .Values.sink.nats.url | quote }} + {{- if .Values.sink.nats.username }} + nats-username: {{ .Values.sink.nats.username | quote }} + nats-password: {{ .Values.sink.nats.password | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "http" }} + http-sink-url: {{ .Values.sink.http.url | quote }} + {{- if eq .Values.sink.http.authType "basic" }} + http-sink-username: {{ .Values.sink.http.username | quote }} + http-sink-password: {{ .Values.sink.http.password | quote }} + {{- end }} + {{- if eq .Values.sink.http.authType "bearer" }} + http-sink-bearer-token: {{ .Values.sink.http.bearerToken | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "pulsar" }} + pulsar-service-url: {{ .Values.sink.pulsar.serviceUrl | quote }} + {{- if .Values.sink.pulsar.authToken }} + pulsar-auth-token: {{ .Values.sink.pulsar.authToken | quote }} + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "eventhubs" }} + eventhubs-connection-string: {{ .Values.sink.eventhubs.connectionString | quote }} + {{- end }} + + {{- /* Schema registry credentials */}} + {{- if .Values.format.schemaRegistry.url }} + schema-registry-url: {{ .Values.format.schemaRegistry.url | quote }} + {{- if .Values.format.schemaRegistry.username }} + schema-registry-username: {{ .Values.format.schemaRegistry.username | quote }} + schema-registry-password: {{ .Values.format.schemaRegistry.password | quote }} + {{- end }} + {{- end }} diff --git a/debezium-server/versions/1.1.0/templates/secret-entrypoint.yaml b/debezium-server/versions/1.1.0/templates/secret-entrypoint.yaml new file mode 100644 index 00000000..469c0503 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/secret-entrypoint.yaml @@ -0,0 +1,46 @@ +{{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} +kind: secret +name: {{ include "debezium.entrypoint.name" . }} +description: Debezium Server entrypoint script for PostgreSQL prerequisites +tags: + {{- include "debezium.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: | + #!/bin/bash + + set -o nounset + + echo "=== Debezium Server Entrypoint ===" + + # Find the PostgreSQL JDBC driver JAR (bundled with Debezium Server) + JDBC_JAR=$(find /debezium -name "postgresql-*.jar" -print -quit 2>/dev/null || true) + if [ -z "${JDBC_JAR}" ]; then + echo "WARNING: PostgreSQL JDBC driver not found. Skipping prerequisites." + exec /debezium/run.sh + fi + echo "Using JDBC driver: ${JDBC_JAR}" + + # Decode pre-compiled PgInit.class (Java 17+ compatible) + # Connects via JDBC, creates heartbeat table + failover replication slot + echo "yv66vgAAAD0AtQoAAgADBwAEDAAFAAYBABBqYXZhL2xhbmcvT2JqZWN0AQAGPGluaXQ+AQADKClWCQAIAAkHAAoMAAsADAEAEGphdmEvbGFuZy9TeXN0ZW0BAANlcnIBABVMamF2YS9pby9QcmludFN0cmVhbTsIAA4BAE5Vc2FnZTogUGdJbml0IDxob3N0PiA8cG9ydD4gPGRibmFtZT4gPHVzZXI+IDxwYXNzd29yZD4gPHNsb3ROYW1lPiA8cGx1Z2luTmFtZT4KABAAEQcAEgwAEwAUAQATamF2YS9pby9QcmludFN0cmVhbQEAB3ByaW50bG4BABUoTGphdmEvbGFuZy9TdHJpbmc7KVYKAAgAFgwAFwAYAQAEZXhpdAEABChJKVYSAAAAGgwAGwAcAQAXbWFrZUNvbmNhdFdpdGhDb25zdGFudHMBAEooTGphdmEvbGFuZy9TdHJpbmc7TGphdmEvbGFuZy9TdHJpbmc7TGphdmEvbGFuZy9TdHJpbmc7KUxqYXZhL2xhbmcvU3RyaW5nOwoAHgAfBwAgDAAhACIBABZqYXZhL3NxbC9Ecml2ZXJNYW5hZ2VyAQANZ2V0Q29ubmVjdGlvbgEATShMamF2YS9sYW5nL1N0cmluZztMamF2YS9sYW5nL1N0cmluZztMamF2YS9sYW5nL1N0cmluZzspTGphdmEvc3FsL0Nvbm5lY3Rpb247CQAIACQMACUADAEAA291dAgAJwEAGENvbm5lY3RlZCB0byBQb3N0Z3JlU1FMLgcAKQEAFWphdmEvc3FsL1NRTEV4Y2VwdGlvbgoAKAArDAAsAC0BAApnZXRNZXNzYWdlAQAUKClMamF2YS9sYW5nL1N0cmluZzsSAAEALwwAGwAwAQAnKElMamF2YS9sYW5nL1N0cmluZzspTGphdmEvbGFuZy9TdHJpbmc7EgACADIMABsAMwEAKChJSUxqYXZhL2xhbmcvU3RyaW5nOylMamF2YS9sYW5nL1N0cmluZzsFAAAAAAAAE4gKADcAOAcAOQwAOgA7AQAQamF2YS9sYW5nL1RocmVhZAEABXNsZWVwAQAEKEopVgcAPQEAHmphdmEvbGFuZy9JbnRlcnJ1cHRlZEV4Y2VwdGlvbgoANwA/DABAAEEBAA1jdXJyZW50VGhyZWFkAQAUKClMamF2YS9sYW5nL1RocmVhZDsKADcAQwwARAAGAQAJaW50ZXJydXB0CABGAQArRW5zdXJpbmcgZGViZXppdW1faGVhcnRiZWF0IHRhYmxlIGV4aXN0cy4uLgsASABJBwBKDABLAEwBABNqYXZhL3NxbC9Db25uZWN0aW9uAQAPY3JlYXRlU3RhdGVtZW50AQAWKClMamF2YS9zcWwvU3RhdGVtZW50OwgATgEAa0NSRUFURSBUQUJMRSBJRiBOT1QgRVhJU1RTIGRlYmV6aXVtX2hlYXJ0YmVhdCAoaWQgSU5URUdFUiBQUklNQVJZIEtFWSwgdHMgVElNRVNUQU1QIE5PVCBOVUxMIERFRkFVTFQgbm93KCkpCwBQAFEHAFIMAFMAVAEAEmphdmEvc3FsL1N0YXRlbWVudAEAB2V4ZWN1dGUBABUoTGphdmEvbGFuZy9TdHJpbmc7KVoIAFYBAFVJTlNFUlQgSU5UTyBkZWJleml1bV9oZWFydGJlYXQgKGlkLCB0cykgVkFMVUVTICgxLCBub3coKSkgT04gQ09ORkxJQ1QgKGlkKSBETyBOT1RISU5HCwBQAFgMAFkABgEABWNsb3NlBwBbAQATamF2YS9sYW5nL1Rocm93YWJsZQoAWgBdDABeAF8BAA1hZGRTdXBwcmVzc2VkAQAYKExqYXZhL2xhbmcvVGhyb3dhYmxlOylWCABhAQAWSGVhcnRiZWF0IHRhYmxlIHJlYWR5LhIAAwBjDAAbAGQBACYoTGphdmEvbGFuZy9TdHJpbmc7KUxqYXZhL2xhbmcvU3RyaW5nOwgAZgEAPVNFTEVDVCBjb3VudCgqKSBGUk9NIHBnX3JlcGxpY2F0aW9uX3Nsb3RzIFdIRVJFIHNsb3RfbmFtZSA9ID8LAEgAaAwAaQBqAQAQcHJlcGFyZVN0YXRlbWVudAEAMChMamF2YS9sYW5nL1N0cmluZzspTGphdmEvc3FsL1ByZXBhcmVkU3RhdGVtZW50OwsAbABtBwBuDABvAHABABpqYXZhL3NxbC9QcmVwYXJlZFN0YXRlbWVudAEACXNldFN0cmluZwEAFihJTGphdmEvbGFuZy9TdHJpbmc7KVYLAGwAcgwAcwB0AQAMZXhlY3V0ZVF1ZXJ5AQAWKClMamF2YS9zcWwvUmVzdWx0U2V0OwsAdgB3BwB4DAB5AHoBABJqYXZhL3NxbC9SZXN1bHRTZXQBAARuZXh0AQADKClaCwB2AHwMAH0AfgEABmdldEludAEABChJKUkSAAQAgAwAGwCBAQA4KExqYXZhL2xhbmcvU3RyaW5nO0xqYXZhL2xhbmcvU3RyaW5nOylMamF2YS9sYW5nL1N0cmluZzsSAAUAYxIABgBjCwBsAFgLAEgAWAgAhwEAF1ByZXJlcXVpc2l0ZXMgY29tcGxldGUuEgAHAGMIAIoBACJEZWJleml1bSBTZXJ2ZXIgd2lsbCBzdGFydCBhbnl3YXkuBwCMAQAGUGdJbml0AQAEQ29kZQEAD0xpbmVOdW1iZXJUYWJsZQEABG1haW4BABYoW0xqYXZhL2xhbmcvU3RyaW5nOylWAQANU3RhY2tNYXBUYWJsZQcAkwEAE1tMamF2YS9sYW5nL1N0cmluZzsHAJUBABBqYXZhL2xhbmcvU3RyaW5nAQAKU291cmNlRmlsZQEAC1BnSW5pdC5qYXZhAQAQQm9vdHN0cmFwTWV0aG9kcwgAmgEAF2pkYmM6cG9zdGdyZXNxbDovLwE6AS8BCACcAQAsRVJST1I6IENvdWxkIG5vdCBjb25uZWN0IGFmdGVyIAEgYXR0ZW1wdHM6IAEIAJ4BACUgIEF0dGVtcHQgAS8BIC0gcmV0cnlpbmcgaW4gNXMuLi4gKAEpCACgAQAwRW5zdXJpbmcgZmFpbG92ZXIgcmVwbGljYXRpb24gc2xvdCAnAScgZXhpc3RzLi4uCACiAQBAU0VMRUNUIHBnX2NyZWF0ZV9sb2dpY2FsX3JlcGxpY2F0aW9uX3Nsb3QoJwEnLCAnAScsIGZhbHNlLCB0cnVlKQgApAEAJkNyZWF0ZWQgZmFpbG92ZXIgcmVwbGljYXRpb24gc2xvdCAnAScuCACmAQAkUmVwbGljYXRpb24gc2xvdCAnAScgYWxyZWFkeSBleGlzdHMuCACoAQAlV0FSTklORzogUHJlcmVxdWlzaXRlIHNldHVwIGZhaWxlZDogAQ8GAKoKAKsArAcArQwAGwCuAQAkamF2YS9sYW5nL2ludm9rZS9TdHJpbmdDb25jYXRGYWN0b3J5AQCYKExqYXZhL2xhbmcvaW52b2tlL01ldGhvZEhhbmRsZXMkTG9va3VwO0xqYXZhL2xhbmcvU3RyaW5nO0xqYXZhL2xhbmcvaW52b2tlL01ldGhvZFR5cGU7TGphdmEvbGFuZy9TdHJpbmc7W0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL2ludm9rZS9DYWxsU2l0ZTsBAAxJbm5lckNsYXNzZXMHALEBACVqYXZhL2xhbmcvaW52b2tlL01ldGhvZEhhbmRsZXMkTG9va3VwBwCzAQAeamF2YS9sYW5nL2ludm9rZS9NZXRob2RIYW5kbGVzAQAGTG9va3VwACEAiwACAAAAAAACAAEABQAGAAEAjQAAAB0AAQABAAAABSq3AAGxAAAAAQCOAAAABgABAAAAAwAJAI8AkAABAI0AAARsAAQAEAAAAgIqvhAHogAPsgAHEg22AA8EuAAVKgMyTCoEMk0qBTJOKgYyOgQqBzI6BSoIMjoGKhAGMjoHKywtugAZAAA6CBAeNgkBOgoENgsVCxUJowBjGQgZBBkFuAAdOgqyACMSJrYAD6cATToMFQsVCaAAGbIABxUJGQy2ACq6AC4AALYADwO4ABWyACMVCxUJGQy2ACq6ADEAALYADxQANLgANqcACzoNuAA+tgBChAsBp/+csgAjEkW2AA8ZCrkARwEAOgsZCxJNuQBPAgBXGQsSVbkATwIAVxkLxgAqGQu5AFcBAKcAIDoMGQvGABYZC7kAVwEApwAMOg0ZDBkNtgBcGQy/sgAjEmC2AA+yACMZBroAYgAAtgAPGQoSZbkAZwIAOgsZCwQZBrkAawMAGQu5AHEBADoMGQy5AHUBAFcZDAS5AHsCAJoAWRkKuQBHAQA6DRkNGQYZB7oAfwAAuQBPAgBXGQ3GACoZDbkAVwEApwAgOg4ZDcYAFhkNuQBXAQCnAAw6DxkOGQ+2AFwZDr+yACMZBroAggAAtgAPpwAQsgAjGQa6AIMAALYADxkLxgAqGQu5AIQBAKcAIDoMGQvGABYZC7kAhAEApwAMOg0ZDBkNtgBcGQy/GQq5AIUBALIAIxKGtgAPpwAdOguyAAcZC7YAKroAiAAAtgAPsgAHEom2AA+xAAkATwBiAGUAKACYAJ4AoQA8AMAA1ADjAFoA6gDxAPQAWgFPAWABbwBaAXYBfQGAAFoBIAGpAbgAWgG/AcYByQBaAK8B5AHnACgAAgCOAAAAwgAwAAAABQAHAAYADwAHABMACQAfAAoAKQALADQADAA+AA4AQgAPAEUAEABPABIAWgATAGIAFABlABUAZwAWAG4AFwCAABgAhAAaAJgAGwCpABAArwAgALcAIQDAACIAygAjANQAJADjACEBAAAlAQgAJwEVACgBIAApASoAKgEzACsBOwAsAUYALQFPAC4BYAAvAW8ALQGMADABnAAyAakANAG4ACgB1QA2AdwANwHkADsB5wA4AekAOQH5ADoCAQA8AJEAAAFIABcT/wA0AAwHAJIHAJQHAJQHAJQHAJQHAJQHAJQHAJQHAJQBBwBIAQAAXAcAKPwAHgcAKFwHADz6AAf6AAX/ADMADAcAkgcAlAcAlAcAlAcAlAcAlAcAlAcAlAcAlAEHAEgHAFAAAQcAWv8AEAANBwCSBwCUBwCUBwCUBwCUBwCUBwCUBwCUBwCUAQcASAcAUAcAWgABBwBaCPkAAv8AbgAOBwCSBwCUBwCUBwCUBwCUBwCUBwCUBwCUBwCUAQcASAcAbAcAdgcAUAABBwBa/wAQAA8HAJIHAJQHAJQHAJQHAJQHAJQHAJQHAJQHAJQBBwBIBwBsBwB2BwBQBwBaAAEHAFoI+QACD/oADE4HAFr/ABAADQcAkgcAlAcAlAcAlAcAlAcAlAcAlAcAlAcAlAEHAEgHAGwHAFoAAQcAWgj5AAJRBwAoGQADAJYAAAACAJcAmAAAADIACACpAAEAmQCpAAEAmwCpAAEAnQCpAAEAnwCpAAEAoQCpAAEAowCpAAEApQCpAAEApwCvAAAACgABALAAsgC0ABk=" | base64 -d > /tmp/PgInit.class + + if [ $? -ne 0 ]; then + echo "WARNING: Failed to decode PgInit.class. Skipping prerequisites." + exec /debezium/run.sh + fi + + echo "Running PostgreSQL prerequisites..." + java -cp "${JDBC_JAR}:/tmp" PgInit \ + "${DB_HOSTNAME}" \ + "{{ include "debezium.databasePort" . }}" \ + "{{ .Values.source.database.name }}" \ + "${DB_USER}" \ + "${DB_PASSWORD}" \ + "{{ .Values.source.postgres.slotName }}" \ + "{{ .Values.source.postgres.pluginName }}" || echo "WARNING: Prerequisites script returned non-zero. Continuing anyway." + + echo "=== Starting Debezium Server ===" + exec /debezium/run.sh +{{- end }} diff --git a/debezium-server/versions/1.1.0/templates/volumeset.yaml b/debezium-server/versions/1.1.0/templates/volumeset.yaml new file mode 100644 index 00000000..b6c4a5c3 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/volumeset.yaml @@ -0,0 +1,15 @@ +{{- if eq (include "debezium.requiresVolumeset" .) "true" }} +kind: volumeset +name: {{ include "debezium.volumeset.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +description: Debezium Server data volumeset for offset and schema history storage +tags: + {{- include "debezium.tags" . | nindent 2 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.volumeset.capacity }} + performanceClass: {{ .Values.volumeset.performanceClass }} + snapshots: + createFinalSnapshot: true + retentionDuration: 7d +{{- end }} diff --git a/debezium-server/versions/1.1.0/templates/workload-debezium.yaml b/debezium-server/versions/1.1.0/templates/workload-debezium.yaml new file mode 100644 index 00000000..e71cc327 --- /dev/null +++ b/debezium-server/versions/1.1.0/templates/workload-debezium.yaml @@ -0,0 +1,221 @@ +kind: workload +name: {{ include "debezium.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +description: Debezium Server CDC workload +tags: + {{- include "debezium.tags" . | nindent 2 }} +spec: + {{- if eq (include "debezium.requiresVolumeset" .) "true" }} + type: stateful + {{- else }} + type: standard + {{- end }} + identityLink: //identity/{{ include "debezium.identity.name" . }} + containers: + - name: debezium-server + {{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} + command: /bin/bash + args: + - '-c' + - >- + cp /scripts/debezium-entrypoint.sh /tmp/ && chmod +x /tmp/debezium-entrypoint.sh && + /tmp/debezium-entrypoint.sh + {{- end }} + image: {{ .Values.image }} + inheritEnv: false + cpu: {{ .Values.resources.cpu | quote }} + memory: {{ .Values.resources.memory | quote }} + ports: + - number: 8080 + protocol: http + env: + # Database credentials from secret + - name: DB_HOSTNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.db-hostname' + - name: DB_USER + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.db-user' + - name: DB_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.db-password' + + {{- /* MongoDB connection string */}} + {{- if and (eq .Values.source.type "mongodb") .Values.source.mongodb.connectionString }} + - name: MONGODB_CONNECTION_STRING + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.mongodb-connection-string' + {{- end }} + + {{- /* Offset storage credentials */}} + {{- if eq .Values.source.offset.storage "redis" }} + - name: OFFSET_REDIS_ADDRESS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-redis-address' + {{- if .Values.source.offset.redis.password }} + - name: OFFSET_REDIS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-redis-password' + {{- end }} + {{- end }} + {{- if eq .Values.source.offset.storage "jdbc" }} + - name: OFFSET_JDBC_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-jdbc-url' + - name: OFFSET_JDBC_USER + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-jdbc-user' + - name: OFFSET_JDBC_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.offset-jdbc-password' + {{- end }} + + {{- /* Schema history storage credentials */}} + {{- if eq (include "debezium.requiresSchemaHistory" .) "true" }} + {{- if eq .Values.source.schemaHistory.storage "redis" }} + - name: SCHEMA_HISTORY_REDIS_ADDRESS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-redis-address' + {{- if .Values.source.schemaHistory.redis.password }} + - name: SCHEMA_HISTORY_REDIS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-redis-password' + {{- end }} + {{- end }} + {{- if eq .Values.source.schemaHistory.storage "jdbc" }} + - name: SCHEMA_HISTORY_JDBC_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-jdbc-url' + - name: SCHEMA_HISTORY_JDBC_USER + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-jdbc-user' + - name: SCHEMA_HISTORY_JDBC_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-history-jdbc-password' + {{- end }} + {{- end }} + + {{- /* Sink credentials */}} + {{- if eq .Values.sink.type "kafka" }} + - name: KAFKA_BOOTSTRAP_SERVERS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.kafka-bootstrap-servers' + {{- if .Values.sink.kafka.saslUsername }} + - name: KAFKA_SASL_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.kafka-sasl-username' + - name: KAFKA_SASL_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.kafka-sasl-password' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "redis" }} + - name: SINK_REDIS_ADDRESS + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.sink-redis-address' + {{- if .Values.sink.redis.password }} + - name: SINK_REDIS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.sink-redis-password' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "nats-jetstream" }} + - name: NATS_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.nats-url' + {{- if .Values.sink.nats.username }} + - name: NATS_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.nats-username' + - name: NATS_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.nats-password' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "http" }} + - name: HTTP_SINK_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-url' + {{- if eq .Values.sink.http.authType "basic" }} + - name: HTTP_SINK_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-username' + - name: HTTP_SINK_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-password' + {{- end }} + {{- if eq .Values.sink.http.authType "bearer" }} + - name: HTTP_SINK_BEARER_TOKEN + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.http-sink-bearer-token' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "pulsar" }} + - name: PULSAR_SERVICE_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.pulsar-service-url' + {{- if .Values.sink.pulsar.authToken }} + - name: PULSAR_AUTH_TOKEN + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.pulsar-auth-token' + {{- end }} + {{- end }} + + {{- if eq .Values.sink.type "eventhubs" }} + - name: EVENTHUBS_CONNECTION_STRING + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.eventhubs-connection-string' + {{- end }} + + {{- /* Schema registry credentials */}} + {{- if .Values.format.schemaRegistry.url }} + - name: SCHEMA_REGISTRY_URL + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-registry-url' + {{- if .Values.format.schemaRegistry.username }} + - name: SCHEMA_REGISTRY_USERNAME + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-registry-username' + - name: SCHEMA_REGISTRY_PASSWORD + value: 'cpln://secret/{{ include "debezium.credentials.name" . }}.schema-registry-password' + {{- end }} + {{- end }} + + volumes: + # Mount application.properties from opaque secret + - path: /debezium/config/application.properties + recoveryPolicy: retain + uri: cpln://secret/{{ include "debezium.config.name" . }}.payload + {{- if and (eq .Values.source.type "postgres") (gt (int .Values.source.postgres.heartbeatIntervalMs) 0) }} + # Mount entrypoint script for PostgreSQL prerequisites + - path: /scripts/debezium-entrypoint.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "debezium.entrypoint.name" . }}.payload + {{- end }} + {{- if eq (include "debezium.requiresVolumeset" .) "true" }} + # Mount data volume for offset and schema history storage + - path: /debezium/data + uri: cpln://volumeset/{{ include "debezium.volumeset.name" . }} + {{- end }} + + readinessProbe: + httpGet: + path: /q/health/ready + port: 8080 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + + livenessProbe: + httpGet: + path: /q/health/live + port: 8080 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + + defaultOptions: + # Debezium should run as a single instance for CDC consistency + autoscaling: + metric: disabled + minScale: 1 + maxScale: 1 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 60 + + {{- if eq (include "debezium.requiresVolumeset" .) "true" }} + securityOptions: + filesystemGroupId: 185 + {{- end }} + + firewallConfig: + external: + outboundAllowCIDR: + {{- toYaml .Values.firewall.external.outboundAllowCIDR | nindent 8 }} + internal: + inboundAllowType: {{ .Values.firewall.internal.inboundAllowType }} + {{- if .Values.firewall.internal.workloads }} + inboundAllowWorkload: + {{- toYaml .Values.firewall.internal.workloads | nindent 8 }} + {{- end }} diff --git a/debezium-server/versions/1.1.0/values.yaml b/debezium-server/versions/1.1.0/values.yaml new file mode 100644 index 00000000..9719f2d1 --- /dev/null +++ b/debezium-server/versions/1.1.0/values.yaml @@ -0,0 +1,219 @@ +# Debezium Server CDC Template +# Documentation: https://debezium.io/documentation/reference/stable/operations/debezium-server.html + +image: quay.io/debezium/server:3.0 + +resources: + cpu: 500m + memory: 512Mi + +# ============================================================================= +# Source Database Configuration +# ============================================================================= +source: + # Database type: postgres, mysql, mongodb, sqlserver, oracle + type: postgres + + # Database connection settings + database: + hostname: "" # Required: database hostname or IP + port: 5432 # Default port varies by type (postgres:5432, mysql:3306, mongodb:27017, sqlserver:1433, oracle:1521) + name: "" # Required: database name (or SID for Oracle) + user: "" # Required: database username + password: "" # Required: database password (stored in credentials secret) + + # Server name used as prefix for topic names + serverName: "dbserver1" + + # Tables to capture (comma-separated, e.g., "public.users,public.orders") + # Leave empty to capture all tables + tableIncludeList: "" + + # Tables to exclude (comma-separated) + tableExcludeList: "" + + # PostgreSQL-specific settings + postgres: + slotName: "debezium" # Replication slot name + publicationName: "dbz_publication" # Publication name + pluginName: "pgoutput" # Logical decoding plugin: pgoutput (default), decoderbufs + slotDropOnStop: false # Keep replication slot on stop (required for HA/failover) + heartbeatIntervalMs: 0 # Heartbeat interval in ms (0=disabled; set 5000 for HA) + heartbeatActionQuery: "" # SQL executed on heartbeat (e.g., "UPDATE debezium_heartbeat SET ts = now() WHERE id = 1") + + # MySQL-specific settings + mysql: + serverId: 85744 # Unique server ID for MySQL replication + includeSchemaChanges: true # Include DDL events + + # MongoDB-specific settings + mongodb: + connectionString: "" # Full connection string (overrides hostname/port) + replicaSet: "" # Replica set name + + # SQL Server-specific settings + sqlserver: + databaseNames: "" # Comma-separated database names to capture + snapshotMode: "initial" # Snapshot mode: initial, schema_only, initial_only + + # Oracle-specific settings + oracle: + pdbName: "" # Pluggable database name + logMiningStrategy: "online_catalog" # Log mining strategy: online_catalog, redo_log_catalog + + # Offset storage configuration + offset: + # Storage type: file, redis, jdbc + storage: file + + # Flush settings + flushIntervalMs: 10000 # How often offsets flush to storage (ms) + flushTimeoutMs: 60000 # Timeout for offset flush operations (ms) + + # File storage settings (requires volumeset) + file: + filename: "/debezium/data/offsets.dat" + + # Redis storage settings + redis: + address: "" # Redis address (e.g., redis.mygvc.cpln.local:6379) + key: "debezium:offsets" # Redis key for offsets + password: "" # Redis password (stored in credentials secret) + ssl: false # Enable SSL/TLS + + # JDBC storage settings + jdbc: + url: "" # JDBC URL (e.g., jdbc:postgresql://host:5432/dbname) + user: "" # JDBC username + password: "" # JDBC password (stored in credentials secret) + tableName: "debezium_offsets" # Table name for storing offsets + + # Schema history storage (required for MySQL and SQL Server) + schemaHistory: + # Storage type: file, redis, jdbc (only used for mysql/sqlserver) + storage: file + + # File storage settings + file: + filename: "/debezium/data/schema-history.dat" + + # Redis storage settings + redis: + address: "" + key: "debezium:schema-history" + password: "" + ssl: false + + # JDBC storage settings + jdbc: + url: "" + user: "" + password: "" + tableName: "debezium_schema_history" + + # Error retry configuration + errors: + retryDelayInitialMs: 300 # Initial retry delay (ms) + retryDelayMaxMs: 10000 # Max retry delay (ms) + maxRetries: -1 # Max retries (-1 = infinite) + +# ============================================================================= +# Sink Configuration +# ============================================================================= +sink: + # Sink type: kafka, redis, nats-jetstream, http, kinesis, pubsub, pulsar, eventhubs + type: kafka + + # Kafka sink settings + kafka: + bootstrapServers: "" # Required: Kafka bootstrap servers (e.g., kafka.mygvc.cpln.local:9092) + topic: "" # Topic prefix (events sent to {topic}.{table}) + securityProtocol: "PLAINTEXT" # Security protocol: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + saslMechanism: "" # SASL mechanism: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512 + saslUsername: "" # SASL username + saslPassword: "" # SASL password (stored in credentials secret) + + # Redis sink settings (Redis Streams) + redis: + address: "" # Required: Redis address (e.g., redis.mygvc.cpln.local:6379) + password: "" # Redis password (stored in credentials secret) + ssl: false # Enable SSL/TLS + streamName: "" # Stream name prefix (events sent to {streamName}.{table}) + + # NATS JetStream sink settings + nats: + url: "" # Required: NATS URL (e.g., nats://nats.mygvc.cpln.local:4222) + subject: "" # Subject prefix (events sent to {subject}.{table}) + username: "" # NATS username + password: "" # NATS password (stored in credentials secret) + + # HTTP sink settings (webhooks) + http: + url: "" # Required: HTTP endpoint URL + headers: {} # Additional headers (key-value pairs) + authType: "" # Auth type: none, basic, bearer + username: "" # Basic auth username + password: "" # Basic auth password (stored in credentials secret) + bearerToken: "" # Bearer token (stored in credentials secret) + + # AWS Kinesis sink settings (uses Universal Cloud Identity) + kinesis: + region: "" # Required: AWS region + streamName: "" # Required: Kinesis stream name + credentialsProvider: "default" # Use "default" for Universal Cloud Identity + # Cloud account for Universal Cloud Identity + cloudAccount: + enabled: false + name: "" # AWS cloud account name in Control Plane + + # GCP Pub/Sub sink settings (uses Universal Cloud Identity) + pubsub: + projectId: "" # Required: GCP project ID + topic: "" # Topic prefix (events sent to {topic}.{table}) + # Cloud account for Universal Cloud Identity + cloudAccount: + enabled: false + name: "" # GCP cloud account name in Control Plane + + # Apache Pulsar sink settings + pulsar: + serviceUrl: "" # Required: Pulsar service URL + topic: "" # Topic prefix + authPluginClassName: "" # Auth plugin class (e.g., org.apache.pulsar.client.impl.auth.AuthenticationToken) + authToken: "" # Auth token (stored in credentials secret) + + # Azure Event Hubs sink settings + eventhubs: + connectionString: "" # Required: Event Hubs connection string (stored in credentials secret) + hubName: "" # Required: Event Hub name + +# ============================================================================= +# Serialization Format +# ============================================================================= +format: + key: json # Key format: json, avro, protobuf + value: json # Value format: json, avro, protobuf + + # Schema registry settings (for avro/protobuf) + schemaRegistry: + url: "" # Schema registry URL + username: "" # Schema registry username + password: "" # Schema registry password (stored in credentials secret) + +# ============================================================================= +# Volumeset Configuration (for file-based offset/schema-history storage) +# ============================================================================= +volumeset: + capacity: 10 # Initial capacity in GiB (minimum 10) + performanceClass: general-purpose-ssd # Performance class: general-purpose-ssd, high-throughput-ssd + +# ============================================================================= +# Firewall Configuration +# ============================================================================= +firewall: + internal: + inboundAllowType: same-gvc # Options: none, same-gvc, same-org, workload-list + workloads: [] # Workload list for inbound access (when type is workload-list) + external: + outboundAllowCIDR: + - 0.0.0.0/0 # Allow all outbound by default (required for database connectivity) diff --git a/etcd/versions/1.4.0/templates/_helpers.tpl b/etcd/versions/1.4.0/templates/_helpers.tpl index 24caaa8a..e9646d75 100644 --- a/etcd/versions/1.4.0/templates/_helpers.tpl +++ b/etcd/versions/1.4.0/templates/_helpers.tpl @@ -39,14 +39,28 @@ etcd Volume Set Name {{/* Validation */}} {{/* -Validate replicas value - must be minimum 3 and an odd number +Validate replicas value - must be minimum 3 and odd (single-location), +or 1 when multi-location is configured (1 per location, locations provide the count) */}} {{- define "etcd.validateReplicas" -}} -{{- if lt (int .Values.replicas) 3 -}} -{{- fail "Error: .Values.replicas must be at least 3" -}} -{{- end -}} -{{- if eq (mod (int .Values.replicas) 2) 0 -}} -{{- fail "Error: .Values.replicas must be an odd number" -}} +{{- if .Values.global.locations -}} + {{- if ne (int .Values.replicas) 1 -}} + {{- fail "Error: .Values.replicas must be 1 when global.locations is set (1 replica per location)" -}} + {{- end -}} + {{- $locCount := len .Values.global.locations -}} + {{- if lt $locCount 3 -}} + {{- fail "Error: global.locations must have at least 3 entries for etcd quorum" -}} + {{- end -}} + {{- if eq (mod $locCount 2) 0 -}} + {{- fail "Error: global.locations must have an odd number of entries for etcd quorum" -}} + {{- end -}} +{{- else -}} + {{- if lt (int .Values.replicas) 3 -}} + {{- fail "Error: .Values.replicas must be at least 3" -}} + {{- end -}} + {{- if eq (mod (int .Values.replicas) 2) 0 -}} + {{- fail "Error: .Values.replicas must be an odd number" -}} + {{- end -}} {{- end -}} {{- end -}} @@ -70,10 +84,6 @@ helm.sh/chart: {{ include "etcd.chart" . }} app.cpln.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.cpln.io/managed-by: {{ .Release.Service }} -cpln/marketplace: "true" -cpln/marketplace-template: etcd -cpln/marketplace-template-version: {{ .Chart.Version }} -cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} {{- end }} {{/* @@ -82,4 +92,4 @@ Selector labels {{- define "etcd.selectorLabels" -}} app.cpln.io/name: {{ .Release.Name }} app.cpln.io/instance: {{ .Release.Name }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/etcd/versions/1.4.0/templates/secret.yaml b/etcd/versions/1.4.0/templates/secret.yaml index 9d31c0ce..205b818e 100644 --- a/etcd/versions/1.4.0/templates/secret.yaml +++ b/etcd/versions/1.4.0/templates/secret.yaml @@ -22,7 +22,23 @@ data: # Self FQDN for peer URLs SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.{{ .Values.global.cpln.gvc }}.cpln.local" - # Build initial cluster list based on replicas + {{- if .Values.global.locations }} + # Multi-location mode: build cluster from all locations (1 replica per location) + LOCATIONS=({{ range .Values.global.locations }}"{{ . }}" {{ end }}) + ETCD_NAME="${WORKLOAD_NAME}-${LOCATION}" + INITIAL_CLUSTER="" + for loc in "${LOCATIONS[@]}"; do + peer="replica-0.${WORKLOAD_NAME}.${loc}.{{ .Values.global.cpln.gvc }}.cpln.local" + entry="${WORKLOAD_NAME}-${loc}=http://${peer}:2380" + if [[ -z "$INITIAL_CLUSTER" ]]; then + INITIAL_CLUSTER="$entry" + else + INITIAL_CLUSTER="${INITIAL_CLUSTER},$entry" + fi + done + {{- else }} + # Single-location mode: build cluster from local replicas + ETCD_NAME="${WORKLOAD_NAME}-${REPLICA_INDEX}" INITIAL_CLUSTER="" for i in $(seq 0 $(({{ .Values.replicas }} - 1))); do peer="replica-${i}.${WORKLOAD_NAME}.${LOCATION}.{{ .Values.global.cpln.gvc }}.cpln.local" @@ -33,6 +49,7 @@ data: INITIAL_CLUSTER="${INITIAL_CLUSTER},$entry" fi done + {{- end }} # Determine cluster state if [ -d "/var/lib/etcd/member" ] && [ "$(ls -A /var/lib/etcd/member)" ]; then @@ -41,11 +58,12 @@ data: INITIAL_CLUSTER_STATE="new" fi - echo "Starting etcd with cluster state: $INITIAL_CLUSTER_STATE" + echo "Starting etcd node '${ETCD_NAME}' with cluster state: $INITIAL_CLUSTER_STATE" + echo "Initial cluster: $INITIAL_CLUSTER" # Run etcd exec etcd \ - --name "${WORKLOAD_NAME}-${REPLICA_INDEX}" \ + --name "${ETCD_NAME}" \ --data-dir /var/lib/etcd \ --listen-client-urls "http://0.0.0.0:2379" \ --advertise-client-urls "http://${SELF_FQDN}:2379" \ From fd15c8e0b573f9505e985a50654a616aabdfd111 Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Tue, 5 May 2026 15:27:00 -0400 Subject: [PATCH 27/58] redis: apply replication setting tweaks from 3.2.0 to 3.3.0. --- .../3.3.0/templates/secret-redis-config.yaml | 5 +- .../3.3.0/templates/workload-redis.yaml | 98 ++++++++++++++++++- .../3.3.0/templates/workload-sentinel.yaml | 11 +++ redis/versions/3.3.0/values.yaml | 34 +++++++ 4 files changed, 143 insertions(+), 5 deletions(-) diff --git a/redis/versions/3.3.0/templates/secret-redis-config.yaml b/redis/versions/3.3.0/templates/secret-redis-config.yaml index 7b35f60d..6f894191 100644 --- a/redis/versions/3.3.0/templates/secret-redis-config.yaml +++ b/redis/versions/3.3.0/templates/secret-redis-config.yaml @@ -9,4 +9,7 @@ data: save 900 1 save 300 10 save 60 10000 - appendonly yes \ No newline at end of file + appendonly yes + repl-backlog-size {{ .Values.redis.replication.backlogSize }} + repl-timeout {{ .Values.redis.replication.timeout }} + client-output-buffer-limit slave {{ .Values.redis.replication.slaveOutputBufferLimit }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/workload-redis.yaml b/redis/versions/3.3.0/templates/workload-redis.yaml index fd3549aa..c75e6b98 100644 --- a/redis/versions/3.3.0/templates/workload-redis.yaml +++ b/redis/versions/3.3.0/templates/workload-redis.yaml @@ -6,6 +6,11 @@ tags: {{- if .Values.redis.tags }} {{ toYaml .Values.redis.tags | indent 2 }} {{- end }} + # Sentinel discovery and replica startup must resolve `.:6379` even + # while the pod is still resyncing or marked NotReady by the replication-aware + # readiness probe. This tag exposes not-yet-Ready pods on the headless service so + # peers can reach them for replication and Sentinel can monitor them. + cpln/publishNotReadyAddresses: "true" {{- include "redis.tags" . | nindent 2 }} spec: type: stateful @@ -93,6 +98,34 @@ spec: fi done echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" + # Self-heal sentinel's view of the slave set. Without this, a + # scale-up race or a sentinel restart (which wipes CONFIG REWRITE + # state via the bootstrap config copy) can leave this replica + # invisible to sentinel and so orphaned at the next failover. + # SENTINEL RESET only refreshes sentinel's bookkeeping — no + # failover is triggered, no clients are disrupted. + ( + while true; do + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping >/dev/null 2>&1 && break + else + redis-cli -p $PORT ping >/dev/null 2>&1 && break + fi + sleep 2 + done + sleep 3 + IDX=0 + while [ $IDX -lt $SENTINEL_REPLICA_COUNT ]; do + S_HOST="${SENTINEL_BASE}-${IDX}.${SENTINEL_BASE}" + S_PORT=$((26380 + IDX)) + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + redis-cli -h $S_HOST -p $S_PORT --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true + else + redis-cli -h $S_HOST -p $S_PORT SENTINEL RESET mymaster >/dev/null 2>&1 || true + fi + IDX=$((IDX + 1)) + done + ) & {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} SENTINEL_HOST="{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" @@ -109,6 +142,29 @@ spec: echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } done echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" + # Self-heal sentinel's view of the slave set (see notes in the + # publicAccess branch above for rationale). + ( + while true; do + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + redis-cli -p 6379 --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping >/dev/null 2>&1 && break + else + redis-cli -p 6379 ping >/dev/null 2>&1 && break + fi + sleep 2 + done + sleep 3 + IDX=0 + while [ $IDX -lt {{ .Values.sentinel.replicas }} ]; do + S_HOST="replica-${IDX}.{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + redis-cli -h $S_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true + else + redis-cli -h $S_HOST -p 26379 SENTINEL RESET mymaster >/dev/null 2>&1 || true + fi + IDX=$((IDX + 1)) + done + ) & {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT {{- else }} SENTINEL_HOST="{{ include "redis.sentinel.name" . }}" @@ -125,6 +181,29 @@ spec: echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } done echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" + # Self-heal sentinel's view of the slave set (see notes in the + # publicAccess branch above for rationale). + ( + while true; do + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + redis-cli -p 6379 --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping >/dev/null 2>&1 && break + else + redis-cli -p 6379 ping >/dev/null 2>&1 && break + fi + sleep 2 + done + sleep 3 + IDX=0 + while [ $IDX -lt {{ .Values.sentinel.replicas }} ]; do + S_HOST="{{ include "redis.sentinel.name" . }}-${IDX}.{{ include "redis.sentinel.name" . }}" + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + redis-cli -h $S_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true + else + redis-cli -h $S_HOST -p 26379 SENTINEL RESET mymaster >/dev/null 2>&1 || true + fi + IDX=$((IDX + 1)) + done + ) & {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT {{- end }} fi @@ -146,10 +225,17 @@ spec: {{- else }} PORT=6379 {{- end }} - if [ ! -z "$CUSTOM_REDIS_PASSWORD" ]; then - redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping; - else - redis-cli -p $PORT ping; + rcli() { + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + redis-cli -p $PORT --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" "$@" + else + redis-cli -p $PORT "$@" + fi + } + rcli ping >/dev/null && \ + if [ "$(rcli role | head -1)" = "slave" ]; then + [ "$(rcli info replication | awk -F: '/master_link_status/{print $2}' | tr -d '\r')" = "up" ] && \ + [ "$(rcli info replication | awk -F: '/master_sync_in_progress/{print $2}' | tr -d '\r')" = "0" ] fi failureThreshold: 10 initialDelaySeconds: 10 @@ -197,6 +283,10 @@ spec: multiZone: enabled: false {{- end }} +{{- if .Values.redis.requestRetryPolicy }} + requestRetryPolicy: +{{ toYaml .Values.redis.requestRetryPolicy | indent 4 }} +{{- end }} {{- if .Values.redis.firewall }} firewallConfig: {{- if or (hasKey .Values.redis.firewall "external_inboundAllowCIDR") (hasKey .Values.redis.firewall "external_outboundAllowCIDR") }} diff --git a/redis/versions/3.3.0/templates/workload-sentinel.yaml b/redis/versions/3.3.0/templates/workload-sentinel.yaml index 6ef60f7c..b2282b2d 100644 --- a/redis/versions/3.3.0/templates/workload-sentinel.yaml +++ b/redis/versions/3.3.0/templates/workload-sentinel.yaml @@ -6,6 +6,13 @@ tags: {{- if .Values.sentinel.tags }} {{ toYaml .Values.sentinel.tags | indent 2 }} {{- end }} + # Currently a no-op: the sentinel readiness probe is a plain `redis-cli ping`, so + # pods become Ready as soon as the port answers and the headless service exposes + # them anyway. Set here as a hedge — if the probe is ever tightened (e.g. to + # require quorum visibility or a known master), peers must still resolve each + # other via `.` during cold start to form the quorum, otherwise + # they deadlock: NotReady because no peers, no peers because NotReady. + cpln/publishNotReadyAddresses: "true" {{- include "redis.tags" . | nindent 2 }} spec: type: stateful @@ -151,6 +158,10 @@ spec: multiZone: enabled: false {{- end }} +{{- if .Values.sentinel.requestRetryPolicy }} + requestRetryPolicy: +{{ toYaml .Values.sentinel.requestRetryPolicy | indent 4 }} +{{- end }} {{- if .Values.sentinel.firewall }} firewallConfig: {{- if or (hasKey .Values.sentinel.firewall "external_inboundAllowCIDR") (hasKey .Values.sentinel.firewall "external_outboundAllowCIDR") }} diff --git a/redis/versions/3.3.0/values.yaml b/redis/versions/3.3.0/values.yaml index 480f054e..737dda09 100644 --- a/redis/versions/3.3.0/values.yaml +++ b/redis/versions/3.3.0/values.yaml @@ -32,6 +32,30 @@ redis: # external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list env: [] tags: {} + # requestRetryPolicy: + # attempts: 2 + # retryOn: + # - connect-failure + # - refused-stream + # - unavailable + # - cancelled + # - resource-exhausted + # - retriable-status-codes + requestRetryPolicy: {} + # Replication tuning. See secret-redis-config.yaml for how these are rendered. + # backlogSize: sized for (peak write throughput × tolerable disconnect window). + # 1mb (Redis default) escalates any brief disconnect to a full RDB resync. 1gb + # covers ~5 minutes of disconnect at ~3MB/s of writes. + # timeout (seconds): bound on full-resync transfer + RDB load + heartbeat. + # 60s (Redis default) is too low for multi-GB datasets — master/slave drop the + # link mid-sync. 300s covers ~30GB at typical 1Gbps + load throughput. + # slaveOutputBufferLimit: " ". Default + # "256mb 64mb 60" can't sustain a full resync of a multi-GB dataset at high + # write rate — master kills the replica mid-stream. Bump for production loads. + replication: + backlogSize: 1gb + timeout: 300 + slaveOutputBufferLimit: "2gb 512mb 300" dataDir: /data persistence: enabled: false @@ -86,6 +110,16 @@ sentinel: # external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list env: [] tags: {} + # requestRetryPolicy: + # attempts: 2 + # retryOn: + # - connect-failure + # - refused-stream + # - unavailable + # - cancelled + # - resource-exhausted + # - retriable-status-codes + requestRetryPolicy: {} persistence: enabled: false volumes: From b087a9d4de0b29473e62d0b860ae71cdcacdaaa1 Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Wed, 6 May 2026 10:54:51 -0400 Subject: [PATCH 28/58] redis v3.2.0: scalingPolicy: Parallel --- redis/versions/3.2.0/templates/workload-redis.yaml | 7 +++++++ redis/versions/3.2.0/templates/workload-sentinel.yaml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/redis/versions/3.2.0/templates/workload-redis.yaml b/redis/versions/3.2.0/templates/workload-redis.yaml index ec85d083..722c04cb 100644 --- a/redis/versions/3.2.0/templates/workload-redis.yaml +++ b/redis/versions/3.2.0/templates/workload-redis.yaml @@ -151,6 +151,13 @@ spec: multiZone: enabled: false {{- end }} + # Parallel pod management. Default OrderedReady serializes replica boots, + # which forces sentinels and redis pods to wait for each peer to be Ready + # before the next starts and produces extended cold-start +sdown noise. + # Parallel boots all replicas at once; publishNotReadyAddresses keeps peer + # DNS resolvable during the simultaneous cold start so the cluster can form. + rolloutOptions: + scalingPolicy: Parallel {{- if .Values.redis.requestRetryPolicy }} requestRetryPolicy: {{ toYaml .Values.redis.requestRetryPolicy | indent 4 }} diff --git a/redis/versions/3.2.0/templates/workload-sentinel.yaml b/redis/versions/3.2.0/templates/workload-sentinel.yaml index b2282b2d..6373c186 100644 --- a/redis/versions/3.2.0/templates/workload-sentinel.yaml +++ b/redis/versions/3.2.0/templates/workload-sentinel.yaml @@ -158,6 +158,10 @@ spec: multiZone: enabled: false {{- end }} + # See workload-redis.yaml for rationale. Parallel boots all sentinels at + # once instead of waiting for each to be Ready in sequence. + rolloutOptions: + scalingPolicy: Parallel {{- if .Values.sentinel.requestRetryPolicy }} requestRetryPolicy: {{ toYaml .Values.sentinel.requestRetryPolicy | indent 4 }} From 486fc60834dbbbeb9f65f26dc94307c002e3daff Mon Sep 17 00:00:00 2001 From: Jakob Nagel Date: Wed, 6 May 2026 11:03:07 -0400 Subject: [PATCH 29/58] ESS version 1.3.5 --- ess/versions/1.5.0/Chart.yaml | 17 ++ ess/versions/1.5.0/README.md | 211 +++++++++++++++++++++ ess/versions/1.5.0/templates/_helpers.tpl | 39 ++++ ess/versions/1.5.0/templates/identity.yaml | 5 + ess/versions/1.5.0/templates/policy.yaml | 10 + ess/versions/1.5.0/templates/secret.yaml | 9 + ess/versions/1.5.0/templates/workload.yaml | 61 ++++++ ess/versions/1.5.0/values.yaml | 82 ++++++++ 8 files changed, 434 insertions(+) create mode 100644 ess/versions/1.5.0/Chart.yaml create mode 100644 ess/versions/1.5.0/README.md create mode 100644 ess/versions/1.5.0/templates/_helpers.tpl create mode 100644 ess/versions/1.5.0/templates/identity.yaml create mode 100644 ess/versions/1.5.0/templates/policy.yaml create mode 100644 ess/versions/1.5.0/templates/secret.yaml create mode 100644 ess/versions/1.5.0/templates/workload.yaml create mode 100644 ess/versions/1.5.0/values.yaml diff --git a/ess/versions/1.5.0/Chart.yaml b/ess/versions/1.5.0/Chart.yaml new file mode 100644 index 00000000..77148d6d --- /dev/null +++ b/ess/versions/1.5.0/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: ess +description: External Secret Syncer for Control Plane +type: application +version: 1.5.0 +appVersion: "1.3.5" + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2025-03-12" + lastModified: "2026-05-06" + category: "secrets" + createsGvc: false \ No newline at end of file diff --git a/ess/versions/1.5.0/README.md b/ess/versions/1.5.0/README.md new file mode 100644 index 00000000..1fcf5ace --- /dev/null +++ b/ess/versions/1.5.0/README.md @@ -0,0 +1,211 @@ +## External Secret Syncer (ESS) + +### Overview + +Creates an application that continuously syncs secrets from external providers into Control Plane secrets on a configurable schedule. Supported providers: **HashiCorp Vault**, **AWS Secrets Manager**, **AWS Parameter Store**, **Doppler**, **GCP Secret Manager**, **1Password**, and **1Password Connect**. + +--- + +### How It Works + +ESS runs as a workload on Control Plane. Your provider configuration and secrets list are stored in a Control Plane secret and mounted into the workload as `sync.yaml`. On startup, ESS schedules a polling loop for each configured secret. At each interval, it fetches the latest value from the external provider and creates or updates the corresponding Control Plane secret via the API. + +ESS tags every secret it manages with `syncer.cpln.io/source` (set to the workload path). This prevents two ESS instances from accidentally overwriting each other's secrets. An hourly cleanup job also deletes any Control Plane secrets that ESS owns but that have been removed from your `sync.yaml` config. + +--- + +### Patch Notes + +This version of ESS fixes a bug preventing the cleanup from running + +### Configuring `values.yaml` + +#### Top-level fields + +| Field | Description | +|---|---| +| `image` | The ESS container image. Do not change unless upgrading. | +| `resources.cpu` / `resources.memory` | Resource limits for the workload container. | +| `port` | Port for the ESS HTTP admin API (default: `3004`). Used for health checks and manual sync triggers. | +| `allowedIp` | List of CIDRs allowed to reach the ESS admin API externally. Replace the placeholder with your IP, or use `0.0.0.0/0` to allow all. | +| `essConfig` | The full sync configuration — providers and secrets (see below). | + +--- + +#### `essConfig.providers` + +Each provider entry requires a unique `name` and exactly one provider block. An optional `syncInterval` sets the default interval for all secrets using that provider. + +**Vault** +```yaml +- name: my-vault + vault: + address: https://my-vault.com:8200 # required + token: # required + syncInterval: 1m # optional — overrides global default +``` + +**AWS Parameter Store** +```yaml +- name: my-aws-ssm + awsParameterStore: + region: us-east-1 + accessKeyId: # optional if using an IAM-linked identity + secretAccessKey: # optional if using an IAM-linked identity +``` + +**AWS Secrets Manager** +```yaml +- name: my-aws-secrets-manager + awsSecretsManager: + region: us-east-1 + accessKeyId: + secretAccessKey: +``` + +**Doppler** +```yaml +- name: my-doppler + doppler: + accessToken: # use a Doppler service token (dp.st....) +``` + +**GCP Secret Manager** +```yaml +- name: my-gcp + gcpSecretManager: + projectId: 123456789876 + credentials: # optional — omit to use Application Default Credentials + clientEmail: + privateKey: +``` + +**1Password** +```yaml +- name: my-1password + onePassword: + serviceAccountToken: + integrationName: my-ess # optional + integrationVersion: 1.0.0 # optional +``` + +**1Password Connect** +```yaml +- name: my-1password-connect + onePasswordConnect: + serverURL: https://my-connect-server.example.com # required + token: # required +``` + +--- + +#### `essConfig.secrets` + +Each secret entry syncs one value (or a set of values) from a provider into a Control Plane secret. + +| Field | Description | +|---|---| +| `name` | Name of the Control Plane secret to create or update. | +| `provider` | Must match a provider `name` defined above. | +| `syncInterval` | Optional. Overrides the provider-level and global default for this specific secret. | + +Each secret must use exactly one of the following sync types: + +--- + +##### `opaque` — Single value (stored as a Control Plane `opaque` secret) + +Shorthand (path only, no fallback): +```yaml +- name: my-secret + provider: my-vault + opaque: /v1/secret/data/myapp +``` + +With options: +```yaml +- name: my-secret + provider: my-vault + opaque: + path: /v1/secret/data/myapp # path to fetch + parse: data.password # optional — extract a key from a JSON/YAML response + default: fallback-value # optional — used if fetch fails + encoding: base64 # optional — base64-decode the fetched value +``` + +> **Note:** If you use the shorthand form (`opaque: /some/path`) with no `default`, a fetch failure causes the sync to fail with no fallback. + +--- + +##### `dictionary` — Multiple values (stored as a Control Plane `dictionary` secret) + +Each key in the dictionary is fetched independently: +```yaml +- name: my-secret + provider: my-vault + dictionary: + PORT: + path: /v1/secret/data/app + parse: data.port + default: 5432 + PASSWORD: + path: /v1/secret/data/app + parse: data.password + USERNAME: + path: /v1/secret/data/app + parse: data.username + default: "no username" +``` + +Each key supports `path`, `parse`, `default`, and `encoding` — the same options as `opaque`. A failure on one key does not block others. + +--- + +##### `dictionaryFromProject` — Entire Doppler project (Doppler only) + +Syncs all secrets from a Doppler project+config in one operation, stored as a Control Plane `dictionary` secret: +```yaml +- name: my-doppler-config + provider: my-doppler + dictionaryFromProject: + path: my-project/dev # format: "project/config" — exactly two segments +``` + +> **Note:** `dictionaryFromProject` is only valid with a Doppler provider. Using it with any other provider causes ESS to exit at startup. + +--- + +#### Doppler Path Formats + +| Sync type | Path format | Example | +|---|---|---| +| `opaque` or `dictionary` key | `project/config/SECRET_NAME` | `my-app/production/DATABASE_URL` | +| `dictionaryFromProject` | `project/config` | `my-app/production` | + +--- + +#### Sync Interval Format + +Intervals use the format `hms`. All parts are optional but at least one is required. + +Examples: `10s`, `5m`, `1h`, `1h30m`, `1h30m10s` + +Priority (highest wins): +1. Secret-level `syncInterval` +2. Provider-level `syncInterval` +3. Global default (`300s`) + +--- + +### Important Notes + +- **Conflict protection:** If a Control Plane secret already exists and is managed by a different ESS instance, the sync for that secret will fail. Two ESS instances cannot manage the same secret. +- **Secret type changes:** Changing a secret from `opaque` to `dictionary` (or vice versa) causes ESS to delete the existing secret and recreate it. There is a brief window where the secret does not exist. +- **Cleanup:** ESS runs an hourly job that deletes Control Plane secrets it owns but that no longer appear in `sync.yaml`. Removing a secret from the config will eventually result in its deletion from Control Plane. +- **Doppler `parse`:** The `parse` field only works when the Doppler secret's value is JSON or YAML. Using `parse` on a plain string secret throws an error. +- **`sync.yaml` hot reload:** ESS watches its config file and automatically restarts when changes are detected (every ~5 seconds). No workload restart is needed after updating the config secret. + +### Resources + +- [ESS Documentation](https://docs.controlplane.com/template-catalog/templates/external-secret-syncer) +- [Image Source Code](https://github.com/controlplane-com/external-secret-syncer) \ No newline at end of file diff --git a/ess/versions/1.5.0/templates/_helpers.tpl b/ess/versions/1.5.0/templates/_helpers.tpl new file mode 100644 index 00000000..95668c35 --- /dev/null +++ b/ess/versions/1.5.0/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* Resource Naming */}} + +{{/* +ESS Workload Name +*/}} +{{- define "ess.name" -}} +{{- printf "%s-ess" .Release.Name }} +{{- end }} + +{{/* +ESS Identity Name +*/}} +{{- define "ess.identity.name" -}} +{{- printf "%s-ess-identity" .Release.Name }} +{{- end }} + +{{/* +ESS Policy Name +*/}} +{{- define "ess.policy.name" -}} +{{- printf "%s-ess-policy" .Release.Name }} +{{- end }} + +{{/* +ESS Secret Config Name +*/}} +{{- define "ess.secret.name" -}} +{{- printf "%s-ess-config" .Release.Name }} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels +*/}} +{{- define "ess.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/ess/versions/1.5.0/templates/identity.yaml b/ess/versions/1.5.0/templates/identity.yaml new file mode 100644 index 00000000..e7176ee7 --- /dev/null +++ b/ess/versions/1.5.0/templates/identity.yaml @@ -0,0 +1,5 @@ +kind: identity +gvc: {{ .Values.global.cpln.gvc }} +name: {{ include "ess.identity.name" . }} +description: ESS identity +tags: {{- include "ess.tags" . | nindent 4 }} diff --git a/ess/versions/1.5.0/templates/policy.yaml b/ess/versions/1.5.0/templates/policy.yaml new file mode 100644 index 00000000..cba2f1dd --- /dev/null +++ b/ess/versions/1.5.0/templates/policy.yaml @@ -0,0 +1,10 @@ +kind: policy +name: {{ include "ess.policy.name" . }} +description: ESS policy +bindings: + - permissions: + - manage + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "ess.identity.name" . }} +target: all +targetKind: secret diff --git a/ess/versions/1.5.0/templates/secret.yaml b/ess/versions/1.5.0/templates/secret.yaml new file mode 100644 index 00000000..764bc110 --- /dev/null +++ b/ess/versions/1.5.0/templates/secret.yaml @@ -0,0 +1,9 @@ +kind: secret +name: {{ include "ess.secret.name" . }} +description: ESS config +tags: {{- include "ess.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: | +{{- toYaml .Values.essConfig | nindent 4 }} \ No newline at end of file diff --git a/ess/versions/1.5.0/templates/workload.yaml b/ess/versions/1.5.0/templates/workload.yaml new file mode 100644 index 00000000..a4106066 --- /dev/null +++ b/ess/versions/1.5.0/templates/workload.yaml @@ -0,0 +1,61 @@ +kind: workload +name: {{ include "ess.name" . }} +description: External Secret Syncer +tags: {{- include "ess.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: ess + cpu: {{ .Values.resources.cpu | quote }} + image: {{ .Values.image }} + inheritEnv: false + memory: {{ .Values.resources.memory | quote }} + ports: + - number: {{ .Values.port }} + protocol: http + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + path: /about + port: {{ .Values.port }} + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumes: + - path: /usr/src/app/sync.yaml + recoveryPolicy: retain + uri: cpln://secret/{{ include "ess.secret.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 3 + metric: cpu + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: + {{- toYaml .Values.allowedIp | nindent 8 }} + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: none + inboundAllowWorkload: [] + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "ess.identity.name" . }} + loadBalancer: + direct: + enabled: false + ports: [] + supportDynamicTags: false diff --git a/ess/versions/1.5.0/values.yaml b/ess/versions/1.5.0/values.yaml new file mode 100644 index 00000000..e012170e --- /dev/null +++ b/ess/versions/1.5.0/values.yaml @@ -0,0 +1,82 @@ +image: ghcr.io/controlplane-com/cpln-build/external-secret-syncer:v1.3.5 + +resources: + cpu: 200m + memory: 256Mi + +port: 3004 + +allowedIp: + - 1.2.3.4 # Replace with your IP + +essConfig: + providers: + - name: my-vault + vault: + address: https://my-vault.com:8200 + token: + syncInterval: 1m + - name: my-aws-ssm + awsParameterStore: + region: us-east-1 + accessKeyId: # alternatively configure identity to natively use AWS permissions + secretAccessKey: # alternatively configure identity to natively use AWS permissions + # - name: my-aws-secrets-manager + # awsSecretsManager: + # region: us-east-1 + # accessKeyId: + # secretAccessKey: + # - name: my-1password + # onePassword: + # serviceAccountToken: + # integrationName: my-ess + # integrationVersion: 1.0.0 + # - name: my-1password-connect + # onePasswordConnect: + # serverURL: https://my-connect-server.example.com + # token: + # - name: my-doppler + # doppler: + # accessToken: + # - name: my-gcp + # gcpSecretManager: + # projectId: 123456789876 + # credentials: + # clientEmail: + # privateKey: + secrets: + - name: auth + provider: my-vault + syncInterval: 20s + dictionary: + PORT: + path: /v1/secret/data/app + parse: data.port + default: 5432 + PASSWORD: + path: /v1/secret/data/app + parse: data.password + USERNAME: + default: "no username" + path: /v1/secret/data/app + parse: data.username + - name: ssm + provider: my-aws + syncInterval: 20s + opaque: /example/app + # - name: secrets-manager + # provider: my-aws-secrets-manager + # dictionary: + # PASSWORD: + # path: /example/app + # parse: password + # - name: doppler-secret + # provider: my-doppler + # opaque: /project/config/SECRET_NAME + # - name: doppler-project + # provider: my-doppler + # dictionaryFromProject: + # path: project/config # syncs all secrets from a Doppler project+config + # - name: gcp + # provider: my-gcp + # opaque: database-password From 5cd45ae516d9cfce29382aab50a13127a2dfc0cc Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Wed, 6 May 2026 11:17:07 -0400 Subject: [PATCH 30/58] cdc-pipeline: add icon --- cdc-pipeline/icon.png | Bin 0 -> 2367 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cdc-pipeline/icon.png diff --git a/cdc-pipeline/icon.png b/cdc-pipeline/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..f89204d10abd7066d4b20fd44858daee51af33d9 GIT binary patch literal 2367 zcmV-F3BdM=P)op68k0oE10Te89*yTmTjTYjoRur`HQ--+agn0)qeu zFnfQ;j|Bh%vv3(?UZ_}Bueg_2jVgn_V?PEUHsZd0EPTu$7a(({?~_<;LiSMsFblxB z`O|)_*9$WO0D~$FdNAc4cc$DETJN}-005v0Lp(UYa?LT8XEa#|!6*T+dx6LZp!+oy zX0j)pQ(AJ|%mBP&5SXdL5cb%zsi}Yf00jW{jh}u~Ony2SfZ#O+2DKW_*;s4|0Kxw) zFvwB`0Sr+9$V_}LKY-wMB`_I@aPtH3k45F|(rsNDLf40>Z9XS#8UWxO>uGT%00c1% zDF~4dQ@H+SsQT`lkQo3NRJyoYFQ+HQ+yDTG%ITMPi=+TRSI}NLAs2w)!Y>?i zsdCR@P?bSz5&*!Uk^0JXXf~Mrbqk z4^@hW_JzX`8kig=ZRfknJzPg#h}52DzLyH{qvn?+W&J1!I*MSOfXX;r3y3|V&c>7Gr3OKSe~&M zVkDsotOLSiChN|wleuAt*&h#7GbBOh&%gY8s$rgVP8kfE?NJ3>2DKREFhnPk1f6R; zE~myGTQ)5Qjr6Dj3xKd0wT+2K}mJ0>ZKeYN6PaqYRc9)L>A1$~_TpXPIUqh zgC&M&Fes+Q{We_BTLu|a0>mscIY$?tXfX&mt#E;vzMFJTh5X{=_M06BO-O3-QdElr zfR_Y?nFBzmy=Msf$l}v(({bJG)MH{3MtcK#;Uc459bP37L&hEn)rIDz$hd zpWBiEf@32kC*)FMg^R28Y6L=CkrghSetB1m0hR{qkyIN5s11Mndz)8bm%Ep!!9G;Xog0ALm% zM68U26fLIRfl^?FwMbk?6##%h%zn9%x5DLw}_RQ+#uxJMr_XZF`?y}aJ3 zmw&A$0H`vA2GDSiKA`z<_fSvM{JUrWs%{bhm?f0^_^T^LUyB>wwSTM%%rQv6*+1IJ z8vBng{!wHI!=I6bgeuSklksTJfesW5Y1>?1nJzMf=}FL78FU$Be7^IHohP>b;&Bn4 zpczgmgDykpU+&rO{NE^dV|hkrhzBWEpvg?;on0q$o$%4cCk8{zrc{9@KoeiRzc;^f zzw~;nnxzUv`+$fR*BSJnM-^D}jZ@oeeM3pGbARAaDx@QePYp25_NW4`=NUvB=xVFI z|J`Zrt)#{l7hj*}995v{YjGz*i+lG6@|D4LzZ0B@YHBiWyIT6v~9Y9P-YViU?JU+7cGz8+yJ^MS%Waa?SeOzFW{h|Gzh;o;+!i{rF zw?bXw?VW$Jf*UZBR@e%(c-Qx4N&wiH6@ai0+H1v-ZI+axNz0<8K#La`BD914tM~VI z*y};^0`QMT0f5m3LD~uro>}^fwnbVzN#{F~0MNBpt2;4DOVPM;S7U_>A3gg%VwQxh z0001#4%%x)+CnYfR%nHb@x9H`0D_^>R%!9Ozy3p8r4^0|Tr~g?%LyvAxN0jLQ~8Pj zgmQvbyb_C6Yo1H(&F9Y@!svWDgeQ9XsOyQo}}DWY4L>Et_lEt z$4Hi{-QuYYlL9TCu+&Ty004ky`aZE4G^b*>cyw3;zqOcYK^8#LQu})F7nZ*pP71)r zPXd`qX9!nTi>LfHvc~?&%4qTQ*uhn0J^&jK%>H@@o^qd)(c($t;qIZ1uf-Q+w0P2Z zGWfH$cb`(Yc>#oe9D1YQ=-lidtubhycMmbU4@eu2b|36K+H;`F5XXY0m-TWuZG2OG lul-H6tN}O%U@i}Y{{p_koVI=bQ&9i_002ovPDHLkV1jDIN=5(x literal 0 HcmV?d00001 From 982e270fa80c23c7edf48afc0276ebfea39e4909 Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Wed, 6 May 2026 12:10:39 -0400 Subject: [PATCH 31/58] redis: liveness probe must be a TCP port check only --- .../3.2.0/templates/workload-redis.yaml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/redis/versions/3.2.0/templates/workload-redis.yaml b/redis/versions/3.2.0/templates/workload-redis.yaml index 722c04cb..5142d35e 100644 --- a/redis/versions/3.2.0/templates/workload-redis.yaml +++ b/redis/versions/3.2.0/templates/workload-redis.yaml @@ -110,6 +110,31 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 4 + # Explicit liveness probe — keep it permissive (just "is the process up?") + # so the platform doesn't kill a pod mid-resync. cpln defaults the liveness + # probe to whatever the readiness probe is, and our readiness probe + # intentionally returns failure during full resync (master_link_status:up + # AND master_sync_in_progress:0). Without this override, a slave doing a + # full resync would be killed before it could finish. + livenessProbe: + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + exec: + command: + - /bin/bash + - "-c" + - |- + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((6380 + POD_ID)) + exec 3<>/dev/tcp/127.0.0.1/$PORT + {{- else }} + tcpSocket: + port: 6379 + {{- end }} + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 inheritEnv: false ports: {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled (gt (.Values.redis.replicas | int) 0) }} From f346eedd2ae462ef7393cffe773817fd47746a51 Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Tue, 5 May 2026 15:27:50 -0400 Subject: [PATCH 32/58] postgres-ha: multi-dc support. --- .../versions/2.2.0/charts/etcd-1.4.0.tgz | Bin 0 -> 3634 bytes .../versions/2.2.0/templates/_helpers.tpl | 12 ++++++++ .../2.2.0/templates/secret-ha-proxy.yaml | 19 +++++++++++- .../2.2.0/templates/secret-startup.yaml | 29 ++++++++++++++++-- .../versions/2.2.0/values.yaml | 12 +++++++- 5 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 postgres-highly-available/versions/2.2.0/charts/etcd-1.4.0.tgz diff --git a/postgres-highly-available/versions/2.2.0/charts/etcd-1.4.0.tgz b/postgres-highly-available/versions/2.2.0/charts/etcd-1.4.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..cdc97d44a7d65d336b1c60dfbb8a50c65ece63d7 GIT binary patch literal 3634 zcmV-24$bi&iwFP!000001MM4YbK5pDpYx6$p? z^Z%OmtkG%g?>C#RPOJN@-l%u#t!Je3zCiiRKwX?+yuCBe9$F} zk$ga%i5D_aC1zO&9VQ;Ih>Qo+3lsRC7mhQq38Q{8K>~_LImSZ)v;Y!*@)-JP;_+~g z2o_I0hao2ppR#ziM`-Bc{E;_KV>t_`gmlaTgr-i+g;>M5s8%Y^K?!{giBy$BevUvC zOv?CD3SLxaH1IFU$U~WV!0U{sF|d*(03~{ricNNRFJl(@pbK%iyW1nDX)shGQGy7; z+CvZ}*>>58rhd{Rtv!GWKn5sh?z+3XI|h*e{4lxdM5)BqfA=$q0G&aT!_t7YgZ5y? z3Ut5%9?v9#H}Yp0jwX5Sk+Xw0JC({QPgsu}su7amA`T6aO`l(}7-WdL^u6%}^_L0| zc9op55&QvGgM#}YSM72~eB7QSNhEr;8g^AZUc=tnv{l`&@yH&hV3f59jom96v)Yct z95aUppgtEUEtaE}2lm6_kop(OGONifVseScdH~cEvn%TR+IGqIrJw<`$5ZEWkEq`~ z9>Z^-ba}<&Pd=xv?Sq9}?yRR_CMnepBIbA_&mmbi*}j^v5Cnlnw1iSp$r#WC0uZkE zD-RXthnaB-VvLBe&nc{7W(=54sprd0`x}4>qbO;;ai9#+=Z>;}kwy`Z6G0>$8zGdj z57YpN_2^nlYoIoi0LJd8w4JB+oPUlc&;LFcej>fggV4C+{I9p`ou%`?*={vA=l^3g z8iqWP+Z7HF0{%5&V{p;Eld%54h)T`FBtavR3vY> z3m(dV?A!IG-E3PG8hP(oEP!T@Oq=j$l>G@iyV|Khd;k$el*q0l=?!uRXZ|EM$3MDj za6(DUee%wy;Aledbv<>IqHpdIAcHOqp(;~DuTgDR>y@Wg!V@;R|3QM0k2-vqOz^(> zU$W^lKxH5@Kq0kJN$pE)$K0%|4y^n-uVB=X!E%Zp*Z9~E+dwV zvcTzkn0cJ(5YQr3V&B|UK6xQHa$_Lczs~1N(AbR-08sGa$`MmYQ^+bw>_?#A31nn0 z=1XlvARxlULrM8>&{M)jsgEZPm)RRQaU3SZCOL0UWW&gvBEwc~=!LZ*6%$Jj(Ri%k z6s#qq%Np82Qe8}gq(`o=`d%epg>(p5{hUJRi)^atUI6Jw0YLWTCWI3PY4TAx?GbrO z>VB0Tm5@?{tJGv%onJ&K6FtbGTHWunZi1<7!X57SNa93mr+-(!WBQc)X}}7^BAQHk zq&B5-&G&}2oY=8p+Jq_bI}Y{(3yT>lD>nhM+#ZP64i>|~A7*CS{SYGIYi|IhG$-h-)MahQgbhDT3$stce_rUl0 zRsZ<#tVce4EDrqu*(}I&X%J|Y0v5)87?L=i`uKhFwiN$eOqd_Rxmr!4uU8AW zBmQf*8_lKozXSi>#D9;`=JVPvIn(h5Ifd;Jj>&Efwqpf{)U|-w@i@*ZauoM19AXfJ z6#_q74@EIJgb}g65jKv(xp&5VMg^-P3ZiG}jh~}5kcTqSr}Ay|)*as^)NP?=-J6SJ zeP-WAf9YH=BWUW(P%lUW(vlZ0+;zz|*1LUXo4WJPUWLb`L1Zypm7cSbv?sDV z8U~OJc@<$s_p)EY+*8S01P6}3WT#9fE9sIEOzwo$5#s{AEzdhW3vg|N3|FG49azL- z;)r@av5w-HL)^X+a2fIFWv>u~g+q{yE7l-|mG4+NLa3+3Ut2Jzj{+dQm&6Cu_gj;n zdbf*B&Zfm%6j^0Ul6^ZJ=N)}A^Ndz!Ws|7TInBi4WG%}#r@ z{=2#Vdz5C|wyx|N)mO%Nk#}pktE~9DuGY0u1zr%6A<)Ux_Bwx#{AbBnM0 z6yUD?-`QV&|JUB{ZubA9w8j2csxRBxg`~4&V`~gwx~(fi_(Q#5lcjwY2c8g%x;S>m zY|5xF|HL59@gnMDbv`%c>zC!giWd!dJ~fcyHYslevK_{~j}A`Bi0lKlX)h`vvS=#eoCSxkL9O5=zNW($GW zVj`rGm~eA3tKLPu@NcVx>-{01j_P=skk9+=i8uhIH z4_m)0&;R!3{r{u1#rdyKxDp3YdRxBg2XwWasQxbx2h?*99?{RRRq%5Fgl3_`F-U+o z#aFKhB05vv;CEYYsONvnUV5l&VwMn_rJO`w#IU4QQPap-8sdF1pq!dAudFE*-w)#4 zlehhY^W(RtFSoVgI*w;+MB)Kb^ciIJ{7-tjB;gv=^dgf2iBvf85=w)c{y?I$-{Yy!zMSsVr|sjK$>V zvlAgl2S+EbE^zFnr59pv%?m8wRJY~@4zME=RQ1*qLH))GtZyl9?y8~dJLKo&O|ixw za2M=nn0mgOecT~sjgRpc;AN%og60P=w}I=JhGTXcBNi?V|HX}?>PZUE(Rp9%!de!M zofD0Eb98)qe13dza?wBed2oJo_R><_@*@ZCfYqQTGtNHz^W%;658`rj4-U#&Xr{hS z6@cYSv|ujXlX$k4eMI*XU+DL0wXHeQzvq;f#yLh5x#_^QEqvy(0L?Q_l&w-q~4YJ~N_0@e--rillI|`>A_qxwu8{RJDhS z-r5%IGpWPB+v}g#KfNQk%Ydg}x=>0ExZ}Mkp(mCZmFMS9ach-NpE8l;0 z>y1tR_ZZED6g|=cSDihs*(13xf-4pMxZP11QiKVT!{dmn%7t{uy?V7#sbnwL<;P_B zg1%8%JaKQnhNEfm%vnFQ;~t-=(YC)**2_t0wzNKvh)Bo z_uYX0=oOH0{oFe1+au;X+&xnGUN%>5g6RONzH`OTk9p|NP%!c-0>lLJX)QjlF?1k? zTgC+#Oo96K`@THmvQKSlg?Q?VCO@_DY}%%6+NN#Vrfu4$ZQ7=7+8?U@5AAXI+W=4i E0EJ0D2mk;8 literal 0 HcmV?d00001 diff --git a/postgres-highly-available/versions/2.2.0/templates/_helpers.tpl b/postgres-highly-available/versions/2.2.0/templates/_helpers.tpl index 08b97cfa..14c31511 100644 --- a/postgres-highly-available/versions/2.2.0/templates/_helpers.tpl +++ b/postgres-highly-available/versions/2.2.0/templates/_helpers.tpl @@ -130,6 +130,18 @@ Validate backup configuration - when backup is enabled, backup.provider must be {{- end }} +{{/* +Validate multi-location configuration +*/}} +{{- define "pg-ha.validateLocations" -}} +{{- if .Values.global.locations }} + {{- if lt (len .Values.global.locations) 3 }} + {{- fail "at least 3 locations are required for multi-DC deployment (Patroni recommendation)" }} + {{- end }} +{{- end }} +{{- end }} + + {{/* Labeling */}} {{/* diff --git a/postgres-highly-available/versions/2.2.0/templates/secret-ha-proxy.yaml b/postgres-highly-available/versions/2.2.0/templates/secret-ha-proxy.yaml index f0207abe..a03639bd 100644 --- a/postgres-highly-available/versions/2.2.0/templates/secret-ha-proxy.yaml +++ b/postgres-highly-available/versions/2.2.0/templates/secret-ha-proxy.yaml @@ -29,15 +29,32 @@ data: CFG="/tmp/haproxy.cfg" SERVERS="" + {{- if .Values.global.locations }} + # Multi-location mode: 1 replica per location, check all locations + {{- range $idx, $loc := .Values.global.locations }} + HOST="replica-0.${WORKLOAD}.{{ $loc }}.${GVC}.cpln.local" + SERVERS="${SERVERS} + server pg-{{ $loc }} ${HOST}:5432 check resolvers cpln_dns init-addr last,libc,none" + {{- end }} + {{- else }} + # Single-location mode: all replicas in local location i=0 while [ "${i}" -lt "${REPLICAS}" ]; do HOST="replica-${i}.${WORKLOAD}.${LOCATION}.${GVC}.cpln.local" SERVERS="${SERVERS} - server pg${i} ${HOST}:5432 check" + server pg${i} ${HOST}:5432 check resolvers cpln_dns init-addr last,libc,none" i=$((i + 1)) done + {{- end }} cat > "${CFG}" <> "$CONFIG_FILE" < Date: Tue, 5 May 2026 15:29:09 -0400 Subject: [PATCH 33/58] kafka: v4.0.0 --- kafka/RELEASES.md | 27 + kafka/versions/4.0.0/.helmignore | 1 + kafka/versions/4.0.0/Chart.yaml | 17 + kafka/versions/4.0.0/README.md | 169 ++++ .../4.0.0/charts/cpln-common-1.0.0.tgz | Bin 0 -> 680 bytes kafka/versions/4.0.0/templates/_helpers.tpl | 908 ++++++++++++++++++ kafka/versions/4.0.0/templates/domain.yaml | 64 ++ kafka/versions/4.0.0/templates/identity.yaml | 4 + kafka/versions/4.0.0/templates/kafbat-ui.yaml | 114 +++ .../4.0.0/templates/kafka-connectors.yaml | 224 +++++ .../4.0.0/templates/kafka-rest-proxy.yaml | 169 ++++ kafka/versions/4.0.0/templates/policy.yaml | 16 + .../secret-controller-configuration.yaml | 99 ++ .../versions/4.0.0/templates/secret-init.yaml | 184 ++++ .../4.0.0/templates/secret-secrets.yaml | 12 + .../versions/4.0.0/templates/volumesets.yaml | 31 + .../templates/workload-kafka-client.yaml | 46 + .../templates/workload-kafka-cluster.yaml | 353 +++++++ .../4.0.0/templates/workload-kafka-ui.yaml | 66 ++ kafka/versions/4.0.0/values.yaml | 503 ++++++++++ 20 files changed, 3007 insertions(+) create mode 100644 kafka/versions/4.0.0/.helmignore create mode 100644 kafka/versions/4.0.0/Chart.yaml create mode 100644 kafka/versions/4.0.0/README.md create mode 100644 kafka/versions/4.0.0/charts/cpln-common-1.0.0.tgz create mode 100644 kafka/versions/4.0.0/templates/_helpers.tpl create mode 100644 kafka/versions/4.0.0/templates/domain.yaml create mode 100644 kafka/versions/4.0.0/templates/identity.yaml create mode 100644 kafka/versions/4.0.0/templates/kafbat-ui.yaml create mode 100644 kafka/versions/4.0.0/templates/kafka-connectors.yaml create mode 100644 kafka/versions/4.0.0/templates/kafka-rest-proxy.yaml create mode 100644 kafka/versions/4.0.0/templates/policy.yaml create mode 100644 kafka/versions/4.0.0/templates/secret-controller-configuration.yaml create mode 100644 kafka/versions/4.0.0/templates/secret-init.yaml create mode 100644 kafka/versions/4.0.0/templates/secret-secrets.yaml create mode 100644 kafka/versions/4.0.0/templates/volumesets.yaml create mode 100644 kafka/versions/4.0.0/templates/workload-kafka-client.yaml create mode 100644 kafka/versions/4.0.0/templates/workload-kafka-cluster.yaml create mode 100644 kafka/versions/4.0.0/templates/workload-kafka-ui.yaml create mode 100644 kafka/versions/4.0.0/values.yaml diff --git a/kafka/RELEASES.md b/kafka/RELEASES.md index 9375966c..95e981e7 100644 --- a/kafka/RELEASES.md +++ b/kafka/RELEASES.md @@ -4,6 +4,33 @@ - **Kafka Cluster Parallel Scaling Policy**: Changed the default `scalingPolicy` for the Kafka cluster stateful workload from `OrderedReady` to `Parallel` +# Release Notes - Version 4.0.0 + +## What's New + +- **kafka-orchestrator sidecar for accurate readiness**: The Kafka cluster workload now runs `ghcr.io/controlplane-com/kafka-orchestrator` as a sidecar container. The sidecar exposes an HTTP `/health/ready` endpoint that validates broker registration, controller election, under-replicated partition count, and log-directory health using franz-go — a much stronger readiness signal than the previous TCP-socket check on port 9093. + - Sidecar readiness probe: `httpGet /health/ready` on port 8080 + - Prometheus metrics exposed at `/metrics` (cgroup memory and OOM-risk ratios) + - SASL credentials are wired automatically from the configured listener (default: `client`) + - The kafka container's existing TCP probes on port 9093 are preserved; workload readiness is now gated on both probes passing + - Configurable under the new `kafka_orchestrator:` section in `values.yaml`; set to `null` or comment out to disable the sidecar + +- **Graceful broker shutdown**: The kafka container's `terminationGracePeriodSeconds` is now exposed via `kafka.terminationGracePeriodSeconds` in `values.yaml` (default `600` seconds, up from the previous hardcoded `30`). Brokers carrying large amounts of data now have time to complete `controlled.shutdown` (leadership transfer + log flush) before SIGKILL. + +- **Init script signal propagation**: The kafka container's bash wrapper now `exec`s into `/tmp/kafka-init.sh`, which already `exec`s into the Kafka run script. PID 1 is now the Kafka JVM itself, so SIGTERM from Control Plane reaches the broker directly and triggers `controlled.shutdown` instead of being absorbed by the bash wrapper. + +- **Suppressed Control Plane's default preStop drain delay (all four containers)**: Control Plane's default container lifecycle injects a `preStop sleep $((terminationGracePeriodSeconds / 2))` on **every** container (the actuator's `getLifecycle` runs per-container in the `for...containers` loop in `workloadDeployment.ts:246-258`). For our 600s grace period that means a 300s idle preStop on each of `kafka`, `kafka-orchestrator`, `kafka-exporter`, and `jmx-exporter`. The drain delay is intended for L7 envoy/ingress connections, none of which apply to a kafka stateful workload — clients reconnect via Metadata refresh, inter-broker traffic is handled by `controlled.shutdown`'s leadership transfer, and the prometheus-scrape sidecars have no draining semantics. All four containers now declare an explicit no-op `preStop: exec: ['true']`, suppressing the default on each. Net effect: the entire pod terminates in seconds (bounded by the kafka container's `controlled.shutdown`), not 300s+ of useless sleep on three sidecars holding the pod hostage. + +- **`cpln/publishNotReadyAddresses=true` on the Kafka cluster workload**: Required so the headless Service exposes not-yet-Ready broker pods in DNS, which is what lets the KRaft controller quorum form on cold start (or after suspend/unsuspend). Earlier versions of the chart got away with this missing because the Kafka container's TCP probe on 9093 briefly flickered Ready every crash-loop iteration, just long enough to publish endpoints. The new kafka-orchestrator sidecar's `/health/ready` probe (correctly) requires actual cluster health, which closes that race — making the tag mandatory rather than optional. Without it, pods crash-loop with `UnknownHostException: etl-cluster-N.etl-cluster:9093`. + +- **Reliability and recovery defaults in `server.properties`**: The chart now emits the following defaults in the broker config (each can be overridden via `kafka.extra_configurations`): + - `default.replication.factor` — auto-derived as `min(3, kafka.replicas)`; clamps correctly when scaling below 3 replicas + - `min.insync.replicas` — auto-derived as `max(1, default.replication.factor - 1)` + - `controlled.shutdown.enable=true`, `controlled.shutdown.max.retries=3`, `controlled.shutdown.retry.backoff.ms=5000` — clean shutdown with retry on leadership-transfer failures + - `unclean.leader.election.enable=false` — never promote out-of-sync replicas; prevents data loss + - `num.recovery.threads.per.data.dir` — auto-derived as `8 * ceil(cores)` from `kafka.cpu` (e.g. `1000m` → 8, `2000m` → 16, `4` → 32). Recovery only runs after a *dirty* shutdown; a clean `controlled.shutdown` (now achievable thanks to the grace-period and signal-propagation fixes above) skips it entirely. + - `num.replica.fetchers=4` — faster follower replication so brokers rejoin the ISR quickly after transient outages + # Release Notes - Version 3.4.0 ## What's New diff --git a/kafka/versions/4.0.0/.helmignore b/kafka/versions/4.0.0/.helmignore new file mode 100644 index 00000000..7d101009 --- /dev/null +++ b/kafka/versions/4.0.0/.helmignore @@ -0,0 +1 @@ +values.yaml \ No newline at end of file diff --git a/kafka/versions/4.0.0/Chart.yaml b/kafka/versions/4.0.0/Chart.yaml new file mode 100644 index 00000000..a5569a4f --- /dev/null +++ b/kafka/versions/4.0.0/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: kafka +description: Kafka cluster app for Control Plane +type: application +version: 4.0.0 +appVersion: "3.9" + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2026-04-28" + lastModified: "2026-04-28" + category: "event-streaming" + createsGvc: false \ No newline at end of file diff --git a/kafka/versions/4.0.0/README.md b/kafka/versions/4.0.0/README.md new file mode 100644 index 00000000..47dc2aa5 --- /dev/null +++ b/kafka/versions/4.0.0/README.md @@ -0,0 +1,169 @@ +## Kafka App + +### How to connect to the cluster + +You can connect to Kafka from the same GVC in which it's deployed using the following methods: + +- To connect using the cluster's general address, use `{kafka-cluster-workload-name}:9092`. + +- To connect to a specific replica, use one of the following addresses based on the replica you wish to connect to: + - `{kafka-cluster-workload-name}-0.{kafka-cluster-workload-name}:9092` + - `{kafka-cluster-workload-name}-1.{kafka-cluster-workload-name}:9092` + - `{kafka-cluster-workload-name}-2.{kafka-cluster-workload-name}:9092` + +- If you're configuring your Kafka for external access, you'll need to provide a domain name for the public address of the listener you want to use. Prerequisites: + - Make sure the dedicated load balancer is enabled on the GVC. See [Configure Domain documentation](https://docs.controlplane.com/guides/configure-domain#dedicated-load-balancing). + - Make sure to register your [Apex domain](https://docs.controlplane.com/reference/domain#apex-domain-considerations) name with Control Plane and set up a DNS record for the Kafka public address CNAME with the canonical GVC endpoint in your DNS provider. + +### Test Kafka Cluster with Kafka Client + +1. To activate the Kafka client, make sure `kafka_client` is uncommented in your values file. If necessary, reinstall the chart with the command: + ```bash + cpln helm install kafka-dev -f values-example.yaml + ``` + +2. To connect to the `kafka-client` workload, navigate through the UI to the appropriate GVC and select the `kafka-client` workload. In the workload details, find and use the **Connect** feature to establish a connection, which can be done either via the UI or by utilizing the CLI command provided there. + +3. Once connected, you can write and consume messages through the `kafka-client` workload. If it's `PLAINTEXT`, producer and consumer configurations should be omitted below: + +```BASH +# Change to bin directory +cd /opt/kafka/bin + +# Create client.properties +echo "security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"your-admin-password\";" > ./client.properties + +# Produce messages to the 'controlplane' topic +kafka-console-producer.sh --bootstrap-server {kafka-cluster-workload-name}:9092 --topic controlplane --producer.config ./client.properties + +# Consume messages from the 'controlplane' topic +kafka-console-consumer.sh --bootstrap-server {kafka-cluster-workload-name}:9092 --topic controlplane --from-beginning --consumer.config ./client.properties +``` + +### Public Listener Domain Configuration + +When configuring Kafka for external access via a public listener, you can choose between two domain routing modes: + +#### **Direct Replica Routing Mode (Recommended)** + +The recommended approach with automatic replica endpoint generation: + +```yaml +kafka: + listeners: + public: + protocol: SASL_PLAINTEXT + name: PUBLIC + directReplicaRouting: + enabled: true + containerPort: 9095 # ports 9091, 9093 and 9094 are reserved + publicAddress: kafka.example.com + sasl: + users: "public-user" + passwords: "your-password" +``` + +**Behavior:** +- Single domain configuration with the specified container port +- DNS01 certificate challenge for automatic SSL +- Platform automatically generates replica-specific subdomains in format: `{replica-name}-{location}.{publicAddress}` +- Replica-aware routing reduces cross-zone traffic costs in multi-zone deployments +- Connection endpoints (auto-generated examples): + - `kafka-cluster-0-aws-us-east-1.kafka.example.com:9095` + - `kafka-cluster-1-aws-us-east-1.kafka.example.com:9095` + - `kafka-cluster-2-aws-us-east-1.kafka.example.com:9095` + +**Prerequisites for Direct Routing:** +- DNS provider must support CNAME records +- Create DNS records for each replica and the ACME challenge record: + 1. `CNAME kafka-cluster-0-aws-us-east-1.kafka.example.com → kafka-cluster--0.aws-us-east-1.controlplane.us` + 2. `CNAME kafka-cluster-1-aws-us-east-1.kafka.example.com → kafka-cluster--1.aws-us-east-1.controlplane.us` + 3. `CNAME kafka-cluster-2-aws-us-east-1.kafka.example.com → kafka-cluster--2.aws-us-east-1.controlplane.us` + 4. `CNAME _acme-challenge.kafka → _acme-challenge.cpln.app` (for certificate validation) + +#### **Multi-Port Routing** + +Each replica gets its own port. Not recommended for multi-zone clusters: + +```yaml +kafka: + listeners: + public: + protocol: SASL_PLAINTEXT + name: PUBLIC + publicAddress: kafka.example.com + sasl: + users: "public-user" + passwords: "your-password" +``` + +**Behavior:** +- Creates ports 3000, 3001, 3002 (one per replica) +- Each port routes to a specific replica +- Custom TLS cipher suites configuration +- Connection format: `kafka.example.com:3000`, `kafka.example.com:3001`, etc. +- **Note**: Not recommended for multi-zone deployments as cross-zone traffic charges may occur + +**Which Mode to Use:** +- Use **Direct Replica Routing** for new deployments that require automatic SSL with zone-aware routing and per-replica hostnames +- Avoid using **Multi-Port Routing** unless you have specific use cases or existing clients configured with port numbers (3000-300X) + +**Configuration Rules:** +- Cannot use both `publicAddress` and `directReplicaRouting.enabled: true` in the same listener +- When `directReplicaRouting.enabled: true`, both `containerPort` and `publicAddress` must be specified within the `directReplicaRouting` section +- Only one listener can have a public address configured across all listeners +- Direct Replica Routing automatically creates DNS entries in format: `{replica-name}-{location}.{publicAddress}:{containerPort}` + +### Enable Custom Encryption using AWS Key Management Service (KMS) + +Custom encryption for volumes can be configured by setting the values under `kafka.volumes.customEncryption`. + +A key must be created in AWS before proceeding with the template. + +In the values file, set `enabled` to `true` and add the proper `region` and `keyId`. + +**Important** - To finish configuring in AWS once the template is installed: + +1. Navigate in the console to the created volume +2. Click on `spec` +3. Follow the `AWS Custom Encryption Instructions` +4. Repeat for each encrypted volume created + +### Kafbat configuration example + +Full configuration Docs: https://ui.docs.kafbat.io/configuration/configuration-file + +```YAML +kafka: + clusters: + - name: "apache-kafka" + bootstrapServers: "kafka-dev-cluster.kafka-dev.cpln.local:9092" + kafkaConnect: + - name: kafka-dev-connect-connect-cluster + address: http://kafka-dev-connect-connect-cluster.kafka-dev.cpln.local:8083 + properties: + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "PLAIN" + sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"your-admin-password\";" + +management: + health: + ldap: + enabled: false + +auth: + type: "LOGIN_FORM" +spring: + security: + user: + name: "admin" + password: "adminPassword" + +server: + port: 8080 +``` + +### Release Notes +See [RELEASES.md](https://github.com/controlplane-com/templates/blob/main/kafka/RELEASES.md) diff --git a/kafka/versions/4.0.0/charts/cpln-common-1.0.0.tgz b/kafka/versions/4.0.0/charts/cpln-common-1.0.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..c41e09051d41e5bd4feeabac39714835119f385c GIT binary patch literal 680 zcmV;Z0$2SXiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PIyykJCO7?Kz)fWN9z^Q(X67eo}6z2RHyiRNCvxO(uzjV+Y$Q zRNejV$oW&UX_rM}#RBSF;yBNq_g*~Xq>?I3bXjUOT^HOqy62^wJZmtgGsqpLF_1Z%M{COg4Xm|tMoai0xk4zp`Gl^LJV9_+R?-s)6fZG`!ATC> zzQXSkc74(rWt0UkZTTM+!}>ouI=wAWR%k4B)id;hC+dF|MaxnBN9=b9wq%YG-x(C970Gj6AFP0 z&KPZw1i}td8KGPTHP5^TyLQ@p;TxH0@c6tp6Ras;d7ZBdy0S z>!qH6@8EN--B|W;eaRKiF%7r-hi+}q>_pP6^w8 replicas, so we clamp to the cluster size. +*/}} +{{- define "kafka.defaultReplicationFactor" -}} +{{- $replicas := .Values.kafka.replicas | int -}} +{{- if lt $replicas 3 -}}{{ $replicas }}{{- else -}}3{{- end -}} +{{- end }} + +{{/* +Sensible default for min.insync.replicas: max(1, defaultReplicationFactor - 1). With +replication.factor=3 this gives 2 (tolerates one broker loss without unavailability and +without losing acked writes); with replicas=2 it gives 1; with replicas=1 it gives 1. +*/}} +{{- define "kafka.minInsyncReplicas" -}} +{{- $rf := include "kafka.defaultReplicationFactor" . | int -}} +{{- if le $rf 1 -}}1{{- else -}}{{ sub $rf 1 }}{{- end -}} +{{- end }} + +{{- define "kafka.validateListenerConfig" -}} + {{- if not .name -}} + {{- fail "Error: 'name' must be provided for the listener" -}} + {{- end -}} + {{- if not .protocol -}} + {{- fail "Error: 'protocol' must be provided for the listener" -}} + {{- end -}} + {{- $hasValidConfig := or .publicAddress .containerPort (and .directReplicaRouting .directReplicaRouting.enabled .directReplicaRouting.containerPort .directReplicaRouting.publicAddress) -}} + {{- if not $hasValidConfig -}} + {{- fail "Error: At least one of 'publicAddress', 'containerPort', or valid 'directReplicaRouting' (with enabled: true, containerPort, and publicAddress) must be provided for the listener" -}} + {{- end -}} + {{- if and .publicAddress .containerPort -}} + {{- fail "Error: When publicAddress is set for the listener, containerPort should not be specified as it will be automatically set to port range 3000-3004" -}} + {{- end -}} + {{- if .containerPort -}} + {{- $port := .containerPort | printf "%s" }} + {{- if or (eq $port "9091") (eq $port "9093") (eq $port "9094") -}} + {{- fail "Error: containerPort cannot be 9091, 9093, or 9094 for listener" -}} + {{- end -}} + {{- end -}} + {{- if and .directReplicaRouting .directReplicaRouting.enabled -}} + {{- if .directReplicaRouting.containerPort -}} + {{- $port := .directReplicaRouting.containerPort | printf "%s" }} + {{- if or (eq $port "9091") (eq $port "9093") (eq $port "9094") -}} + {{- fail "Error: directReplicaRouting.containerPort cannot be 9091, 9093, or 9094 for listener" -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- define "kafka.validateAdminExists" -}} +{{- $adminFound := false -}} +{{- $saslPlaintextExists := false -}} +{{- range .Values.kafka.listeners -}} + {{- if eq .protocol "SASL_PLAINTEXT" -}} + {{- $saslPlaintextExists = true -}} + {{- if and .sasl .sasl.admin -}} + {{- $adminFound = true -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- if and $saslPlaintextExists (not $adminFound) -}} + {{- fail "Error: At least one SASL_PLAINTEXT listener must have an admin user configured in sasl.admin" -}} +{{- end -}} +{{- end -}} + +{{- define "kafka.validateAuthConfig" -}} +{{- if eq .protocol "SASL_PLAINTEXT" -}} + {{- if not .sasl -}} + {{- fail (printf "Error: SASL_PLAINTEXT protocol requires sasl configuration to be enabled for listener '%s'" .name) -}} + {{- else if not .sasl.users -}} + {{- fail (printf "Error: SASL_PLAINTEXT protocol requires at least one user to be defined in sasl.users for listener '%s'" .name) -}} + {{- else -}} + {{- $userCount := len (splitList "," .sasl.users) -}} + {{- if not .sasl.passwords -}} + {{- fail (printf "Error: sasl.passwords must be provided when sasl.users is defined for listener '%s'" .name) -}} + {{- else -}} + {{- $passwordCount := len (splitList "," .sasl.passwords) -}} + {{- if ne $userCount $passwordCount -}} + {{- fail (printf "Error: Number of users (%d) does not match number of passwords (%d) for listener '%s'" $userCount $passwordCount .name) -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{- define "kafka.validateReplicas" -}} +{{- $replicas := .Values.kafka.replicas | int }} +{{- if or (gt $replicas 5) (eq $replicas 2) -}} + {{- fail "Invalid value for kafka.replicas. It must be less than or equal to 5 and not equal to 2." -}} +{{- end -}} +{{- end -}} + +{{- define "kafka.validateOnePublicAddress" -}} +{{- $publicAddressCount := 0 -}} +{{- range .Values.kafka.listeners }} + {{- if .publicAddress }} + {{- $publicAddressCount = add $publicAddressCount 1 -}} + {{- end }} + {{- if and .directReplicaRouting .directReplicaRouting.enabled .directReplicaRouting.publicAddress }} + {{- $publicAddressCount = add $publicAddressCount 1 -}} + {{- end }} +{{- end }} +{{- if gt $publicAddressCount 1 -}} + {{- fail "There must be at most one listener with a publicAddress set (either listener.publicAddress or listener.directReplicaRouting.publicAddress)." -}} +{{- end }} +{{- end -}} + +{{- define "kafka.validatedirectReplicaRoutingConfig" -}} +{{- range $key, $listener := .Values.kafka.listeners }} + {{- if and $listener.publicAddress $listener.directReplicaRouting }} + {{- if $listener.directReplicaRouting.enabled }} + {{- fail (printf "Error in listener '%s': Cannot have both 'publicAddress' at listener level and 'directReplicaRouting.enabled: true'. Use either legacy mode (publicAddress only) or new mode (directReplicaRouting with enabled: true)." $key) -}} + {{- end }} + {{- end }} + {{- if and $listener.directReplicaRouting $listener.directReplicaRouting.enabled }} + {{- if not $listener.directReplicaRouting.publicAddress }} + {{- fail (printf "Error in listener '%s': When directReplicaRouting.enabled is true, directReplicaRouting.publicAddress must be specified." $key) -}} + {{- end }} + {{- if not $listener.directReplicaRouting.containerPort }} + {{- fail (printf "Error in listener '%s': When directReplicaRouting.enabled is true, directReplicaRouting.containerPort must be specified." $key) -}} + {{- end }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kafka.validateKafkaImage" -}} +{{- $image := .Values.kafka.image -}} +{{- if contains "bitnami" $image -}} + {{- fail (printf "Error: This chart does not support Bitnami images, please use Apache Kafka images instead. Current value: %s" $image) -}} +{{- end -}} +{{- end -}} + +{{- define "kafka.validateImage" -}} +{{- $image := .image -}} +{{- if contains "bitnami" $image -}} + {{- fail (printf "Error: This chart does not support Bitnami images, please use Apache Kafka images instead. Current value: %s" $image) -}} +{{- end -}} +{{- end -}} + +{{- define "kafka.clientBootstrapAddress" -}} +{{- $clusterName := include "kafka.clusterName" . -}} +{{- $bootstrapAddress := "" -}} +{{- $listenerName := "" -}} + +{{- if .listenerName -}} + {{- $listenerName = .listenerName -}} +{{- else if .Values.kafka_connectors -}} + {{- range .Values.kafka_connectors -}} + {{- if .listener -}} + {{- $listenerName = .listener -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- if $listenerName -}} + {{- if hasKey .Values.kafka.listeners $listenerName -}} + {{- $listener := index .Values.kafka.listeners $listenerName -}} + {{- if $listener.publicAddress -}} + {{- $bootstrapAddress = printf "%s:3000" $listener.publicAddress -}} + {{- else -}} + {{- $containerPort := $listener.containerPort | int -}} + {{- $bootstrapAddress = printf "%s:%d" $clusterName $containerPort -}} + {{- end -}} + {{- else -}} + {{- $bootstrapAddress = include "kafka.bootstrapAddress" . -}} + {{- end -}} +{{- else -}} + {{- $bootstrapAddress = include "kafka.bootstrapAddress" . -}} +{{- end -}} + +{{- $bootstrapAddress -}} +{{- end -}} + +{{- define "kafka.propertiesMapToList" -}} +{{- range $key, $value := . -}} +{{ $key }}={{ $value }} +{{- end -}} +{{- end -}} + + + +{{- define "kafka.connectors.download.script" -}} +#!/bin/sh +set -e{{- if .verbose }}x{{- end }} + +download_file() { + local url=$1 + local output_file=$2 + + if echo "$url" | grep -q "@.*jfrog"; then + echo "Handling JFrog redirect for: $url" + local redirect_url=$(wget -S --spider "$url" 2>&1 | grep 'Location:' | awk '{print $2}') + if [ -n "$redirect_url" ]; then + echo "Downloading from redirect URL..." + wget -q "$redirect_url" -O "$output_file" + else + echo "Failed to get redirect URL, trying direct download..." + wget -q "$url" -O "$output_file" + fi + else + wget -q "$url" -O "$output_file" + fi +} + +# Function to download and extract artifacts +download_and_extract() { + local artifact_type=$1 + local artifact_url=$2 + local plugin_path=$3 + local plugin_name=$4 + local temp_dir=$(mktemp -d) + + echo "Downloading artifact from $artifact_url" + + if [ "$artifact_type" == "jar" ]; then + # For jar files, create a directory for the plugin if it doesn't exist + local plugin_dir="$plugin_path/$plugin_name" + mkdir -p "$plugin_dir" + + # Download jar file to the plugin-specific directory + download_file "$artifact_url" "$plugin_dir/${plugin_name}.jar" + echo "Downloaded JAR file to $plugin_dir/${plugin_name}.jar" + else + # For archives, download to temp dir and extract + local archive_file="$temp_dir/archive.${artifact_type}" + download_file "$artifact_url" "$archive_file" + + echo "Extracting $artifact_type archive to $plugin_path" + case "$artifact_type" in + "tgz"|"tar.gz") + tar -xzf "$archive_file" -C "$plugin_path" + ;; + "tar") + tar -xf "$archive_file" -C "$plugin_path" + ;; + "zip") + unzip -o "$archive_file" -d "$plugin_path" + ;; + *) + echo "Unsupported archive type: $artifact_type" + ;; + esac + + rm -rf "$temp_dir" + fi +} +# Process each Kafka connector +echo "Setting up Kafka connector plugins" + +# Download and extract artifacts for each enabled plugin +{{- range .plugins }} +{{- if eq .enabled true }} +echo "Processing plugin: {{ .name }}" +{{- $pluginName := .name }} +{{- range .artifacts }} +download_and_extract "{{ .type }}" "{{ .url }}" "{{ $.plugins_folder }}" "{{ $pluginName }}" +{{- end }} +{{- else }} +echo "Skipping disabled plugin: {{ .name }}" +{{- end }} +{{- end }} + +echo "All Kafka connector plugins have been downloaded and extracted." +echo "Sleeping..." +sleep infinity +{{- end }} + +{{- define "kafka.connectors.run.script" -}} +#!/bin/bash +set -e{{- if .verbose }}x{{- end }} + +# Function to create or update a connector +create_or_update_connector() { + local connector_name=$1 + local config=$2 + local cluster_connectors=$3 + + echo "Checking if connector $connector_name exists in cluster..." + + # Check if connector exists in the cluster-wide list (reliable in distributed mode) + local exists=false + if echo "$cluster_connectors" | grep -q "\"$connector_name\""; then + exists=true + echo "Connector $connector_name found in cluster list" + else + echo "Connector $connector_name does not exist in cluster" + fi + + if [ "$exists" = true ]; then + echo "Connector $connector_name exists. Updating configuration..." + + # Extract just the config part from the full connector JSON + # Remove the "name" line, remove everything up to and including "config": {, remove last two lines (both closing braces) + local config_content=$(echo "$config" | sed '/^[[:space:]]*"name":/d' | sed '1,/^[[:space:]]*"config":[[:space:]]*{/d' | sed '$d' | sed '$d') + local update_config="{${config_content}}" + + echo "Updating connector $connector_name with config:" + echo "$update_config" + + # Update the connector using PUT with nc (BusyBox wget doesn't support PUT) + local content_length=$(echo -n "$update_config" | wc -c | xargs) + local response + response=$(echo -e "PUT /connectors/$connector_name/config HTTP/1.1\r\nHost: localhost:8083\r\nContent-Type: application/json\r\nContent-Length: $content_length\r\nConnection: close\r\n\r\n$update_config" | nc localhost 8083 2>&1) + echo "HTTP response for $connector_name update: $(echo "$response" | head -1)" + if ! echo "$response" | grep -q "HTTP/1\.. 2"; then + echo "ERROR: Failed to update connector $connector_name. Response: $(echo "$response" | head -5)" + else + echo "Connector $connector_name updated successfully" + fi + else + echo "Connector $connector_name does not exist. Creating..." + + echo "Creating connector $connector_name with config:" + echo "$config" + + # Create the connector using POST (this may also update if connector exists) + wget -q -O /dev/null "http://localhost:8083/connectors" \ + --header="Content-Type: application/json" \ + --post-data="$config" + + echo "Connector $connector_name created successfully" + + # Add a delay after creation to allow connector to initialize + sleep 2 + fi +} + +# Function to check and setup truststore for SSL connections +truststore_init() { + local hostname=$1 + local port=$2 + local alias=$3 + local jdbc_props=$4 + + echo "Setting up truststore for $hostname:$port with alias $alias" + + # Parse JDBC connection properties + local truststore_path + local truststore_password + + # First check if ssl.truststore.location is provided in the config + if [[ -n "${SSL_TRUSTSTORE_LOCATION}" ]]; then + truststore_path="${SSL_TRUSTSTORE_LOCATION}" + echo "Using ssl.truststore.location from config: $truststore_path" + # Then check if it's in JDBC properties + elif [[ "$jdbc_props" =~ ssl\.truststore\.location=([^;]+) ]]; then + truststore_path="${BASH_REMATCH[1]}" + echo "Using ssl.truststore.location from JDBC properties: $truststore_path" + elif [[ "$jdbc_props" =~ trustStorePath=([^;]+) ]]; then + truststore_path="${BASH_REMATCH[1]}" + echo "Using trustStorePath from JDBC properties: $truststore_path" + else + # Use default path if not specified + truststore_path="/tmp/kafka.client.truststore.jks" + echo "No truststore path specified, using default: $truststore_path" + fi + + # Check if ssl.truststore.password is provided + if [[ -n "${SSL_TRUSTSTORE_PASSWORD}" ]]; then + truststore_password="${SSL_TRUSTSTORE_PASSWORD}" + echo "Using ssl.truststore.password from config" + # Then check if it's in JDBC properties + elif [[ "$jdbc_props" =~ ssl\.truststore\.password=([^;]+) ]]; then + truststore_password="${BASH_REMATCH[1]}" + echo "Using ssl.truststore.password from JDBC properties" + elif [[ "$jdbc_props" =~ trustStorePassword=([^;]+) ]]; then + truststore_password="${BASH_REMATCH[1]}" + echo "Using trustStorePassword from JDBC properties" + else + # Generate random password if not specified + truststore_password=$(openssl rand -base64 12) + export SSL_TRUSTSTORE_PASSWORD="${truststore_password}" + echo "Generated random ssl.truststore.password: ${truststore_password}" + fi + + # Create certs directory if it doesn't exist + mkdir -p $(dirname "$truststore_path") + + # Download CA certificate + echo "Downloading CA certificate for $hostname:$port" + echo | openssl s_client -connect $hostname:$port -showcerts 2>/dev/null | \ + openssl x509 -outform PEM > $(dirname "$truststore_path")/$alias.pem || \ + echo "WARNING: Failed to download certificate from $hostname:$port, truststore may be incomplete" + + # Create truststore if it doesn't exist or override existing one + echo "Creating new truststore from $JAVA_HOME/lib/security/cacerts" + if [[ -f "$JAVA_HOME/lib/security/cacerts" ]]; then + cp "$JAVA_HOME/lib/security/cacerts" "$truststore_path" || \ + echo "WARNING: Failed to copy cacerts to $truststore_path, truststore setup may be incomplete" + else + echo "WARNING: $JAVA_HOME/lib/security/cacerts not found, skipping truststore creation for $hostname" + return 0 + fi + + # Change the default password to our password + echo "Setting truststore password" + keytool -storepasswd -keystore "$truststore_path" \ + -storepass "changeit" -new "${truststore_password}" || \ + echo "WARNING: Failed to change truststore password for $hostname, continuing with default password" + + # Import certificate into truststore (only if cert file is non-empty) + local cert_file="$(dirname "$truststore_path")/$alias.pem" + if [[ -s "$cert_file" ]]; then + echo "Importing certificate into truststore" + keytool -import -noprompt -alias $alias -file "$cert_file" \ + -keystore "$truststore_path" -storepass "${truststore_password}" || \ + echo "WARNING: Failed to import certificate for $alias, truststore may be incomplete" + else + echo "WARNING: Skipping certificate import for $alias, cert file is empty or missing" + fi + + echo "Truststore setup completed for $hostname" +} + +# Function to setup multi-domain truststore from values configuration +setup_multi_domain_truststore() { + local plugin_name="$1" + local ssl_truststore_config="$2" + + echo "Setting up multi-domain truststore for plugin: $plugin_name" + + # Parse the ssl_truststore configuration (passed as JSON-like string) + local generate=$(echo "$ssl_truststore_config" | grep -o '"generate"[[:space:]]*:[[:space:]]*true' | wc -l) + + if [[ $generate -eq 0 ]]; then + echo "Multi-domain truststore generation disabled for $plugin_name" + return 0 + fi + + echo "Multi-domain truststore generation enabled for $plugin_name" + + # Extract truststore path (REQUIRED) + local truststore_path=$(echo "$ssl_truststore_config" | grep -o '"truststore_path"[[:space:]]*:[[:space:]]*"[^"]*"' | sed 's/.*"truststore_path"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/') + if [[ -z "$truststore_path" ]]; then + echo "ERROR: ssl_truststore.truststore_path is required when ssl_truststore.generate is true for plugin $plugin_name" + exit 1 + fi + + # Extract password environment variable name (REQUIRED) + local password_env=$(echo "$ssl_truststore_config" | grep -o '"truststore_password_env"[[:space:]]*:[[:space:]]*"[^"]*"' | sed 's/.*"truststore_password_env"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/') + if [[ -z "$password_env" ]]; then + echo "ERROR: ssl_truststore.truststore_password_env is required when ssl_truststore.generate is true for plugin $plugin_name" + exit 1 + fi + + # Check if password already exists + if [[ -n "${!password_env}" ]]; then + echo "Using existing password from environment variable: $password_env" + local truststore_password="${!password_env}" + else + # Generate random password + local truststore_password=$(openssl rand -base64 12) + export "$password_env"="$truststore_password" + echo "Generated random password for $password_env: $truststore_password" + fi + + # Create truststore directory if it doesn't exist + mkdir -p $(dirname "$truststore_path") + + # Create truststore if it doesn't exist or if we're starting fresh + if [[ ! -f "$truststore_path" ]]; then + echo "Creating new multi-domain truststore at: $truststore_path" + + # Validate JAVA_HOME exists + if [[ -z "$JAVA_HOME" ]]; then + echo "ERROR: JAVA_HOME environment variable is not set, required for truststore creation for plugin $plugin_name" + exit 1 + fi + + # Validate cacerts file exists + if [[ ! -f "$JAVA_HOME/lib/security/cacerts" ]]; then + echo "ERROR: Java cacerts file not found at $JAVA_HOME/lib/security/cacerts for plugin $plugin_name" + exit 1 + fi + + # Copy cacerts as base truststore + if ! cp "$JAVA_HOME/lib/security/cacerts" "$truststore_path"; then + echo "ERROR: Failed to create truststore file at $truststore_path for plugin $plugin_name" + exit 1 + fi + + # Change the default password to our password + echo "Setting truststore password" + if ! keytool -storepasswd -keystore "$truststore_path" \ + -storepass "changeit" -new "$truststore_password" >/dev/null 2>&1; then + echo "ERROR: Failed to set truststore password for plugin $plugin_name" + exit 1 + fi + fi + + # Extract and process hostnames (REQUIRED) + local hostnames=$(echo "$ssl_truststore_config" | grep -o '"hostnames"[[:space:]]*:[[:space:]]*\[[^]]*\]' | sed 's/.*"hostnames"[[:space:]]*:[[:space:]]*\[\([^]]*\)\].*/\1/' | tr ',' '\n') + + if [[ -z "$hostnames" ]]; then + echo "ERROR: ssl_truststore.hostnames is required and must be a non-empty array when ssl_truststore.generate is true for plugin $plugin_name" + exit 1 + fi + + # Validate that hostnames array is not empty + local hostname_count=$(echo "$hostnames" | grep -v '^$' | wc -l) + if [[ $hostname_count -eq 0 ]]; then + echo "ERROR: ssl_truststore.hostnames must contain at least one hostname when ssl_truststore.generate is true for plugin $plugin_name" + exit 1 + fi + + # Download and import certificates for each hostname + while IFS= read -r hostname_entry; do + if [[ -n "$hostname_entry" ]]; then + # Clean up the hostname (remove quotes and whitespace) + local clean_hostname=$(echo "$hostname_entry" | sed 's/[[:space:]]*"\([^"]*\)".*/\1/' | xargs) + + if [[ -n "$clean_hostname" ]]; then + local hostname=$(echo "$clean_hostname" | cut -d':' -f1) + local port=$(echo "$clean_hostname" | cut -d':' -f2) + + # Validate hostname:port format + if [[ -z "$hostname" || -z "$port" || "$hostname" == "$port" ]]; then + echo "ERROR: Invalid hostname format '$clean_hostname' in ssl_truststore.hostnames for plugin $plugin_name. Expected format: 'hostname:port'" + exit 1 + fi + + # Validate port is numeric + if ! [[ "$port" =~ ^[0-9]+$ ]]; then + echo "ERROR: Invalid port '$port' in hostname '$clean_hostname' for plugin $plugin_name. Port must be numeric." + exit 1 + fi + + local alias="$plugin_name-$(echo $hostname | tr '.' '-')" + + echo "Downloading certificate for $hostname:$port with alias $alias" + + # Download CA certificate + local cert_file="$(dirname "$truststore_path")/$alias.pem" + if ! echo | openssl s_client -connect $hostname:$port -showcerts 2>/dev/null | \ + openssl x509 -outform PEM > "$cert_file"; then + echo "ERROR: Failed to download certificate from $hostname:$port for plugin $plugin_name" + exit 1 + fi + + # Validate certificate file is not empty + if [[ ! -s "$cert_file" ]]; then + echo "ERROR: Downloaded certificate from $hostname:$port is empty for plugin $plugin_name" + exit 1 + fi + + # Import certificate into truststore (skip if already exists) + if keytool -list -keystore "$truststore_path" -storepass "$truststore_password" -alias "$alias" >/dev/null 2>&1; then + echo "Certificate with alias $alias already exists in truststore, skipping" + else + echo "Importing certificate with alias $alias into truststore" + if ! keytool -import -noprompt -alias "$alias" -file "$cert_file" \ + -keystore "$truststore_path" -storepass "$truststore_password" >/dev/null 2>&1; then + echo "ERROR: Failed to import certificate with alias $alias into truststore for plugin $plugin_name" + exit 1 + fi + fi + fi + fi + done <<< "$hostnames" + + + + echo "Multi-domain truststore setup completed for $plugin_name at: $truststore_path" +} + +# Function to setup connectors in the background +setup_connectors() { + echo "Starting connector setup process..." + + # Wait for Kafka Connect to start + echo "Waiting for Kafka Connect to start..." + until wget -q http://localhost:8083/ -O /dev/null; do + echo "Waiting for Kafka Connect REST API..." + sleep 5 + done + + echo "Kafka Connect REST API is up. Waiting for connector plugins to load..." + # Wait for connector plugins to be available (indicates full initialization) + local retry_count=0 + local max_retries=12 + until wget -q -O - http://localhost:8083/connector-plugins 2>/dev/null | grep -q "class" || [ $retry_count -ge $max_retries ]; do + echo "Waiting for connector plugins to load... (attempt $((retry_count+1))/$max_retries)" + sleep 5 + retry_count=$((retry_count+1)) + done + + echo "Kafka Connect plugins loaded. Now waiting for existing connectors to be restored from connect-config topic..." + # Poll for connectors to be restored, but with a timeout + local connector_wait=0 + local max_connector_wait=20 + local prev_count=-1 + while [ $connector_wait -lt $max_connector_wait ]; do + INSTALLED_CONNECTORS=$(wget -q -O - http://localhost:8083/connectors 2>/dev/null || echo "[]") + local current_count=$(echo "$INSTALLED_CONNECTORS" | tr -d '[]"' | tr ',' '\n' | grep -v '^$' | wc -l | xargs) + + if [ "$current_count" != "$prev_count" ]; then + echo "Connectors being restored... Found $current_count connector(s) so far: $INSTALLED_CONNECTORS" + prev_count=$current_count + connector_wait=0 # Reset wait counter when we see changes + else + if [ $connector_wait -eq 0 ] && [ "$current_count" -gt 0 ]; then + echo "Connector count stable at $current_count. Waiting 5 more seconds to ensure restoration is complete..." + fi + connector_wait=$((connector_wait+1)) + fi + + sleep 1 + done + + # Get final list of currently installed connectors + echo "Fetching final list of installed connectors..." + INSTALLED_CONNECTORS=$(wget -q -O - http://localhost:8083/connectors 2>/dev/null || echo "[]") + echo "Installed connectors: $INSTALLED_CONNECTORS" + + # Build list of desired connectors from values file + DESIRED_CONNECTORS=({{- range .plugins }} "{{ .name }}"{{- end }}) + echo "Desired connectors from values: ${DESIRED_CONNECTORS[@]}" + echo "Number of desired connectors: ${#DESIRED_CONNECTORS[@]}" + + # Remove connectors that are not in the desired list + if [[ "$INSTALLED_CONNECTORS" != "[]" && "$INSTALLED_CONNECTORS" != "" ]]; then + echo "$INSTALLED_CONNECTORS" | tr -d '[]"' | tr ',' '\n' | while IFS= read -r connector; do + connector=$(echo "$connector" | xargs) # trim whitespace + if [[ -n "$connector" ]]; then + found=false + for desired in "${DESIRED_CONNECTORS[@]}"; do + if [[ "$connector" == "$desired" ]]; then + found=true + break + fi + done + if [[ "$found" == "false" ]]; then + echo "Connector '$connector' is not enabled. Removing..." + (echo -e "DELETE /connectors/$connector HTTP/1.1\r\nHost: localhost:8083\r\nConnection: close\r\n\r\n" | nc localhost 8083 > /dev/null 2>&1) || true + fi + fi + done + fi + + # Create/update connectors + {{- range .plugins }} + echo "Processing connector: {{ .name }}" + {{- if hasKey . "enabled" }} + {{- if not .enabled }} + echo "Connector {{ .name }} is disabled. Removing if it exists..." + (echo -e "DELETE /connectors/{{ .name }} HTTP/1.1\r\nHost: localhost:8083\r\nConnection: close\r\n\r\n" | nc localhost 8083 > /dev/null 2>&1) || true + + # Wait a bit between connectors to allow Kafka Connect API to stabilize + sleep 3 + {{- else }} + echo "Creating/updating connector: {{ .name }}" + +{{- if hasKey . "ssl_truststore" }} +# Setup multi-domain truststore if configured +SSL_TRUSTSTORE_CONFIG='{"generate":{{ if hasKey .ssl_truststore "generate" }}{{ .ssl_truststore.generate }}{{ else }}false{{ end }}{{- if hasKey .ssl_truststore "truststore_path" }},"truststore_path":"{{ .ssl_truststore.truststore_path }}"{{- end }}{{- if hasKey .ssl_truststore "truststore_password_env" }},"truststore_password_env":"{{ .ssl_truststore.truststore_password_env }}"{{- end }}{{- if hasKey .ssl_truststore "hostnames" }},"hostnames":[{{- range $i, $hostname := .ssl_truststore.hostnames }}{{- if $i }},{{- end }}"{{ $hostname }}"{{- end }}]{{- end }}}' +setup_multi_domain_truststore "{{ .name }}" "$SSL_TRUSTSTORE_CONFIG" +{{- end }} + +{{- if and (hasKey .config "ssl") (eq .config.ssl "true") }} +# Check if SSL is enabled +# Export ssl.truststore.location if it exists +{{- if hasKey .config "ssl.truststore.location" }} +export SSL_TRUSTSTORE_LOCATION={{ index .config "ssl.truststore.location" | quote }} +{{- end }} +# Export ssl.truststore.password if it exists +{{- if hasKey .config "ssl.truststore.password" }} +export SSL_TRUSTSTORE_PASSWORD={{ index .config "ssl.truststore.password" | quote }} +{{- end }} +# Setup truststore +truststore_init "{{ .config.hostname }}" "{{ .config.port }}" "{{ .name }}" "{{ default "" .config.jdbcConnectionProperties }}" +{{- end }} + +CONFIG=$(cat << 'EOF' +{ + "name": "{{ .name }}", + "config": { + {{- $first := true }} + {{- range $key, $value := .config }} + {{- if $first }}{{ $first = false }}{{ else }},{{ end }} + "{{ $key }}": "{{ $value }}" + {{- end }} + {{- if and (hasKey .config "ssl") (eq .config.ssl "true") (not (hasKey .config "ssl.truststore.password")) }} + ,"ssl.truststore.password": "${SSL_TRUSTSTORE_PASSWORD}" + {{- end }} + } +} +EOF +) + +# If we have a generated password, replace it in the config +if [[ -n "${SSL_TRUSTSTORE_PASSWORD}" && ! "{{ if hasKey .config "ssl.truststore.password" }}true{{ else }}false{{ end }}" == "true" ]]; then + CONFIG=$(echo "$CONFIG" | sed "s|\${SSL_TRUSTSTORE_PASSWORD}|${SSL_TRUSTSTORE_PASSWORD}|g") +fi + + {{- if hasKey . "ssl_truststore" }} + {{- if hasKey .ssl_truststore "truststore_password_env" }} + # Replace plugin-specific truststore password if it exists + PLUGIN_PASSWORD_VAR="{{ .ssl_truststore.truststore_password_env }}" + echo "DEBUG: Looking for password in environment variable: $PLUGIN_PASSWORD_VAR" + echo "DEBUG: Password value: ${!PLUGIN_PASSWORD_VAR}" + if [[ -n "${!PLUGIN_PASSWORD_VAR}" ]]; then + echo "DEBUG: Replacing \${${PLUGIN_PASSWORD_VAR}} with password in config" + CONFIG=$(echo "$CONFIG" | sed "s|\${${PLUGIN_PASSWORD_VAR}}|${!PLUGIN_PASSWORD_VAR}|g") + echo "DEBUG: Config after replacement:" + echo "$CONFIG" + else + echo "DEBUG: No password found in $PLUGIN_PASSWORD_VAR" + fi + {{- end }} + {{- end }} + +# Try to create connector with retry logic +max_retries=5 +retry_count=0 +while [ $retry_count -lt $max_retries ]; do + if create_or_update_connector "{{ .name }}" "$CONFIG" "$INSTALLED_CONNECTORS"; then + echo "Successfully created/updated connector {{ .name }} on attempt $((retry_count+1))" + break + else + retry_count=$((retry_count+1)) + if [ $retry_count -lt $max_retries ]; then + echo "Failed to create/update connector {{ .name }}, retrying in 10 seconds (attempt $retry_count/$max_retries)..." + sleep 10 + else + echo "Failed to create/update connector {{ .name }} after $max_retries attempts" + fi + fi +done + +# Wait a bit between connectors to allow Kafka Connect API to stabilize +sleep 3 +{{- end }} + {{- else }} + echo "Creating/updating connector: {{ .name }}" + {{- if and (hasKey .config "ssl") (eq .config.ssl "true") }} + # Check if SSL is enabled + # Export ssl.truststore.location if it exists + {{- if hasKey .config "ssl.truststore.location" }} + export SSL_TRUSTSTORE_LOCATION={{ index .config "ssl.truststore.location" | quote }} + {{- end }} + # Export ssl.truststore.password if it exists + {{- if hasKey .config "ssl.truststore.password" }} + export SSL_TRUSTSTORE_PASSWORD={{ index .config "ssl.truststore.password" | quote }} + {{- end }} + # Setup truststore + truststore_init "{{ .config.hostname }}" "{{ .config.port }}" "{{ .name }}" "{{ default "" .config.jdbcConnectionProperties }}" + {{- end }} + + CONFIG=$(cat << 'EOF' +{ + "name": "{{ .name }}", + "config": { + {{- $first := true }} + {{- range $key, $value := .config }} + {{- if $first }}{{ $first = false }}{{ else }},{{ end }} + "{{ $key }}": "{{ $value }}" + {{- end }} + {{- if and (hasKey .config "ssl") (eq .config.ssl "true") (not (hasKey .config "ssl.truststore.password")) }} + {{- if not $first }},{{ end }} + "ssl.truststore.password": "${SSL_TRUSTSTORE_PASSWORD}" + {{- end }} + } +} +EOF +) + + # If we have a generated password, replace it in the config + if [[ -n "${SSL_TRUSTSTORE_PASSWORD}" && ! "{{ if hasKey .config "ssl.truststore.password" }}true{{ else }}false{{ end }}" == "true" ]]; then + CONFIG=$(echo "$CONFIG" | sed "s|\${SSL_TRUSTSTORE_PASSWORD}|${SSL_TRUSTSTORE_PASSWORD}|g") + fi + + {{- if hasKey . "ssl_truststore" }} +{{- if hasKey .ssl_truststore "truststore_password_env" }} +# Replace plugin-specific truststore password if it exists +PLUGIN_PASSWORD_VAR="{{ .ssl_truststore.truststore_password_env }}" +echo "DEBUG: Looking for password in environment variable: $PLUGIN_PASSWORD_VAR" +echo "DEBUG: Password value: ${!PLUGIN_PASSWORD_VAR}" +if [[ -n "${!PLUGIN_PASSWORD_VAR}" ]]; then + echo "DEBUG: Replacing \${${PLUGIN_PASSWORD_VAR}} with password in config" + CONFIG=$(echo "$CONFIG" | sed "s|\${${PLUGIN_PASSWORD_VAR}}|${!PLUGIN_PASSWORD_VAR}|g") + echo "DEBUG: Config after replacement:" + echo "$CONFIG" +else + echo "DEBUG: No password found in $PLUGIN_PASSWORD_VAR" +fi +{{- end }} +{{- end }} + + # Try to create connector with retry logic +max_retries=5 +retry_count=0 +while [ $retry_count -lt $max_retries ]; do + if create_or_update_connector "{{ .name }}" "$CONFIG" "$INSTALLED_CONNECTORS"; then + echo "Successfully created/updated connector {{ .name }} on attempt $((retry_count+1))" + break + else + retry_count=$((retry_count+1)) + if [ $retry_count -lt $max_retries ]; then + echo "Failed to create/update connector {{ .name }}, retrying in 10 seconds (attempt $retry_count/$max_retries)..." + sleep 10 + else + echo "Failed to create/update connector {{ .name }} after $max_retries attempts" + fi + fi +done + +# Wait a bit between connectors to allow Kafka Connect API to stabilize +sleep 3 + {{- end }} + {{- end }} + + echo "All Kafka connectors have been configured and started." +} + +# Signal handler for graceful shutdown +cleanup() { + echo "Received shutdown signal, stopping Kafka Connect..." + if [[ -n $KAFKA_PID ]]; then + kill -TERM $KAFKA_PID + wait $KAFKA_PID + fi + exit 0 +} + +# Set up signal handlers +trap cleanup SIGTERM SIGINT + +echo "Starting Kafka Connect distributed worker..." + +# Updating rest.advertised.host.name dynamically +POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) +WORKLOAD_NAME=$(echo $CPLN_WORKLOAD | sed 's|.*/workload/\([^/]*\)$|\1|') +cp /opt/kafka/config/connect-distributed.properties /opt/kafka/config/connect-distributed-updated.properties +echo "" >> /opt/kafka/config/connect-distributed-updated.properties +echo "rest.advertised.host.name=${WORKLOAD_NAME}-${POD_ID}.${WORKLOAD_NAME}" >> /opt/kafka/config/connect-distributed-updated.properties + +# Start the connector setup process in the background +setup_connectors & +SETUP_PID=$! + +# Start Kafka Connect in the foreground +echo "Starting Kafka Connect in foreground mode..." +exec /opt/kafka/bin/connect-distributed.sh /opt/kafka/config/connect-distributed-updated.properties & +KAFKA_PID=$! + +# Wait for either process to finish +wait $KAFKA_PID +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels +*/}} +{{- define "kafka.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/domain.yaml b/kafka/versions/4.0.0/templates/domain.yaml new file mode 100644 index 00000000..acf09fa9 --- /dev/null +++ b/kafka/versions/4.0.0/templates/domain.yaml @@ -0,0 +1,64 @@ +{{- include "kafka.validateOnePublicAddress" . }} +{{- include "kafka.validatedirectReplicaRoutingConfig" . }} +{{- range $key, $listener := .Values.kafka.listeners }} +{{- if and $listener.directReplicaRouting $listener.directReplicaRouting.enabled }} +--- +kind: domain +name: {{ $listener.directReplicaRouting.publicAddress }} +description: {{ $listener.directReplicaRouting.publicAddress }} +spec: + acceptAllHosts: false + acceptAllSubdomains: false + certChallengeType: dns01 + dnsMode: cname + ports: + - number: {{ $listener.directReplicaRouting.containerPort }} + protocol: tcp + routes: + - port: {{ $listener.directReplicaRouting.containerPort }} + prefix: / + workloadLink: //gvc/{{ $.Values.global.cpln.gvc }}/workload/{{ include "kafka.clusterName" $ }} + tls: + cipherSuites: + - ECDHE-ECDSA-AES256-GCM-SHA384 + - ECDHE-ECDSA-CHACHA20-POLY1305 + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES256-GCM-SHA384 + - ECDHE-RSA-CHACHA20-POLY1305 + - ECDHE-RSA-AES128-GCM-SHA256 + - AES256-GCM-SHA384 + - AES128-GCM-SHA256 + minProtocolVersion: TLSV1_2 + workloadLink: //gvc/{{ $.Values.global.cpln.gvc }}/workload/{{ include "kafka.clusterName" $ }} +{{- else if $listener.publicAddress }} +--- +kind: domain +name: {{ $listener.publicAddress }} +description: {{ $listener.publicAddress }} +spec: + acceptAllHosts: false + dnsMode: cname + ports: + {{- $replicaCount := $.Values.kafka.replicas | int }} + {{- range $i := until $replicaCount }} + - number: {{ add 3000 $i }} + protocol: tcp + routes: + - port: {{ add 3000 $i }} + prefix: / + replica: {{ $i }} + workloadLink: //gvc/{{ $.Values.global.cpln.gvc }}/workload/{{ include "kafka.clusterName" $ }} + tls: + cipherSuites: + - ECDHE-ECDSA-AES256-GCM-SHA384 + - ECDHE-ECDSA-CHACHA20-POLY1305 + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES256-GCM-SHA384 + - ECDHE-RSA-CHACHA20-POLY1305 + - ECDHE-RSA-AES128-GCM-SHA256 + - AES256-GCM-SHA384 + - AES128-GCM-SHA256 + minProtocolVersion: TLSV1_2 + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/identity.yaml b/kafka/versions/4.0.0/templates/identity.yaml new file mode 100644 index 00000000..571ed3b6 --- /dev/null +++ b/kafka/versions/4.0.0/templates/identity.yaml @@ -0,0 +1,4 @@ +kind: identity +name: {{ include "kafka.name" . }} +description: {{ include "kafka.clusterName" . }} identity +gvc: {{ .Values.global.cpln.gvc }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/kafbat-ui.yaml b/kafka/versions/4.0.0/templates/kafbat-ui.yaml new file mode 100644 index 00000000..992c5ba1 --- /dev/null +++ b/kafka/versions/4.0.0/templates/kafbat-ui.yaml @@ -0,0 +1,114 @@ +{{- if .Values.kafbat_ui.enabled }} +{{- if .Values.kafbat_ui.domain }} +kind: domain +name: {{ .Values.kafbat_ui.domain }} +description: {{ .Values.kafbat_ui.domain }} +spec: + acceptAllHosts: false + dnsMode: cname + ports: + - number: 443 + protocol: http2 + routes: + - port: 8080 + prefix: / + workloadLink: //gvc/{{ $.Values.global.cpln.gvc }}/workload/{{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} + tls: + cipherSuites: + - ECDHE-ECDSA-AES256-GCM-SHA384 + - ECDHE-ECDSA-CHACHA20-POLY1305 + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES256-GCM-SHA384 + - ECDHE-RSA-CHACHA20-POLY1305 + - ECDHE-RSA-AES128-GCM-SHA256 + - AES256-GCM-SHA384 + - AES128-GCM-SHA256 + minProtocolVersion: TLSV1_2 +--- +{{- end }} +kind: policy +name: {{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +description: {{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +tags: {{- include "kafka.tags" . | nindent 2 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ $.Values.global.cpln.gvc }}/identity/{{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +targetKind: secret +targetLinks: + - //secret/{{ .Values.kafbat_ui.configuration_secret }} +--- +kind: identity +name: {{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +description: {{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +gvc: {{ $.Values.global.cpln.gvc }} +--- +kind: workload +name: {{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +description: {{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} +tags: + {{- if .Values.kafbat_ui.deletionProtection }} + cpln/protected: true + {{- end }} + {{- include "kafka.tags" . | nindent 2 }} +spec: + type: standard + containers: + - name: kafbat-ui + cpu: {{ .Values.kafbat_ui.cpu }} + {{- if .Values.kafbat_ui.minCpu }} + minCpu: '{{ .Values.kafbat_ui.minCpu }}' + {{- end }} + env: + - name: SPRING_CONFIG_ADDITIONAL-LOCATION + value: /etc/config.yaml + image: {{ .Values.kafbat_ui.image }} + inheritEnv: false + memory: {{ .Values.kafbat_ui.memory }} + {{- if .Values.kafbat_ui.minMemory }} + minMemory: {{ .Values.kafbat_ui.minMemory }} + {{- end }} + ports: + - number: 8080 + volumes: + - path: /etc/config.yaml + recoveryPolicy: retain + uri: cpln://secret/{{ .Values.kafbat_ui.configuration_secret }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: {{ .Values.kafbat_ui.replicas }} + metric: disabled + minScale: {{ .Values.kafbat_ui.replicas }} + scaleToZeroDelay: 300 + target: 100 + {{- if or .Values.kafbat_ui.minCpu .Values.kafbat_ui.minMemory }} + capacityAI: true + {{- else }} + capacityAI: false + {{- end }} + debug: false + suspend: false + timeoutSeconds: {{ .Values.kafbat_ui.timeoutSeconds }} +{{- if .Values.kafbat_ui.firewall }} + firewallConfig: + {{- if or (hasKey .Values.kafbat_ui.firewall "external_inboundAllowCIDR") (hasKey .Values.kafbat_ui.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.kafbat_ui.firewall.external_inboundAllowCIDR }}{{ .Values.kafbat_ui.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.kafbat_ui.firewall.external_outboundAllowCIDR }}{{ .Values.kafbat_ui.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.kafbat_ui.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.kafbat_ui.firewall.internal_inboundAllowType }} + {{- end }} +{{- end }} + identityLink: //gvc/{{ $.Values.global.cpln.gvc }}/identity/{{ include "kafka.name" $ }}-{{ .Values.kafbat_ui.name }} + loadBalancer: + direct: + enabled: false + ports: [] + securityOptions: + filesystemGroupId: 101 + supportDynamicTags: false +{{- end }} diff --git a/kafka/versions/4.0.0/templates/kafka-connectors.yaml b/kafka/versions/4.0.0/templates/kafka-connectors.yaml new file mode 100644 index 00000000..4d0b8300 --- /dev/null +++ b/kafka/versions/4.0.0/templates/kafka-connectors.yaml @@ -0,0 +1,224 @@ +{{- if .Values.kafka_connectors }} +{{- range .Values.kafka_connectors }} +{{- include "kafka.validateImage" . -}} +kind: policy +name: {{ include "kafka.name" $ }}-connect-{{ .name }} +description: {{ include "kafka.name" $ }}-connect-{{ .name }} +tags: {{- include "kafka.tags" $ | nindent 2 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ $.Values.global.cpln.gvc }}/identity/{{ include "kafka.name" $ }}-connect-{{ .name }} +targetKind: secret +targetLinks: + - //secret/{{ include "kafka.name" $ }}-connect-{{ .name }}-props + - //secret/{{ include "kafka.name" $ }}-connect-{{ .name }}-init + - //secret/{{ include "kafka.name" $ }}-connect-{{ .name }}-download + {{- if and .extraVolumes (ne (len .extraVolumes) 0) }} + {{- range .extraVolumes }} + {{- if contains "/secret/" .uri }} + {{- $secretName := regexReplaceAll ".*//secret/([^.]+).*" .uri "${1}" }} + - //secret/{{ $secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- /* Check for secrets in ssl_truststore configuration values */ -}} + {{- if hasKey . "ssl_truststore" }} + {{- range $key, $value := .ssl_truststore }} + {{- if and (kindIs "string" $value) (contains "//secret/" $value) }} + {{- $secretName := regexReplaceAll ".*//secret/([^.]+).*" $value "${1}" }} + - //secret/{{ $secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- /* Check for secrets in env variables */ -}} + {{- if hasKey . "env" }} + {{- range .env }} + {{- if and (hasKey . "value") (contains "//secret/" .value) }} + {{- $secretName := regexReplaceAll ".*//secret/([^.]+).*" .value "${1}" }} + - //secret/{{ $secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- /* Check for secrets in connector_properties */ -}} + {{- if hasKey . "connector_properties" }} + {{- range $key, $value := .connector_properties }} + {{- if and (kindIs "string" $value) (contains "//secret/" $value) }} + {{- $secretName := regexReplaceAll ".*//secret/([^.]+).*" $value "${1}" }} + - //secret/{{ $secretName }} + {{- end }} + {{- end }} + {{- end }} +--- +kind: secret +name: {{ include "kafka.name" $ }}-connect-{{ .name }}-download +type: opaque +data: + encoding: plain + payload: | + {{- include "kafka.connectors.download.script" (dict "plugins" .plugins "plugins_folder" .plugins_folder "verbose" .verbose) | nindent 4 }} +--- +kind: secret +name: {{ include "kafka.name" $ }}-connect-{{ .name }}-init +type: opaque +data: + encoding: plain + payload: | + {{- include "kafka.connectors.run.script" (dict "plugins" .plugins "plugins_folder" .plugins_folder "verbose" .verbose) | nindent 4 }} +--- +kind: secret +name: {{ include "kafka.name" $ }}-connect-{{ .name }}-props +description: {{ include "kafka.name" $ }}-connect-{{ .name }}-props +tags: {{- include "kafka.tags" $ | nindent 2 }} +type: opaque +data: + encoding: plain + payload: |- + {{- if not (hasKey .connector_properties "bootstrap.servers") }} + bootstrap.servers={{ include "kafka.clientBootstrapAddress" $ }} + {{- end }} + {{- range $key, $value := .connector_properties }} + {{ $key }}={{ $value }} + {{- end }} +--- +kind: identity +name: {{ include "kafka.name" $ }}-connect-{{ .name }} +description: {{ include "kafka.name" $ }}-connect-{{ .name }} +gvc: {{ $.Values.global.cpln.gvc }} +--- +kind: volumeset +name: {{ include "kafka.name" $ }}-connect-{{ .name }} +description: {{ include "kafka.name" $ }}-connect-{{ .name }} +tags: {{- include "kafka.tags" $ | nindent 2 }} +spec: + fileSystemType: {{ dig "volumes" "fileSystemType" "ext4" . }} + initialCapacity: {{ dig "volumes" "initialCapacity" 10 . }} + performanceClass: {{ dig "volumes" "performanceClass" "general-purpose-ssd" . }} + {{- if and .volumes .volumes.customEncryption .volumes.customEncryption.enabled }} + customEncryption: + regions: + {{ .volumes.customEncryption.region }}: + keyId: '{{ .volumes.customEncryption.keyId }}' + {{- end }} + {{- if and .volumes .volumes.snapshots }} + snapshots: + createFinalSnapshot: {{ .volumes.snapshots.createFinalSnapshot | default true }} + retentionDuration: {{ .volumes.snapshots.retentionDuration | default "7d" }} + {{- if .volumes.snapshots.schedule }} + schedule: {{ .volumes.snapshots.schedule }} + {{- end }} + {{- else }} + snapshots: + createFinalSnapshot: true + retentionDuration: 7d + {{- end }} +--- +kind: workload +name: {{ include "kafka.name" $ }}-connect-{{ .name }} +description: {{ include "kafka.name" $ }}-connect-{{ .name }} +gvc: {{ $.Values.global.cpln.gvc }} +tags: + {{- if .deletionProtection }} + cpln/protected: true + {{- end }} + {{- include "kafka.tags" $ | nindent 2 }} +spec: + type: stateful + containers: + - name: kafka-connect + {{- if and .env (ne (len .env) 0) }} + env: + {{- toYaml .env | nindent 8 }} + {{- end }} + args: + - '-c' + - sleep 60 && cp /opt/kafka/init.sh /opt/kafka/init-run.sh && chmod +x /opt/kafka/init-run.sh && /opt/kafka/init-run.sh + command: /bin/bash + cpu: {{ .cpu }} + {{- if .minCpu }} + minCpu: {{ .minCpu }} + {{- end }} + image: {{ .image }} + inheritEnv: false + memory: {{ .memory }} + {{- if .minMemory }} + minMemory: {{ .minMemory }} + {{- end }} + ports: + - number: 8083 + protocol: http + volumes: + - path: /opt/kafka/plugins + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "kafka.name" $ }}-connect-{{ .name }} + - path: /opt/kafka/config/connect-distributed.properties + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" $ }}-connect-{{ .name }}-props + - path: /opt/kafka/init.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" $ }}-connect-{{ .name }}-init + {{- if .extraVolumes }} + {{- toYaml .extraVolumes | nindent 8 }} + {{- end }} + - name: plugins-downloader + args: + - '-c' + - cp /opt/kafka/download.sh /opt/kafka/download-run.sh && chmod +x /opt/kafka/download-run.sh && /opt/kafka/download-run.sh + command: /bin/sh + cpu: 80m + image: busybox:musl + inheritEnv: false + memory: 120Mi + ports: [] + volumes: + - path: /opt/kafka/plugins + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "kafka.name" $ }}-connect-{{ .name }} + - path: /opt/kafka/download.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" $ }}-connect-{{ .name }}-download + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: {{ .replicas }} + metric: cpu + minScale: {{ .replicas }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + {{- if .multiZone }} + multiZone: + enabled: true + {{- else }} + multiZone: + enabled: false + {{- end }} + suspend: false + timeoutSeconds: {{ .timeoutSeconds }} +{{- if .firewall }} + firewallConfig: + {{- if or (hasKey .firewall "external_inboundAllowCIDR") (hasKey .firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .firewall.external_inboundAllowCIDR }}{{ .firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .firewall.external_outboundAllowCIDR }}{{ .firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "none" .firewall.internal_inboundAllowType }} + {{- if hasKey .firewall "inboundAllowWorkload" }} + inboundAllowWorkload: {{ .firewall.inboundAllowWorkload | toYaml | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} + identityLink: //gvc/{{ $.Values.global.cpln.gvc }}/identity/{{ include "kafka.name" $ }}-connect-{{ .name }} + loadBalancer: + direct: + enabled: false + ports: [] + securityOptions: + filesystemGroupId: 1001 + supportDynamicTags: false +{{- end }} +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/kafka-rest-proxy.yaml b/kafka/versions/4.0.0/templates/kafka-rest-proxy.yaml new file mode 100644 index 00000000..177f8c2b --- /dev/null +++ b/kafka/versions/4.0.0/templates/kafka-rest-proxy.yaml @@ -0,0 +1,169 @@ +{{- if .Values.kafka_rest_proxy.enabled }} +{{- if .Values.kafka_rest_proxy.password_properties }} +kind: secret +name: {{ include "kafka.name" . }}-rest-password-properties +description: {{ include "kafka.name" . }}-rest-password-properties +tags: {{- include "kafka.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: |- + {{- range $key, $value := .Values.kafka_rest_proxy.password_properties }} + {{ $key }}: {{ $value }} + {{- end }} +{{- end }} +--- +kind: secret +name: {{ include "kafka.name" . }}-rest-properties +description: {{ include "kafka.name" . }}-rest-properties +tags: {{- include "kafka.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: |- + {{- range $key, $value := .Values.kafka_rest_proxy.properties }} + {{ $key }}={{ $value }} + {{- end }} +--- +kind: secret +name: {{ include "kafka.name" . }}-rest-jaas-conf +description: {{ include "kafka.name" . }}-rest-jaas-conf +tags: {{- include "kafka.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: >- + {{- .Values.kafka_rest_proxy.jaas_conf | nindent 4 }} +--- +kind: identity +name: {{ include "kafka.name" . }}-rest-proxy-identity +description: Identity for Kafka Rest Proxy {{ include "kafka.name" . }} +gvc: {{ .Values.global.cpln.gvc }} +--- +kind: policy +name: {{ include "kafka.name" . }}-rest-proxy-policy +origin: default +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "kafka.name" . }}-rest-proxy-identity +targetKind: secret +targetLinks: +{{- if .Values.kafka_rest_proxy.password_properties }} + - //secret/{{ include "kafka.name" . }}-rest-password-properties +{{- end }} + - //secret/{{ include "kafka.name" . }}-rest-properties + - //secret/{{ include "kafka.name" . }}-rest-jaas-conf +--- +kind: workload +name: {{ include "kafka.name" . }}-{{ .Values.kafka_rest_proxy.name }} +description: Kafka Rest Proxy +gvc: {{ .Values.global.cpln.gvc }} +tags: + {{- if .Values.kafka_rest_proxy.deletionProtection }} + cpln/protected: true + {{- end }} + cpln/marketplace: "true" + cpln/marketplace-template: kafka + cpln/marketplace-template-version: {{ .Chart.Version }} +spec: + type: standard + containers: + - name: rest-proxy + args: + - '-c' + - >- + KAFKAREST_OPTS="-Djava.security.auth.login.config=/etc/kafka-rest/kafka-rest.jaas.conf" + kafka-rest-start /etc/kafka-rest/kafka-rest.properties + command: /bin/bash + cpu: {{ .Values.kafka_rest_proxy.cpu }} + image: {{ .Values.kafka_rest_proxy.image }} + inheritEnv: false + memory: {{ .Values.kafka_rest_proxy.memory }} + {{- if and .Values.kafka_rest_proxy.capacityAI .Values.kafka_rest_proxy.capacityAI.enabled }} + {{- if .Values.kafka_rest_proxy.capacityAI.minCpu }} + minCpu: {{ .Values.kafka_rest_proxy.capacityAI.minCpu }} + {{- end }} + {{- if .Values.kafka_rest_proxy.capacityAI.minMemory }} + minMemory: {{ .Values.kafka_rest_proxy.capacityAI.minMemory }} + {{- end }} + {{- end }} + ports: + - number: 8082 + protocol: http + volumes: + {{- if .Values.kafka_rest_proxy.password_properties }} + - path: /etc/kafka-rest/password.properties + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" . }}-rest-password-properties + {{- end }} + - path: /etc/kafka-rest/kafka-rest.jaas.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" . }}-rest-jaas-conf + - path: /etc/kafka-rest/kafka-rest.properties + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" . }}-rest-properties + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: {{ .Values.kafka_rest_proxy.replicas }} + metric: disabled + minScale: {{ .Values.kafka_rest_proxy.replicas }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: {{ .Values.kafka_rest_proxy.capacityAI.enabled }} + debug: false + suspend: false + timeoutSeconds: {{ .Values.kafka_rest_proxy.timeoutSeconds }} +{{- if .Values.kafka_rest_proxy.firewall }} + firewallConfig: + {{- if or (hasKey .Values.kafka_rest_proxy.firewall "external_inboundAllowCIDR") (hasKey .Values.kafka_rest_proxy.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.kafka_rest_proxy.firewall.external_inboundAllowCIDR }}{{ .Values.kafka_rest_proxy.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.kafka_rest_proxy.firewall.external_outboundAllowCIDR }}{{ .Values.kafka_rest_proxy.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.kafka_rest_proxy.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.kafka_rest_proxy.firewall.internal_inboundAllowType }} + {{- if .Values.kafka_rest_proxy.firewall.inboundAllowWorkload }} + inboundAllowWorkload: {{ .Values.kafka_rest_proxy.firewall.inboundAllowWorkload | toYaml | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} + identityLink: //identity/{{ include "kafka.name" . }}-rest-proxy-identity + loadBalancer: + direct: + enabled: false + ports: [] + securityOptions: + filesystemGroupId: 1000 + supportDynamicTags: false +{{ if .Values.kafka_rest_proxy.domain }} +--- +kind: domain +name: {{ .Values.kafka_rest_proxy.domain }} +description: {{ .Values.kafka_rest_proxy.domain }} +spec: + acceptAllHosts: false + dnsMode: cname + ports: + - number: 443 + protocol: http2 + routes: + - port: 8082 + prefix: / + workloadLink: //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "kafka.name" . }}-{{ .Values.kafka_rest_proxy.name }} + tls: + cipherSuites: + - ECDHE-ECDSA-AES256-GCM-SHA384 + - ECDHE-ECDSA-CHACHA20-POLY1305 + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES256-GCM-SHA384 + - ECDHE-RSA-CHACHA20-POLY1305 + - ECDHE-RSA-AES128-GCM-SHA256 + - AES256-GCM-SHA384 + - AES128-GCM-SHA256 + minProtocolVersion: TLSV1_2 +{{- end }} +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/policy.yaml b/kafka/versions/4.0.0/templates/policy.yaml new file mode 100644 index 00000000..fb9a338b --- /dev/null +++ b/kafka/versions/4.0.0/templates/policy.yaml @@ -0,0 +1,16 @@ +kind: policy +name: {{ include "kafka.name" . }} +origin: default +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "kafka.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "kafka.name" . }}-controller-configuration + - //secret/{{ include "kafka.name" . }}-init + - //secret/{{ include "kafka.name" . }}-secrets +{{- if .Values.jmx_exporter }} + - //secret/{{ include "kafka.name" . }}-jmx-exporter-conf +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/secret-controller-configuration.yaml b/kafka/versions/4.0.0/templates/secret-controller-configuration.yaml new file mode 100644 index 00000000..c66a36e8 --- /dev/null +++ b/kafka/versions/4.0.0/templates/secret-controller-configuration.yaml @@ -0,0 +1,99 @@ +kind: secret +name: {{ include "kafka.name" . }}-controller-configuration +type: opaque +data: + encoding: plain + payload: | + {{- include "kafka.validateReplicas" . }} + + # Listeners configuration + listeners-placeholder + advertised.listeners=INTERNAL://advertised-address-placeholder:9094,CONTROLLER://advertised-controller-address-placeholder:9093{{- range .Values.kafka.listeners }}{{- include "kafka.validateListenerConfig" . }},{{ .name | upper }}://advertised-{{ .name | lower }}-address-placeholder{{- end }} + listener.security.protocol.map=INTERNAL:SASL_PLAINTEXT,CONTROLLER:SASL_PLAINTEXT{{- range .Values.kafka.listeners }},{{ .name | upper }}:{{ .protocol }}{{- end }} + + # KRaft process roles + process.roles=process-roles-placeholder + + #node.id= + controller.listener.names=CONTROLLER + {{$replicaCount := int .Values.kafka.replicas -}} + {{- if eq $replicaCount 2 -}} + {{- fail "Invalid number of Kraft replicas: must not be 2" -}} + {{- end -}} + controller.quorum.voters= {{- $result := "" }} + {{- range $i := until $replicaCount }} + {{- if and (ge $i 0) (lt $i 5) }} + {{- if $i }} + {{- $result = print $result "," }} + {{- end }} + {{- $result = print $result (printf "%d@%s-%s-%d.%s-%s:9093" $i $.Release.Name $.Values.kafka.name $i $.Release.Name $.Values.kafka.name ) }} + {{- end }} + {{- end }} + {{- $result }} + + # Kraft Controller listener SASL settings + sasl.mechanism.controller.protocol=PLAIN + listener.name.controller.sasl.enabled.mechanisms=PLAIN + listener.name.controller.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="controller_user" password="controller-password-placeholder" user_controller_user="controller-password-placeholder"; + log.dirs={{ .Values.kafka.logDirs }} + sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512 + + # Interbroker configuration + inter.broker.listener.name=INTERNAL + sasl.mechanism.inter.broker.protocol=PLAIN + + # Listeners SASL JAAS configuration +{{- include "kafka.validateAdminExists" . }} +{{- range .Values.kafka.listeners }} + {{- include "kafka.validateAuthConfig" . }} + {{- if .sasl }} + {{- $adminConfig := "" }} + {{- if .sasl.admin }} + {{- $adminConfig = printf "user_%s=\"%s\"" .sasl.admin.username .sasl.admin.password }} + {{- end }} + listener.name.{{ .name | lower }}.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required {{- if $adminConfig }} {{ $adminConfig }}{{- end }}{{- $users := .sasl.users | split "," }}{{- $passwords := .sasl.passwords | split "," }}{{- range $index, $user := $users }}{{- $password := index $passwords $index }} user_{{ $user }}="{{ $password }}"{{- end }}; + listener.name.{{ .name | lower }}.scram-sha-256.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required {{- if $adminConfig }} {{ $adminConfig }}{{- end }}{{- range $index, $user := $users }}{{- $password := index $passwords $index }} user_{{ $user }}="{{ $password }}"{{- end }}; + listener.name.{{ .name | lower }}.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required {{- if $adminConfig }} {{ $adminConfig }}{{- end }}{{- range $index, $user := $users }}{{- $password := index $passwords $index }} user_{{ $user }}="{{ $password }}"{{- end }}; + {{- end }} +{{- end }} + listener.name.internal.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="inter_broker_user" password="interbroker-password-placeholder" user_inter_broker_user="interbroker-password-placeholder"{{- range .Values.kafka.listeners }}{{- if and .sasl .sasl.admin }} user_{{ .sasl.admin.username }}="{{ .sasl.admin.password }}"{{- break }}{{- end }}{{- end }}; + listener.name.internal.scram-sha-256.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="inter_broker_user" password="interbroker-password-placeholder"; + listener.name.internal.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="inter_broker_user" password="interbroker-password-placeholder"; + # End of SASL JAAS configuration + + + {{- if .Values.kafka.acl }} + + # Enable ACL + authorizer.class.name=org.apache.kafka.metadata.authorizer.StandardAuthorizer + super.users=User:controller_user;User:inter_broker_user{{- if .Values.kafka.acl.superUsers }};{{ .Values.kafka.acl.superUsers }}{{- end }} + allow.everyone.if.no.acl.found={{ .Values.kafka.acl.allowEveryoneIfNoAclFound | default "false" }} + # End of ACL configuration + {{- end }} + + # Reliability and recovery defaults. Kafka's properties file uses last-value-wins for + # duplicate keys, so anything the operator sets in `extra_configurations` below + # overrides these defaults. + default.replication.factor={{ include "kafka.defaultReplicationFactor" . }} + min.insync.replicas={{ include "kafka.minInsyncReplicas" . }} + # controlled.shutdown drives leadership transfer + log flush before exit so a SIGTERM + # to the broker (now PID 1 thanks to exec in the init script) results in a clean exit + # within terminationGracePeriodSeconds and no recovery on next start. + controlled.shutdown.enable=true + controlled.shutdown.max.retries=3 + controlled.shutdown.retry.backoff.ms=5000 + # Never elect an out-of-sync replica as leader: prevents data loss after broker + # restart. Pair with min.insync.replicas above. + unclean.leader.election.enable=false + # Recovery thread pool sized to the broker's CPU budget. Recovery only runs on a + # *dirty* shutdown; a clean controlled.shutdown skips it entirely. When recovery + # does happen, this controls per-data-dir parallelism. + num.recovery.threads.per.data.dir={{ include "kafka.recoveryThreads" . }} + # Faster follower replication so replicas catch up and rejoin the ISR quickly after + # a transient broker outage. + num.replica.fetchers=4 + + # Extra configurations (these override any defaults above) + {{- range $key, $value := .Values.kafka.extra_configurations }} + {{ $key }}={{ $value }} + {{- end }} diff --git a/kafka/versions/4.0.0/templates/secret-init.yaml b/kafka/versions/4.0.0/templates/secret-init.yaml new file mode 100644 index 00000000..ff7575ef --- /dev/null +++ b/kafka/versions/4.0.0/templates/secret-init.yaml @@ -0,0 +1,184 @@ +{{- include "kafka.validatedirectReplicaRoutingConfig" . }} +kind: secret +name: {{ include "kafka.name" . }}-init +type: opaque +data: + encoding: plain + payload: | + #!/bin/bash + + set -o errexit + set -o nounset + set -o pipefail + + WORKLOAD_NAME=$(echo $CPLN_WORKLOAD | sed 's|.*/workload/\([^/]*\)$|\1|') + + error(){ + local message="${1:?missing message}" + echo "ERROR: ${message}" + exit 1 + } + + retry_while() { + local -r cmd="${1:?cmd is missing}" + local -r retries="${2:-12}" + local -r sleep_time="${3:-5}" + local return_value=1 + + read -r -a command <<< "$cmd" + for ((i = 1 ; i <= retries ; i+=1 )); do + "${command[@]}" && return_value=0 && break + sleep "$sleep_time" + done + return $return_value + } + + replace_in_file() { + local filename="${1:?filename is required}" + local match_regex="${2:?match regex is required}" + local substitute_regex="${3:?substitute regex is required}" + local posix_regex=${4:-true} + + local result + + # We should avoid using 'sed in-place' substitutions + # 1) They are not compatible with files mounted from ConfigMap(s) + # 2) We found incompatibility issues with Debian10 and "in-place" substitutions + local -r del=$'\001' # Use a non-printable character as a 'sed' delimiter to avoid issues + if [[ $posix_regex = true ]]; then + result="$(sed -E "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")" + else + result="$(sed "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")" + fi + echo "$result" > "$filename" + } + + kafka_conf_set() { + local file="${1:?missing file}" + local key="${2:?missing key}" + local value="${3:?missing value}" + + # Check if the value was set before + if grep -q "^[#\\s]*$key\s*=.*" "$file"; then + # Update the existing key + replace_in_file "$file" "^[#\\s]*${key}\s*=.*" "${key}=${value}" false + else + # Add a new key + printf '\n%s=%s' "$key" "$value" >>"$file" + fi + } + + replace_placeholder() { + local placeholder="${1:?missing placeholder value}" + local password="${2:?missing password value}" + sed -i "s|$placeholder|$password|g" "$KAFKA_CONFIG_FILE" + } + + configure_external_access() { + # Configure external hostname + if [[ -f "/shared/external-host.txt" ]]; then + host=$(cat "/shared/external-host.txt") + elif [[ -n "${EXTERNAL_ACCESS_HOST:-}" ]]; then + host="$EXTERNAL_ACCESS_HOST" + elif [[ -n "${EXTERNAL_ACCESS_HOSTS_LIST:-}" ]]; then + read -r -a hosts <<<"$(tr ',' ' ' <<<"${EXTERNAL_ACCESS_HOSTS_LIST}")" + host="${hosts[$POD_ID]}" + elif [[ "$EXTERNAL_ACCESS_HOST_USE_PUBLIC_IP" =~ ^(yes|true)$ ]]; then + host=$(curl -s https://ipinfo.io/ip) + else + error "External access hostname not provided" + fi + + # Configure external port + if [[ -f "/shared/external-port.txt" ]]; then + port=$(cat "/shared/external-port.txt") + elif [[ -n "${EXTERNAL_ACCESS_PORT:-}" ]]; then + if [[ "${EXTERNAL_ACCESS_PORT_AUTOINCREMENT:-}" =~ ^(yes|true)$ ]]; then + port="$((EXTERNAL_ACCESS_PORT + POD_ID))" + else + port="$EXTERNAL_ACCESS_PORT" + fi + elif [[ -n "${EXTERNAL_ACCESS_PORTS_LIST:-}" ]]; then + read -r -a ports <<<"$(tr ',' ' ' <<<"${EXTERNAL_ACCESS_PORTS_LIST}")" + port="${ports[$POD_ID]}" + else + error "External access port not provided" + fi + # Configure Kafka advertised listeners + sed -i -E "s|^(advertised\.listeners=\S+)$|\1,EXTERNAL://${host}:${port}|" "$KAFKA_CONFIG_FILE" + } + + configure_kafka_sasl() { + + # Replace placeholders with passwords + replace_placeholder "interbroker-password-placeholder" "$KAFKA_INTER_BROKER_PASSWORD" + replace_placeholder "controller-password-placeholder" "$KAFKA_CONTROLLER_PASSWORD" + } + + export KAFKA_CONFIG_FILE=${KAFKA_CONFIG_FILE:-/mnt/shared/config/server.properties} + cp /configmaps/server.properties $KAFKA_CONFIG_FILE + + # Get pod ID and role, last and second last fields in the pod name respectively + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + export KAFKA_CFG_NODE_ID="$POD_ID" + LOCATION_NAME=$(echo "$CPLN_LOCATION" | sed 's|.*/location/\([^/]*\)$|\1|') + + # Configure POD Role + if [ "$POD_ID" -le 4 ]; then + replace_placeholder "process-roles-placeholder" "controller,broker" + replace_placeholder "advertised-controller-address-placeholder" "${POD_NAME}.${WORKLOAD_NAME}.${CPLN_GVC_ALIAS}.svc.cluster.local" + replace_placeholder "listeners-placeholder" "listeners=INTERNAL://:9094,CONTROLLER://:9093{{- range $i, $key := keys .Values.kafka.listeners | sortAlpha }} + {{- $listener := index $.Values.kafka.listeners $key -}} + {{- include "kafka.validateListenerConfig" $listener -}},{{ $listener.name | upper }}://:{{- if and $listener.directReplicaRouting $listener.directReplicaRouting.enabled }}{{ $listener.directReplicaRouting.containerPort }}{{- else if $listener.publicAddress }}300${POD_ID}{{- else }}{{ $listener.containerPort }}{{- end }}{{- end }}" + else + replace_placeholder "process-roles-placeholder" "broker" + replace_placeholder ",CONTROLLER://advertised-controller-address-placeholder:9093" "" + replace_placeholder "listeners-placeholder" "listeners=INTERNAL://:9094{{- range $i, $key := keys .Values.kafka.listeners | sortAlpha }} + {{- $listener := index $.Values.kafka.listeners $key -}} + {{- include "kafka.validateListenerConfig" $listener -}},{{ $listener.name | upper }}://:{{- if and $listener.directReplicaRouting $listener.directReplicaRouting.enabled }}{{ $listener.directReplicaRouting.containerPort }}{{- else if $listener.publicAddress }}300${POD_ID}{{- else }}{{ $listener.containerPort }}{{- end }}{{- end }}" + fi + + # Configure node.id and/or broker.id + ID=$((POD_ID + KAFKA_MIN_ID)) + kafka_conf_set "$KAFKA_CONFIG_FILE" "node.id" "$ID" + + replace_placeholder "advertised-address-placeholder" "${POD_NAME}.${WORKLOAD_NAME}.${CPLN_GVC_ALIAS}.svc.cluster.local" + + {{- range $key, $listener := .Values.kafka.listeners }} + {{- include "kafka.validateListenerConfig" . }} + {{- if and $listener.directReplicaRouting $listener.directReplicaRouting.enabled }} + replace_placeholder "advertised-{{ $listener.name | lower }}-address-placeholder" "${POD_NAME}-${LOCATION_NAME}.{{ $listener.directReplicaRouting.publicAddress }}:{{ $listener.directReplicaRouting.containerPort }}" + {{- else if $listener.publicAddress }} + replace_placeholder "advertised-{{ $listener.name | lower }}-address-placeholder" "{{ $listener.publicAddress }}:300${POD_ID}" + {{- else }} + replace_placeholder "advertised-{{ $listener.name | lower }}-address-placeholder" "${POD_NAME}.${WORKLOAD_NAME}.${CPLN_GVC_ALIAS}.svc.cluster.local:{{ .containerPort }}" + {{- end }} + {{- end }} + + if [[ "${EXTERNAL_ACCESS_ENABLED:-false}" =~ ^(yes|true)$ ]]; then + configure_external_access + fi + + configure_kafka_sasl + + # Initialize log directories for Apache Kafka + {{- $root := . -}} + {{- $logDirs := split "," $root.Values.kafka.logDirs }} + {{- $counter := 0 }} + {{- range $path := $logDirs }} + # Create log directory if it doesn't exist + mkdir -p {{ $path }} + + # Remove lost+found if it exists (common with mounted volumes) + rm -rf {{ $path }}/lost+found 2>/dev/null || true + + # Ensure proper ownership for Apache Kafka (runs as appuser) + chown -R $(id -u):$(id -g) {{ $path }} 2>/dev/null || true + {{- $counter = add $counter 1 }} + {{- end }} + + # Ensure data directory exists (Apache Kafka default) + mkdir -p /var/lib/kafka/data + chown -R $(id -u):$(id -g) /var/lib/kafka/data 2>/dev/null || true + + exec /etc/kafka/docker/run diff --git a/kafka/versions/4.0.0/templates/secret-secrets.yaml b/kafka/versions/4.0.0/templates/secret-secrets.yaml new file mode 100644 index 00000000..86060a2d --- /dev/null +++ b/kafka/versions/4.0.0/templates/secret-secrets.yaml @@ -0,0 +1,12 @@ +kind: secret +name: {{ include "kafka.name" . }}-secrets +type: dictionary +data: + kraft-cluster-id: {{ .Values.kafka.secrets.kraft_cluster_id }} + {{- range $key, $listener := .Values.kafka.listeners }} + {{- if and $listener.sasl $listener.sasl.admin }} + {{ $listener.name | lower }}-admin-password: {{ $listener.sasl.admin.password }} + {{- end }} + {{- end }} + inter-broker-password: {{ .Values.kafka.secrets.inter_broker_password }} + controller-password: {{ .Values.kafka.secrets.controller_password }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/volumesets.yaml b/kafka/versions/4.0.0/templates/volumesets.yaml new file mode 100644 index 00000000..dba17d86 --- /dev/null +++ b/kafka/versions/4.0.0/templates/volumesets.yaml @@ -0,0 +1,31 @@ +{{- $root := . -}} +{{- $logDirs := split "," $root.Values.kafka.logDirs }} +{{- $counter := 0 }} +{{- range $index, $path := $logDirs }} +kind: volumeset +name: {{ include "kafka.name" $root }}-logs-{{ $counter }} +description: {{ include "kafka.name" $root }} logs {{ $counter }} +gvc: {{ $root.Values.global.cpln.gvc }} +spec: + initialCapacity: {{ $root.Values.kafka.volumes.logs.initialCapacity }} + performanceClass: {{ $root.Values.kafka.volumes.logs.performanceClass }} + fileSystemType: {{ $root.Values.kafka.volumes.logs.fileSystemType }} + {{- if and $root.Values.kafka.volumes.logs.customEncryption $root.Values.kafka.volumes.logs.customEncryption.enabled }} + customEncryption: + regions: + {{ $root.Values.kafka.volumes.logs.customEncryption.region }}: + keyId: '{{ $root.Values.kafka.volumes.logs.customEncryption.keyId }}' + {{- end }} + autoscaling: + maxCapacity: {{ $root.Values.kafka.volumes.logs.autoscaling.maxCapacity }} + minFreePercentage: {{ $root.Values.kafka.volumes.logs.autoscaling.minFreePercentage }} + scalingFactor: {{ $root.Values.kafka.volumes.logs.autoscaling.scalingFactor }} +{{- if $root.Values.kafka.volumes.logs.snapshots }} + snapshots: + createFinalSnapshot: {{ $root.Values.kafka.volumes.logs.snapshots.createFinalSnapshot }} + retentionDuration: {{ $root.Values.kafka.volumes.logs.snapshots.retentionDuration }} + schedule: {{ $root.Values.kafka.volumes.logs.snapshots.schedule }} +{{- end }} +--- +{{- $counter = add $counter 1 }} +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/workload-kafka-client.yaml b/kafka/versions/4.0.0/templates/workload-kafka-client.yaml new file mode 100644 index 00000000..08a0a78a --- /dev/null +++ b/kafka/versions/4.0.0/templates/workload-kafka-client.yaml @@ -0,0 +1,46 @@ +{{- if .Values.kafka_client }} +kind: workload +name: {{ include "kafka.name" . }}-{{ .Values.kafka_client.name }} +gvc: {{ .Values.global.cpln.gvc }} +spec: + type: standard + containers: + - name: kafka + args: + - '-c' + - sleep infinity + command: /bin/bash + cpu: {{ .Values.kafka_client.cpu }} + image: {{ .Values.kafka_client.image }} + inheritEnv: false + memory: {{ .Values.kafka_client.memory }} + ports: + - number: 9092 + protocol: tcp + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 3 + metric: cpu + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 +{{- if .Values.kafka_client.firewall }} + firewallConfig: + {{- if or (hasKey .Values.kafka_client.firewall "external_inboundAllowCIDR") (hasKey .Values.kafka_client.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.kafka_client.firewall.external_inboundAllowCIDR }}{{ .Values.kafka_client.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.kafka_client.firewall.external_outboundAllowCIDR }}{{ .Values.kafka_client.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.kafka_client.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.kafka_client.firewall.internal_inboundAllowType }} + {{- end }} +{{- end }} + localOptions: [] + supportDynamicTags: false +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/workload-kafka-cluster.yaml b/kafka/versions/4.0.0/templates/workload-kafka-cluster.yaml new file mode 100644 index 00000000..c35064e2 --- /dev/null +++ b/kafka/versions/4.0.0/templates/workload-kafka-cluster.yaml @@ -0,0 +1,353 @@ +{{- include "kafka.validateKafkaImage" . -}} +{{- include "kafka.validatedirectReplicaRoutingConfig" . -}} +{{- if .Values.jmx_exporter }} +kind: secret +name: {{ include "kafka.name" . }}-jmx-exporter-conf +type: opaque +data: + encoding: plain + payload: |- + {{ .Values.jmx_exporter.config | toYaml | nindent 4 }} +--- +{{- end }} +kind: workload +name: {{ include "kafka.clusterName" . }} +gvc: {{ .Values.global.cpln.gvc }} +tags: + {{- if .Values.kafka.deletionProtection }} + cpln/protected: true + {{- end }} + # KRaft brokers must resolve `.:9093` to find their peers and form + # the controller quorum on cold start, before any replica is Ready. This tag flips + # publishNotReadyAddresses=true on the headless Service so EndpointSlice exposes + # not-yet-Ready pods for peer DNS. Without it, suspend/unsuspend (or any cold + # start where the kafka-orchestrator readiness probe gates the workload-Ready + # signal) deadlocks: pods can't form a quorum because they can't resolve each + # other, and they can't become Ready until the quorum forms. + cpln/publishNotReadyAddresses: "true" + {{- include "kafka.tags" . | nindent 2 }} +spec: + type: stateful + containers: + - name: kafka + args: + - '-c' + - >- + cp /scripts/kafka-init.sh /tmp/ && chmod +x /tmp/kafka-init.sh && + exec /tmp/kafka-init.sh + command: /bin/bash + # Override Control Plane's default preStop hook (sleep for half the grace + # period to drain envoy connections). Kafka has no L7 connection drain to + # wait for — its clients reconnect to other brokers via Metadata refresh, + # and inter-broker traffic is handled by controlled.shutdown's leadership + # transfer. Sleeping here just delays SIGTERM and shortens the window + # available for the actual graceful shutdown. + lifecycle: + preStop: + exec: + command: ['true'] + cpu: '{{ .Values.kafka.cpu }}' + {{- if .Values.kafka.minCpu }} + minCpu: '{{ .Values.kafka.minCpu }}' + {{- end }} + env: + {{- if .Values.kafka.env }} +{{ toYaml .Values.kafka.env | indent 8 }} + {{- end }} + {{- if .Values.jmx_exporter }} + - name: JMX_PORT + value: {{ .Values.jmx_exporter.kafkaJmxPort | quote }} + {{- end }} + - name: KAFKA_CONTROLLER_PASSWORD + value: 'cpln://secret/{{ include "kafka.name" . }}-secrets.controller-password' + - name: KAFKA_CONTROLLER_USER + value: controller_user + - name: KAFKA_HEAP_OPTS + value: "{{ .Values.kafka.overrideHeapOpts | default (include "kafka.heap.opts" .) | trim }}" + - name: KAFKA_INTER_BROKER_PASSWORD + value: 'cpln://secret/{{ include "kafka.name" . }}-secrets.inter-broker-password' + - name: KAFKA_INTER_BROKER_USER + value: inter_broker_user + - name: KAFKA_KRAFT_BOOTSTRAP_SCRAM_USERS + value: 'true' + - name: CLUSTER_ID + value: 'cpln://secret/{{ include "kafka.name" . }}-secrets.kraft-cluster-id' + - name: KAFKA_MIN_ID + value: '0' + image: {{ .Values.kafka.image }} + inheritEnv: false + livenessProbe: + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 15 + successThreshold: 1 + tcpSocket: + port: 9093 + timeoutSeconds: 15 + memory: {{ .Values.kafka.memory }} + {{- if .Values.kafka.minMemory }} + minMemory: {{ .Values.kafka.minMemory }} + {{- end }} + ports: +{{ range $key, $listener := .Values.kafka.listeners }} +{{- include "kafka.validateListenerConfig" $listener }} +{{- if and $listener.directReplicaRouting $listener.directReplicaRouting.enabled }} + - number: {{ $listener.directReplicaRouting.containerPort }} + protocol: tcp +{{- else if $listener.publicAddress }} + {{- $startPort := 3000 }} + {{- $replicas := $.Values.kafka.replicas | int }} + {{- range $replicaIndex := until $replicas }} + - number: {{ add $startPort $replicaIndex }} + protocol: tcp + {{- end }} +{{- else }} + - number: {{ $listener.containerPort }} + protocol: tcp +{{- end }} +{{- end }} + - number: 9093 + protocol: tcp + - number: 9094 + protocol: tcp +{{- if .Values.jmx_exporter }} + - number: {{ .Values.jmx_exporter.kafkaJmxPort }} + protocol: tcp +{{- end }} + readinessProbe: + failureThreshold: 20 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 6 + tcpSocket: + port: 9093 + timeoutSeconds: 5 + volumes: + {{- $root := . -}} + {{- $logDirs := split "," $root.Values.kafka.logDirs }} + {{- $counter := 0 }} + {{- range $path := $logDirs }} + - path: {{ $path | trim }} + recoveryPolicy: retain + uri: 'cpln://volumeset/{{ include "kafka.name" $root }}-logs-{{ $counter }}' + {{- $counter = add $counter 1 }} + {{- end }} + - path: /configmaps/server.properties + recoveryPolicy: retain + uri: 'cpln://secret/{{ include "kafka.name" $root }}-controller-configuration' + - path: /scripts/kafka-init.sh + recoveryPolicy: retain + uri: 'cpln://secret/{{ include "kafka.name" $root }}-init' +{{- if .Values.kafka_orchestrator }} +{{- $orchestratorListenerName := .Values.kafka_orchestrator.listener }} +{{- if not (hasKey .Values.kafka.listeners $orchestratorListenerName) }} + {{- fail (printf "Error: Listener '%s' specified in kafka_orchestrator.listener does not exist" $orchestratorListenerName) }} +{{- end }} +{{- $orchestratorListener := index .Values.kafka.listeners $orchestratorListenerName }} + - name: kafka-orchestrator + image: {{ .Values.kafka_orchestrator.image }} + inheritEnv: false + # Suppress Control Plane's default preStop sleep (half of grace period). + # The orchestrator is a read-only health/metrics HTTP server; nothing to + # drain. Letting it exit immediately keeps pod termination time bounded + # by the kafka container's controlled.shutdown rather than 300s of idle. + lifecycle: + preStop: + exec: + command: ['true'] + cpu: {{ .Values.kafka_orchestrator.cpu }} + {{- if .Values.kafka_orchestrator.minCpu }} + minCpu: '{{ .Values.kafka_orchestrator.minCpu }}' + {{- end }} + memory: {{ .Values.kafka_orchestrator.memory }} + {{- if .Values.kafka_orchestrator.minMemory }} + minMemory: {{ .Values.kafka_orchestrator.minMemory }} + {{- end }} + ports: + - number: 8080 + protocol: http + metrics: + path: /metrics + port: 8080 + readinessProbe: + failureThreshold: 20 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + httpGet: + path: /health/ready + port: 8080 + timeoutSeconds: 5 + env: + {{- if .Values.kafka_orchestrator.env }} +{{ toYaml .Values.kafka_orchestrator.env | indent 8 }} + {{- end }} + - name: REPLICA_COUNT + value: '{{ .Values.kafka.replicas }}' + - name: KAFKA_PORT + value: '{{ $orchestratorListener.containerPort }}' + - name: LOG_LEVEL + value: '{{ .Values.kafka_orchestrator.logLevel | default "info" }}' + - name: CHECK_TIMEOUT + value: '{{ .Values.kafka_orchestrator.checkTimeout | default "10s" }}' + {{- if eq $orchestratorListener.protocol "SASL_PLAINTEXT" }} + - name: SASL_ENABLED + value: 'true' + - name: SASL_MECHANISM + value: PLAIN + - name: SASL_USERNAME + value: '{{ $orchestratorListener.sasl.admin.username }}' + - name: SASL_PASSWORD + value: 'cpln://secret/{{ include "kafka.name" . }}-secrets.{{ $orchestratorListener.name | lower }}-admin-password' + {{- end }} +{{- end }} +{{- if .Values.kafka_exporter }} + - name: kafka-exporter + args: + - '-c' + - >- +{{- $listenerName := .Values.kafka_exporter.listener }} +{{- if not (hasKey .Values.kafka.listeners $listenerName) }} + {{- fail (printf "Error: Listener '%s' specified in kafka_exporter.listener does not exist" $listenerName) }} +{{- end }} +{{- $listener := index .Values.kafka.listeners $listenerName }} +{{- $port := 3000 }} +{{- if $listener.containerPort }} + {{- $port = $listener.containerPort }} +{{- else }} + {{- $port = "$(echo $((3000 + $POD_ID)))" }} +{{- end }} +{{- if eq $listener.protocol "SASL_PLAINTEXT" }} + {{- if not (and $listener.sasl $listener.sasl.admin) }} + {{- fail (printf "Error: SASL_PLAINTEXT listener '%s' must have sasl.admin configured for kafka_exporter" $listenerName) }} + {{- end }} + sleep 60 && POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) && kafka_exporter --kafka.server=localhost:{{ if not $listener.containerPort }}$(echo $((3000 + $POD_ID))){{ else }}{{ $port }}{{ end }} + --sasl.enabled --sasl.username={{ $listener.sasl.admin.username }} --sasl.mechanism=plain + --sasl.password=${KAFKA_CLIENT_PASSWORDS} --web.listen-address=:9308 +{{- else if eq $listener.protocol "PLAINTEXT" }} + sleep 60 && POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) && kafka_exporter --kafka.server=localhost:{{ if not $listener.containerPort }}$(echo $((3000 + $POD_ID))){{ else }}{{ $port }}{{ end }} + --no-sasl.handshake --web.listen-address=:9308 +{{- else }} + sleep 60 && POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) && kafka_exporter --kafka.server=localhost:{{ if not $listener.containerPort }}$(echo $((3000 + $POD_ID))){{ else }}{{ $port }}{{ end }} + --no-sasl.handshake --web.listen-address=:9308 +{{- end }} + command: /bin/sh + cpu: {{ .Values.kafka_exporter.cpu }} + # Suppress Control Plane's default preStop sleep — prometheus-scrape + # sidecar with no L7 connections to drain. See kafka-orchestrator block + # for full reasoning. + lifecycle: + preStop: + exec: + command: ['true'] + metrics: + path: /metrics + port: 9308 + dropMetrics: {{- if .Values.kafka_exporter.dropMetrics }}{{ .Values.kafka_exporter.dropMetrics | toYaml | nindent 8 }}{{- else }} []{{- end }} + env: +{{- if .Values.kafka_exporter.env }} +{{ toYaml .Values.kafka_exporter.env | indent 8 }} +{{- end }} +{{- $listenerName := .Values.kafka_exporter.listener }} +{{- if not (hasKey .Values.kafka.listeners $listenerName) }} + {{- fail (printf "Error: Listener '%s' specified in kafka_exporter.listener does not exist" $listenerName) }} +{{- end }} +{{- $listener := index .Values.kafka.listeners $listenerName }} +{{- if eq $listener.protocol "SASL_PLAINTEXT" }} + - name: KAFKA_CLIENT_PASSWORDS + value: 'cpln://secret/{{ include "kafka.name" $ }}-secrets.{{ $listener.name | lower }}-admin-password' +{{- end }} + image: {{ .Values.kafka_exporter.image }} + inheritEnv: false + memory: {{ .Values.kafka_exporter.memory }} + ports: + - number: 9308 + protocol: tcp +{{- end }} +{{- if .Values.jmx_exporter }} + - name: jmx-exporter + command: java + args: + - -XX:MaxRAMPercentage=100 + - -XshowSettings:vm + - -jar + - jmx_prometheus_standalone.jar + - {{ .Values.jmx_exporter.exporterPort | quote }} + - /etc/jmx-kafka/jmx-kafka-prometheus.yml + cpu: {{ .Values.jmx_exporter.cpu }} + {{- if .Values.jmx_exporter.minCpu }} + minCpu: '{{ .Values.jmx_exporter.minCpu }}' + {{- end }} + # Suppress Control Plane's default preStop sleep — prometheus-scrape + # sidecar with no L7 connections to drain. See kafka-orchestrator block + # for full reasoning. + lifecycle: + preStop: + exec: + command: ['true'] + metrics: + path: /metrics + port: {{ .Values.jmx_exporter.exporterPort }} + dropMetrics: {{- if .Values.jmx_exporter.dropMetrics }}{{ .Values.jmx_exporter.dropMetrics | toYaml | nindent 8 }}{{- else }} []{{- end }} + image: {{ .Values.jmx_exporter.image }} + inheritEnv: false + memory: {{ .Values.jmx_exporter.memory }} + {{- if .Values.jmx_exporter.minMemory }} + minMemory: {{ .Values.jmx_exporter.minMemory }} + {{- end }} + ports: + - number: {{ .Values.jmx_exporter.exporterPort }} + protocol: tcp + volumes: + - path: /etc/jmx-kafka/jmx-kafka-prometheus.yml + recoveryPolicy: retain + uri: cpln://secret/{{ include "kafka.name" . }}-jmx-exporter-conf +{{- end }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: {{ .Values.kafka.replicas }} + metric: disabled + minScale: {{ .Values.kafka.replicas }} + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + {{- if .Values.kafka.multiZone }} + multiZone: + enabled: true + {{- else }} + multiZone: + enabled: false + {{- end }} + suspend: {{ .Values.kafka.suspend }} + timeoutSeconds: {{ .Values.kafka.terminationGracePeriodSeconds | default 600 }} +{{- if .Values.kafka.firewall }} + firewallConfig: + {{- if or (hasKey .Values.kafka.firewall "external_inboundAllowCIDR") (hasKey .Values.kafka.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.kafka.firewall.external_inboundAllowCIDR }}{{ .Values.kafka.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.kafka.firewall.external_outboundAllowCIDR }}{{ .Values.kafka.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.kafka.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.kafka.firewall.internal_inboundAllowType }} + {{- if .Values.kafka.firewall.inboundAllowWorkload }} + inboundAllowWorkload: {{ .Values.kafka.firewall.inboundAllowWorkload | toYaml | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + identityLink: //identity/{{ include "kafka.name" . }} + rolloutOptions: + maxSurgeReplicas: 25% + maxUnavailableReplicas: '1' + minReadySeconds: {{ .Values.kafka.minReadySeconds }} + scalingPolicy: Parallel + securityOptions: + filesystemGroupId: 1001 + supportDynamicTags: false \ No newline at end of file diff --git a/kafka/versions/4.0.0/templates/workload-kafka-ui.yaml b/kafka/versions/4.0.0/templates/workload-kafka-ui.yaml new file mode 100644 index 00000000..dc2726df --- /dev/null +++ b/kafka/versions/4.0.0/templates/workload-kafka-ui.yaml @@ -0,0 +1,66 @@ +{{- if and .Values.kafka_ui .Values.kafka_ui.enabled }} +kind: workload +name: {{ include "kafka.name" . }}-{{ .Values.kafka_ui.name }} +description: kafka-ui +gvc: {{ .Values.global.cpln.gvc }} +spec: + type: standard + containers: + - name: kafka-ui + cpu: {{ .Values.kafka_ui.cpu }} + env: + - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS + value: "{{- $replicas := int .Values.kafka.replicas -}}{{- $bootstrapServers := list -}}{{- range $i := until $replicas -}}{{- if $i -}},{{- end -}}{{- printf "%s-%s-%d.%s-%s:9092" $.Release.Name $.Values.kafka.name $i $.Release.Name $.Values.kafka.name -}}{{- end }}" + - name: KAFKA_CLUSTERS_0_NAME + value: {{ include "kafka.name" . }} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM + value: '' + - name: LOGGING_LEVEL_ROOT + value: INFO +{{- $listenerName := .Values.kafka_ui.listener }} +{{- $listener := index .Values.kafka.listeners $listenerName }} +{{- if eq $listener.protocol "SASL_PLAINTEXT" }} + {{- if not (and $listener.sasl $listener.sasl.admin) }} + {{- fail (printf "Error: SASL_PLAINTEXT listener '%s' must have sasl.admin configured for kafka_exporter" $listenerName) }} + {{- end }} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL + value: {{ $listener.protocol }} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM + value: PLAIN + - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG + value: >- + org.apache.kafka.common.security.plain.PlainLoginModule required username="{{ $listener.sasl.admin.username }}" password="{{ $listener.sasl.admin.password }}"; +{{- end }} + image: 'provectuslabs/kafka-ui:latest' + inheritEnv: false + memory: {{ .Values.kafka_ui.memory }} + ports: + - number: 8080 + protocol: http + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 1 + metric: cpu + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 +{{- if .Values.kafka_ui.firewall }} + firewallConfig: + {{- if or (hasKey .Values.kafka_ui.firewall "external_inboundAllowCIDR") (hasKey .Values.kafka_ui.firewall "external_outboundAllowCIDR") }} + external: + inboundAllowCIDR: {{- if .Values.kafka_ui.firewall.external_inboundAllowCIDR }}{{ .Values.kafka_ui.firewall.external_inboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + outboundAllowCIDR: {{- if .Values.kafka_ui.firewall.external_outboundAllowCIDR }}{{ .Values.kafka_ui.firewall.external_outboundAllowCIDR | splitList "," | toYaml | nindent 8 }}{{- else }} []{{- end }} + {{- end }} + {{- if hasKey .Values.kafka_ui.firewall "internal_inboundAllowType" }} + internal: + inboundAllowType: {{ default "[]" .Values.kafka_ui.firewall.internal_inboundAllowType }} + {{- end }} +{{- end }} + localOptions: [] + supportDynamicTags: false +{{- end }} \ No newline at end of file diff --git a/kafka/versions/4.0.0/values.yaml b/kafka/versions/4.0.0/values.yaml new file mode 100644 index 00000000..bbd3a797 --- /dev/null +++ b/kafka/versions/4.0.0/values.yaml @@ -0,0 +1,503 @@ +kafka: + name: cluster + image: apache/kafka:3.9.1 + suspend: false + deletionProtection: false + replicas: 3 # must not be 2 + minReadySeconds: 0 + debug: false + multiZone: false # If true: It's recommended to enable multi-zone on the Dedicated Load Balancer setting on GVC to reduce the cross-zone traffic + logDirs: /opt/kafka/logs-0,/opt/kafka/logs-1 + env: [] # If you need to set environment variables, add them here + # How long Control Plane waits for a graceful broker shutdown (controlled.shutdown completes, + # leadership transfers, log flushes finish) before SIGKILL. Brokers carrying large amounts + # of data benefit from a long grace window — set this comfortably above your largest broker's + # observed shutdown time. Default 600s (10m); raise for very large clusters. + terminationGracePeriodSeconds: 600 + volumes: + logs: + initialCapacity: 10 # In GB + performanceClass: general-purpose-ssd # general-purpose-ssd / high-throughput-ssd (Min 1000GB) + fileSystemType: ext4 # ext4 / xfs + snapshots: + createFinalSnapshot: true + retentionDuration: 7d + schedule: 0 0 * * * # UTC + autoscaling: + maxCapacity: 1000 # In GB + minFreePercentage: 20 + scalingFactor: 1.2 + # customEncryption: + # enabled: false + # region: aws-us-east-2 # Replace with the appropriate region + # keyId: arn:aws:kms:us-east-2:1234567890:key/d411f35a-1d31-4515-9934-4f193e042d80 # Replace with your AWS KMS key ARN + cpu: 1000m # For millicores us 'm' like 500m + memory: 2000Mi # Gi / Mi + minCpu: 250m # For millicores us 'm' like 500m + minMemory: 2000Mi # Gi / Mi + # overrideHeapOpts: "-Xmx1024m -Xms1024m" # Override the default heap Options settings + # To disable all traffic, comment out the corresponding rule. Docs: https://docs.controlplane.com/concepts/security#firewall + firewall: + internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc(Recommended) + # external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list + # # You can specify additional workloads with either same-gvc or workload-list: + # inboundAllowWorkload: + # - //gvc/main-kafka/workload/main-kafka-kafbat-ui + # - //gvc/client-gvc/workload/client + # external_outboundAllowCIDR: "111.222.333.444/16,111.222.444.333/32" # Provide a comma-separated list + listeners: + # @param listeners.client.name Name for the Kafka client listener + # @param listeners.client.containerPort Port for the Kafka client listener. Except ports 9091,9093,9094 + # @param listeners.client.protocol Security protocol for the Kafka client listener. Allowed values are 'PLAINTEXT', 'SASL_PLAINTEXT' + # @param listeners.client.publicAddress DNS address for public access to brokers. Must be the same as kafka.replicas + client: + protocol: SASL_PLAINTEXT + name: CLIENT + containerPort: 9092 # If publicAddress is enabled, Client automatically set to port range 3000-3004 + sasl: + ## @param listeners.client.sasl.users Comma-separated list of usernames for client communications when SASL is enabled + ## @param listeners.client.passwords Comma-separated list of passwords for client communications when SASL is enabled, must match the number of client.sasl.users + ## @param listeners.client.admin Admin username and password for client communications when SASL is enabled + admin: + username: admin + password: "your-admin-password" + users: "user" + passwords: "your-user-password" + # public: + # protocol: SASL_PLAINTEXT # TLS enforced, Kafka clients should use SASL_SSL to access 'publicAddress' if provided + # name: PUBLIC + # # containerPort: 9095 # Uncomment only when no directReplicaRouting or publicAddress is provided + # # Use directReplicaRouting for automatic public replica endpoints with DNS01 cert challenge + # directReplicaRouting: + # enabled: true + # containerPort: 9095 # ports 9093 and 9094 are reserved for controller and inter-broker communication + # publicAddress: kafka.example.com # Make sure Dedicate Load Balancer is enabled on the GVC + # sasl: + # ## @param listeners.client.sasl.users Comma-separated list of usernames for client communications when SASL is enabled + # ## @param listeners.client.brokersAddresses Comma-separated list of passwords for client communications when SASL is enabled, must match the number of client.sasl.users + # ## @param listeners.client.admin Admin username and password for client communications when SASL is enabled + # # admin: + # # username: admin + # # password: tgtgtg + # users: "public-user" + # passwords: "your-public-user-password" + acl: + superUsers: "User:admin" # User:admin;User:connectors (for multiple users) + allowEveryoneIfNoAclFound: false + secrets: + kraft_cluster_id: your-kraft-cluster-id # Example:bkdDtS1Rsf536si7BGM0JY + inter_broker_password: your-inter-broker-password # Example: HfcgCHp32e + controller_password: your-controller-password # Example: ayd8iJwqXe + extra_configurations: + # default.replication.factor and min.insync.replicas are auto-derived from kafka.replicas + # in the chart (defaults to min(3, replicas) and rf-1 respectively). Set them here only + # if you want to override. + auto.create.topics.enable: true # auto.create.topics.enable + log.retention.hours: 168 # The number of hours to keep a log file before deleting it (in hours) + +# Sidecar that exposes /health/ready, /health/live, and Prometheus metrics for the Kafka brokers. +# Set to null (or comment the block out) to disable the sidecar. +kafka_orchestrator: + image: ghcr.io/controlplane-com/kafka-orchestrator:v0.1.0 + cpu: 100m + memory: 128Mi + minCpu: 50m + minMemory: 64Mi + listener: client # name of an entry under kafka.listeners; the sidecar uses this listener's port and SASL config for health checks + logLevel: info # debug / info / warn / error + checkTimeout: 10s # per-check timeout used by the orchestrator's franz-go client + env: [] # extra env vars (e.g. BROKER_ID / BOOTSTRAP_SERVERS overrides for non-standard setups) + +kafka_exporter: + name: exporter + image: danielqsj/kafka-exporter:v1.9.0 + debug: false + cpu: 50m + memory: 128Mi + listener: client + env: [] # If you need to set environment variables, add them here + dropMetrics: [] # e.g., ["kafka_consumergroup.*", "^kafka_topic_partition_current_offset"] + +jmx_exporter: + name: jmx-exporter + image: ghcr.io/controlplane-com/bitnami/jmx-exporter + kafkaJmxPort: 5557 # Ensure this port matches the port in the jmxUrl below + exporterPort: 5556 + debug: false + cpu: 250m + memory: 256Mi + minCpu: 80m + minMemory: 125Mi + listener: client + dropMetrics: [] # e.g., ["kafka_consumergroup.*", "^kafka_topic_partition_current_offset"] + config: + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5557/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false + whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* + - kafka.producer:* + - kafka.consumer:* + rules: + - labels: + request: "$3" + name: kafka_request_count + pattern: kafka.network<>(Count) + - labels: + request: "$3" + stat: "$4" + name: kafka_request_metrics_totaltimems + pattern: kafka.network<>(.+) + - labels: + request: "$3" + component: "$2" + stat: "$4" + name: kafka_request_latency_ms + pattern: kafka.network<>(.+) + - labels: + client_type: "$3" + metric: "$2" + stat: "$4" + name: kafka_client_metrics + pattern: kafka.network<>(.+) + - labels: + client_id: "$1" + metric: "$2" + name: kafka_consumer_metrics + pattern: kafka.consumer<>(.+) + - labels: + client_id: "$1" + metric: "$2" + name: kafka_producer_metrics + pattern: kafka.producer<>(.+) + - name: kafka_server_$1_$2_$3 + pattern: kafka.server<>(Count|Value) + - name: java_lang_$1_$2 + pattern: java.lang<>(.+) + +kafbat_ui: + enabled: true + deletionProtection: false + name: kafbat-ui + image: ghcr.io/kafbat/kafka-ui + cpu: 300m + memory: 1000Mi + minCpu: 100m + minMemory: 400Mi + replicas: 1 + timeoutSeconds: 30 + configuration_secret: kafka-kafbat-ui-config # Pre-create a secret with the configuration; Example in README + # Domain name for the UI. + # Make sure the required DNS records are created in your DNS server + # https://docs.controlplane.com/guides/configure-domain#subdomain-e-g-sample-domain-com-cname-mode-path-based-routing + # domain: kafbat-ui.example.com # Domain name for the UI. + # To disable all traffic, comment out the corresponding rule. Docs: https://docs.controlplane.com/concepts/security#firewall + firewall: + # internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc + external_inboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list + external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list + +# kafka_connectors: +# - name: cluster +# image: apache/kafka:3.9.1 +# multiZone: true +# cpu: 400m +# memory: 1500Mi +# minCpu: 100m +# minMemory: 375Mi +# plugins_folder: /opt/kafka/plugins +# timeoutSeconds: 15 +# replicas: 1 +# verbose: false +# extraVolumes: [] +# # Volume configuration for Kafka Connect (Optional - defaults shown below) +# volumes: +# initialCapacity: 10 # In GB (Default: 10) +# performanceClass: general-purpose-ssd # general-purpose-ssd / high-throughput-ssd (Min 1000GB) (Default: general-purpose-ssd) +# fileSystemType: ext4 # ext4 / xfs (Default: ext4) +# snapshots: +# createFinalSnapshot: true # Default: true +# retentionDuration: 7d # Default: 7d +# # schedule: 0 0 * * * # UTC (Optional) +# # customEncryption: +# # enabled: true # Encrypting is only possible for new volumes. Existing volumes cannot be re-encrypted after creation. +# # region: aws-us-east-2 # Replace with the appropriate region +# # keyId: arn:aws:kms:us-east-2:1234567890:key/fewf2f43-1d31-2332-9934-efhg4334gfe # Replace with your AWS KMS key ARN +# env: +# - name: KAFKA_HEAP_OPTS +# value: '-Xms900m -Xmx900m' # set to 50%-75% of the memory +# # To disable all traffic, comment out the corresponding rule. Docs: https://docs.controlplane.com/concepts/security#firewall +# firewall: +# external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list +# internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc +# # You can specify additional workloads with either same-gvc or workload-list: +# inboundAllowWorkload: +# - //gvc/main-kafka/workload/main-kafka-kafbat-ui +# - //gvc/client-gvc/workload/client +# external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list +# listener: client # Provide the listener name to connect to +# connector_properties: +# # bootstrap.servers: "kafka-dev-cluster:9092" # Optional. If not set, the bootstrap address will be the cluster name or publicAddress +# group.id: "connect-cluster" +# security.protocol: "SASL_PLAINTEXT" +# sasl.mechanism: "PLAIN" +# sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"your-admin-password\";" +# consumer.security.protocol: "SASL_PLAINTEXT" +# consumer.sasl.mechanism: "PLAIN" +# consumer.sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"your-admin-password\";" +# producer.security.protocol: "SASL_PLAINTEXT" +# producer.sasl.mechanism: "PLAIN" +# producer.sasl.jaas.config: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"your-admin-password\";" +# key.converter.schemas.enable: "false" +# value.converter.schemas.enable: "false" +# offset.storage.topic: "connect-offsets" +# offset.storage.replication.factor: "3" +# config.storage.topic: "connect-configs" +# config.storage.replication.factor: "3" +# status.storage.topic: "connect-status" +# status.storage.replication.factor: "3" +# offset.flush.interval.ms: "10000" +# plugin.path: "/opt/kafka/plugins" +# key.converter: "org.apache.kafka.connect.storage.StringConverter" +# value.converter: "org.apache.kafka.connect.converters.ByteArrayConverter" +# plugins: +# - name: kafka-mirror-test-1 +# enabled: true +# config: +# connector.class: org.apache.kafka.connect.mirror.MirrorSourceConnector +# tasks.max: '1' +# offset-syncs.topic.location: source +# source.cluster.alias: remote +# target.cluster.alias: local +# source.bootstrap.servers: kafka-dev-cluster:9092 +# target.bootstrap.servers: kafka-dev-cluster:9092 +# source.consumer.bootstrap.servers: kafka-dev-cluster:9092 +# target.consumer.bootstrap.servers: kafka-dev-cluster:9092 +# source.producer.bootstrap.servers: kafka-dev-cluster:9092 +# target.producer.bootstrap.servers: kafka-dev-cluster:9092 +# source.admin.bootstrap.servers: kafka-dev-cluster:9092 +# target.admin.bootstrap.servers: kafka-dev-cluster:9092 +# replication.policy.class: org.apache.kafka.connect.mirror.DefaultReplicationPolicy +# topics: mirror-test-1-a,mirror-test-1-b +# groups: .* +# sync.topic.configs.enabled: 'true' +# sync.topic.acls.enabled: 'false' +# refresh.topics.interval.seconds: '60' +# refresh.groups.interval.seconds: '60' +# replication.factor: '3' +# offset.syncs.topic.replication.factor: '3' +# checkpoints.topic.replication.factor: '3' +# heartbeats.topic.replication.factor: '3' +# source.consumer.security.protocol: SASL_PLAINTEXT +# source.consumer.sasl.mechanism: PLAIN +# source.consumer.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# source.producer.security.protocol: SASL_PLAINTEXT +# source.producer.sasl.mechanism: PLAIN +# source.producer.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# target.producer.security.protocol: SASL_PLAINTEXT +# target.producer.sasl.mechanism: PLAIN +# target.producer.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# target.consumer.security.protocol: SASL_PLAINTEXT +# target.consumer.sasl.mechanism: PLAIN +# target.consumer.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# admin.security.protocol: SASL_PLAINTEXT +# admin.sasl.mechanism: PLAIN +# admin.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# consumer.security.protocol: SASL_PLAINTEXT +# consumer.sasl.mechanism: PLAIN +# consumer.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# producer.security.protocol: SASL_PLAINTEXT +# producer.sasl.mechanism: PLAIN +# producer.sasl.jaas.config: >- +# org.apache.kafka.common.security.plain.PlainLoginModule required +# username='admin' password='your-admin-password'; +# - name: "camel-s3-sink" +# enabled: true +# artifacts: +# - type: tgz +# url: https://repo.maven.apache.org/maven2/org/apache/camel/kafkaconnector/camel-aws-s3-sink-kafka-connector/4.8.5/camel-aws-s3-sink-kafka-connector-4.8.5-package.tar.gz +# config: +# "connector.class": "org.apache.camel.kafkaconnector.awss3sink.CamelAwss3sinkSinkConnector" +# "tasks.max": "1" +# "topics": "your-topic" +# "camel.kamelet.aws-s3-sink.useSessionCredentials": "false" +# "camel.kamelet.aws-s3-sink.bucketNameOrArn": "your-bucket-name" +# "camel.kamelet.aws-s3-sink.keyName": "your-topic-sink-${exchangeId}.txt" +# "camel.kamelet.aws-s3-sink.region": "your-region" +# "camel.kamelet.aws-s3-sink.autoCreateBucket": "true" +# "camel.kamelet.aws-s3-sink.accessKey": "your-access-key" +# "camel.kamelet.aws-s3-sink.secretKey": "your-secret-key" +# - name: "clickhouse-sink" +# enabled: true +# ssl_truststore: +# generate: true +# truststore_path: /tmp/kafka.autogenerated.truststore.jks +# truststore_password_env: "SSL_CLICKHOUSE_SINK_TRUSTSTORE_PASSWORD" +# hostnames: +# - domain1.clickhouse-sink.com +# - domain2.clickhouse-sink.com +# artifacts: +# - type: zip +# url: https://github.com/ClickHouse/clickhouse-kafka-connect/releases/download/v1.2.8/clickhouse-kafka-connect-v1.2.8.zip +# config: +# "connector.class": "com.clickhouse.kafka.connect.ClickHouseSinkConnector" +# "tasks.max": "1" +# "topics": "your-topic" +# "security.protocol": "SASL_PLAINTEXT" # Connect to Kafka cluster using PLAINTEXT protocol - Internal connection mTLS encrypted +# "hostname": "your-hostname" +# "username": "your-username" +# "database": "your-database" +# "password": "your-password" +# "port": "8443" +# "value.converter.schemas.enable": "false" +# "ssl": "true" # Connect to ClickHouse using SSL protocol +# "value.converter": "org.apache.kafka.connect.json.JsonConverter" +# "key.converter": "org.apache.kafka.connect.storage.StringConverter" +# "errors.retry.timeout": "30" +# "schemas.enable": "false" +# "jdbcConnectionProperties": "?sslmode=STRICT" +# "ssl.truststore.location": "/tmp/kafka.autogenerated.truststore.jks" +# "ssl.truststore.password": "${SSL_CLICKHOUSE_SINK_TRUSTSTORE_PASSWORD}" +# "errors.tolerance": "all" +# "errors.log.enable": "true" +# "errors.log.include.messages": "true" +# - name: "snowflake-sink" +# enabled: true +# artifacts: +# - type: jar +# url: https://repo1.maven.org/maven2/com/snowflake/snowflake-kafka-connector/3.1.1/snowflake-kafka-connector-3.1.1.jar +# - type: jar +# url: https://repo1.maven.org/maven2/org/bouncycastle/bc-fips/2.1.0/bc-fips-2.1.0.jar +# - type: jar +# url: https://repo1.maven.org/maven2/org/bouncycastle/bcpkix-fips/2.1.9/bcpkix-fips-2.1.9.jar +# config: +# "connector.class": "com.snowflake.kafka.connector.SnowflakeSinkConnector" +# "tasks.max": "1" +# "topics": "your-topic" +# "key.converter": "org.apache.kafka.connect.storage.StringConverter" +# "value.converter": "com.snowflake.kafka.connector.records.SnowflakeJsonConverter" +# "value.converter.schemas.enable": "false" +# "security.protocol": "SASL_PLAINTEXT" # Connect to Kafka cluster using SASL_PLAINTEXT protocol - Internal connection mTLS encrypted +# "snowflake.url.name": "your-snowflake-url" +# "snowflake.user.name": "your-snowflake-username" +# "snowflake.private.key": "your-snowflake-private-key" +# "snowflake.private.key.passphrase": "your-snowflake-private-key-passphrase" +# "snowflake.warehouse.name": "your-snowflake-warehouse-name" +# "snowflake.database.name": "your-snowflake-database-name" +# "snowflake.schema.name": "your-snowflake-schema-name" +# "snowflake.topic2table.map": "your-topic:your-table" +# "snowflake.role.name": "your-snowflake-role-name" +# "snowflake.enable.schematization": "false" +# "snowflake.disable.ssl.certificate.verification": "true" +# "snowflake.log.enable": "true" +# "snowflake.log.level": "DEBUG" +# "buffer.count.records": "10000" +# "buffer.flush.time": "120" +# "buffer.size.bytes": "10000000" +# "errors.tolerance": "all" +# "errors.log.enable": "true" +# "errors.log.include.messages": "true" + +kafka_rest_proxy: + enabled: true + deletionProtection: false + name: rest-proxy + image: confluentinc/cp-kafka-rest:latest + cpu: 500m + memory: 1000Mi + capacityAI: + enabled: true + minCpu: 125m # This only applied when capacityAI is enabled + minMemory: 200Mi # This only applied when capacityAI is enabled + replicas: 1 + timeoutSeconds: 15 + # domain: kafka-rest.example.com # Domain name for the Kafka Rest Proxy. + + # To disable all traffic, comment out the corresponding rule. Docs: https://docs.controlplane.com/concepts/security#firewall + firewall: + # internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc(Recommended) + external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list + # # You can specify additional workloads with either same-gvc or workload-list: + # inboundAllowWorkload: + # - //gvc/main-kafka/workload/main-kafka-kafbat-ui + # - //gvc/client-gvc/workload/client + external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list + properties: + # host.name: kafka-rest.example.com + bootstrap.servers: SASL_PLAINTEXT://kafka-dev-cluster:9092 + resource.extension: ALL + api.v3.enable: true + api.v2.enable: true + client.sasl.mechanism: PLAIN + api.compatibility.mode: BOTH + log4j.opts: -Dlog4j.configuration=file:/tmp/log4j.properties + listeners: http://0.0.0.0:8082 + authentication.realm: KafkaRest + authentication.method: BASIC + authentication.roles: user + client.security.protocol: SASL_PLAINTEXT + + # JAAS configuration for Kafka client and Kafka Rest Proxy + # https://docs.confluent.io/platform/current/kafka-rest/production-deployment/confluent-server/security.html#authentication-between-the-admin-rest-and-ak-brokers + jaas_conf: + KafkaClient { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="admin" + password="your-admin-password"; + }; + KafkaRest { + org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required + debug="true" + file="/etc/kafka-rest/password.properties"; + }; + + # # Password properties for Kafka Rest Proxy + # # Required when authentication.method is set to BASIC + # # https://docs.confluent.io/platform/current/kafka-rest/production-deployment/confluent-server/security.html#password-properties + password_properties: + user: your-user-password,user + user1: password213,user + user2: password214,user + +kafka_client: + name: client + image: apache/kafka:3.9.1 + cpu: 500m + memory: 1000Mi + # To disable all traffic, comment out the corresponding rule. Docs: https://docs.controlplane.com/concepts/security#firewall + firewall: + # internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc + # external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list + external_outboundAllowCIDR: "0.0.0.0/0" # Provide a comma-separated list + +# DEPRECATED NOTICE https://github.com/provectus/kafka-ui +# PLEASE USE KAFBAT UI INSTEAD +kafka_ui: + enabled: false + name: ui + image: provectuslabs/kafka-ui:latest + cpu: 200m + memory: 600Mi + listener: client + # To disable all traffic, comment out the corresponding rule. Docs: https://docs.controlplane.com/concepts/security#firewall + firewall: {} + # internal_inboundAllowType: "same-gvc" # Options: same-org / same-gvc + # external_inboundAllowCIDR: 0.0.0.0/0 # Provide a comma-separated list + # external_outboundAllowCIDR: "111.222.333.444/16,111.222.444.333/32" # Provide a comma-separated list From 5b7540beeecfa7ba4dc2b5d2c4f7ef63059ac25d Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Thu, 7 May 2026 09:09:42 -0400 Subject: [PATCH 34/58] redis updates - improve resilience in v3.2.0 - v3.3.0 is not v3.2.0 + master discovery on startup for redis replicas. --- .../3.2.0/templates/workload-redis.yaml | 12 +- .../3.2.0/templates/workload-sentinel.yaml | 95 +++++--- redis/versions/3.2.0/values.yaml | 3 + redis/versions/3.3.0/Chart.yaml | 9 +- redis/versions/3.3.0/templates/_helpers.tpl | 28 ++- .../3.3.0/templates/workload-redis.yaml | 220 ++++++++---------- .../3.3.0/templates/workload-sentinel.yaml | 99 +++++--- redis/versions/3.3.0/values.yaml | 3 + 8 files changed, 273 insertions(+), 196 deletions(-) diff --git a/redis/versions/3.2.0/templates/workload-redis.yaml b/redis/versions/3.2.0/templates/workload-redis.yaml index 5142d35e..43167f17 100644 --- a/redis/versions/3.2.0/templates/workload-redis.yaml +++ b/redis/versions/3.2.0/templates/workload-redis.yaml @@ -64,15 +64,19 @@ spec: echo "\nreplica-announce-port 6379" >> /etc/redis/redis.conf {{ end }} + # exec replaces the shell with redis-server so SIGTERM from the + # platform routes directly to redis-server instead of being swallowed + # by /bin/sh. Without exec, shutdown waits the full grace period + # before SIGKILL since the shell doesn't forward signals to children. if [ "$(hostname)" = "{{ include "redis.name" . }}-0" ]; then - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} + exec {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} else {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof {{ .Values.redis.publicAccess.address }} 6380 + exec {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof {{ .Values.redis.publicAccess.address }} 6380 {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof replica-0.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 + exec {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof replica-0.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 {{- else }} - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 + exec {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 {{- end }} fi command: /bin/sh diff --git a/redis/versions/3.2.0/templates/workload-sentinel.yaml b/redis/versions/3.2.0/templates/workload-sentinel.yaml index 6373c186..82ab5787 100644 --- a/redis/versions/3.2.0/templates/workload-sentinel.yaml +++ b/redis/versions/3.2.0/templates/workload-sentinel.yaml @@ -26,43 +26,72 @@ spec: {{- else }} mkdir -p /etc/sentinel {{- end }} - cp /config/sentinel.conf /etc/sentinel/sentinel.conf - if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then - echo "\nsentinel auth-pass mymaster $CUSTOM_REDIS_PASSWORD" >> /etc/sentinel/sentinel.conf - fi + # bootstrap_sentinel_conf: write /etc/sentinel/sentinel.conf only if + # sentinel hasn't already taken ownership of the file. The marker we + # check is the `sentinel myid ` directive — sentinel writes this + # via CONFIG REWRITE within milliseconds of its first successful + # startup, and the chart's static template never contains it. So: + # - marker present → previous boot got far enough that sentinel + # owns the file. Preserve everything (current master after any + # failover, known replicas, peer sentinels). + # - marker absent → file missing, empty, or written but sentinel + # never reached its first CONFIG REWRITE. Re-run bootstrap; + # idempotent against the static config so safe to re-run. + # When sentinel.persistence is disabled this still runs every boot + # (ephemeral filesystem, no marker survives) — same behavior as the + # pre-marker chart. The check only changes behavior when /etc/sentinel + # is backed by a persistent volume. + bootstrap_sentinel_conf() { + if [ -s /etc/sentinel/sentinel.conf ] && grep -q "^sentinel myid " /etc/sentinel/sentinel.conf; then + echo "sentinel.conf already bootstrapped; preserving sentinel-managed state" + return 0 + fi + echo "Bootstrapping sentinel.conf from static config" + cp /config/sentinel.conf /etc/sentinel/sentinel.conf - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - echo "\nrequirepass $CUSTOM_SENTINEL_PASSWORD" >> /etc/sentinel/sentinel.conf - fi + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + echo "\nsentinel auth-pass mymaster $CUSTOM_REDIS_PASSWORD" >> /etc/sentinel/sentinel.conf + fi - {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} - POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) - PORT=$((26380 + POD_ID)) - echo "\nport $PORT" >> /etc/sentinel/sentinel.conf - echo "\nsentinel announce-ip {{ .Values.sentinel.publicAccess.address }}" >> /etc/sentinel/sentinel.conf - echo "\nsentinel announce-port $PORT" >> /etc/sentinel/sentinel.conf - {{ else }} - echo "\nport 26379" >> /etc/sentinel/sentinel.conf - POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) - LOCATION=${CPLN_LOCATION##*/} - CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" - if [ -n "$REPLICA_DIRECT" ]; then - echo "\nsentinel announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/sentinel/sentinel.conf - else - echo "\nsentinel announce-ip ${HOSTNAME}.{{ include "redis.sentinel.name" . }}" >> /etc/sentinel/sentinel.conf - fi - echo "\nsentinel announce-port 26379" >> /etc/sentinel/sentinel.conf - {{ end }} - {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} - echo "sentinel monitor mymaster {{ .Values.redis.publicAccess.address }} 6380 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf - {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} - echo "sentinel monitor mymaster replica-0.{{ include "redis.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf - {{- else }} - echo "sentinel monitor mymaster {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf - {{- end }} + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + echo "\nrequirepass $CUSTOM_SENTINEL_PASSWORD" >> /etc/sentinel/sentinel.conf + fi + + {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((26380 + POD_ID)) + echo "\nport $PORT" >> /etc/sentinel/sentinel.conf + echo "\nsentinel announce-ip {{ .Values.sentinel.publicAccess.address }}" >> /etc/sentinel/sentinel.conf + echo "\nsentinel announce-port $PORT" >> /etc/sentinel/sentinel.conf + {{ else }} + echo "\nport 26379" >> /etc/sentinel/sentinel.conf + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + LOCATION=${CPLN_LOCATION##*/} + CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" + if [ -n "$REPLICA_DIRECT" ]; then + echo "\nsentinel announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/sentinel/sentinel.conf + else + echo "\nsentinel announce-ip ${HOSTNAME}.{{ include "redis.sentinel.name" . }}" >> /etc/sentinel/sentinel.conf + fi + echo "\nsentinel announce-port 26379" >> /etc/sentinel/sentinel.conf + {{ end }} + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + echo "sentinel monitor mymaster {{ .Values.redis.publicAccess.address }} 6380 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} + echo "sentinel monitor mymaster replica-0.{{ include "redis.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- else }} + echo "sentinel monitor mymaster {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- end }} + } + + bootstrap_sentinel_conf - redis-sentinel /etc/sentinel/sentinel.conf + # exec replaces the shell with redis-sentinel so SIGTERM from the + # platform routes directly to redis-sentinel for clean shutdown. + # Without exec, the shell swallows the signal and sentinel only dies + # when the platform sends SIGKILL after the grace period expires. + exec redis-sentinel /etc/sentinel/sentinel.conf command: /bin/sh cpu: {{ .Values.sentinel.resources.cpu }} memory: {{ .Values.sentinel.resources.memory }} diff --git a/redis/versions/3.2.0/values.yaml b/redis/versions/3.2.0/values.yaml index 737dda09..28adcd93 100644 --- a/redis/versions/3.2.0/values.yaml +++ b/redis/versions/3.2.0/values.yaml @@ -120,6 +120,9 @@ sentinel: # - resource-exhausted # - retriable-status-codes requestRetryPolicy: {} + # If all sentinels are lost AND the topology changed underfoot during the + # outage, persisted `mymaster` survives but disagrees with the chart's + # redis-0-as-default-master bootstrap, deadlocking recovery. Default OFF. persistence: enabled: false volumes: diff --git a/redis/versions/3.3.0/Chart.yaml b/redis/versions/3.3.0/Chart.yaml index 3eb414a2..5a24875d 100644 --- a/redis/versions/3.3.0/Chart.yaml +++ b/redis/versions/3.3.0/Chart.yaml @@ -5,13 +5,8 @@ type: application version: 3.3.0 appVersion: "7.4" -dependencies: - - name: cpln-common - version: 1.0.0 - repository: "oci://ghcr.io/controlplane-com/templates" - annotations: created: "2026-01-29" - lastModified: "2026-05-01" + lastModified: "2026-05-07" category: "cache" - createsGvc: false \ No newline at end of file + createsGvc: false diff --git a/redis/versions/3.3.0/templates/_helpers.tpl b/redis/versions/3.3.0/templates/_helpers.tpl index 10f8995b..93eed473 100644 --- a/redis/versions/3.3.0/templates/_helpers.tpl +++ b/redis/versions/3.3.0/templates/_helpers.tpl @@ -236,8 +236,32 @@ Validate auth configuration block {{/* Labeling */}} {{/* -Common labels - delegated to cpln-common +Create chart name and version as used by the chart label. +*/}} +{{- define "redis.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels */}} {{- define "redis.tags" -}} -{{- include "cpln-common.tags" . }} +helm.sh/chart: {{ include "redis.chart" . }} +{{ include "redis.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.cpln.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.cpln.io/managed-by: {{ .Release.Service }} +cpln/marketplace: "true" +cpln/marketplace-template: redis +cpln/marketplace-template-version: {{ .Chart.Version }} +cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "redis.selectorLabels" -}} +app.cpln.io/name: {{ .Release.Name }} +app.cpln.io/instance: {{ .Release.Name }} {{- end }} \ No newline at end of file diff --git a/redis/versions/3.3.0/templates/workload-redis.yaml b/redis/versions/3.3.0/templates/workload-redis.yaml index c75e6b98..d7302624 100644 --- a/redis/versions/3.3.0/templates/workload-redis.yaml +++ b/redis/versions/3.3.0/templates/workload-redis.yaml @@ -38,8 +38,7 @@ spec: - name: CUSTOM_SENTINEL_PASSWORD value: cpln://secret/{{ include "redis.sentinelSecretPassword.name" . }}.password {{- end }} - {{- $hasSentinelAuth := or (and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled) (and (hasKey .Values.sentinel "auth") (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled) }} - {{- if not (or .Values.redis.env .Values.redis.replicaDirect (and (hasKey .Values.redis "auth") (or (and (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled) (and (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled))) $hasSentinelAuth) }} + {{- if not (or .Values.redis.env .Values.redis.replicaDirect (and (hasKey .Values.redis "auth") (or (and (hasKey .Values.redis.auth "fromSecret") .Values.redis.auth.fromSecret.enabled) (and (hasKey .Values.redis.auth "password") .Values.redis.auth.password.enabled))) (and (hasKey .Values.sentinel "auth") (or (and (hasKey .Values.sentinel.auth "fromSecret") .Values.sentinel.auth.fromSecret.enabled) (and (hasKey .Values.sentinel.auth "password") .Values.sentinel.auth.password.enabled)))) }} [] {{- end }} args: @@ -59,51 +58,77 @@ spec: echo "\nport $PORT" >> /etc/redis/redis.conf echo "\nreplica-announce-ip {{ .Values.redis.publicAccess.address }}" >> /etc/redis/redis.conf echo "\nreplica-announce-port $PORT" >> /etc/redis/redis.conf + SELF_HOST="{{ .Values.redis.publicAccess.address }}" + SELF_PORT=$PORT {{ else }} + PORT=6379 echo "\nport 6379" >> /etc/redis/redis.conf POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) LOCATION=${CPLN_LOCATION##*/} CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" if [ -n "$REPLICA_DIRECT" ]; then echo "\nreplica-announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/redis/redis.conf + SELF_HOST="replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" else echo "\nreplica-announce-ip ${HOSTNAME}.{{ include "redis.name" . }}" >> /etc/redis/redis.conf + SELF_HOST="${HOSTNAME}.{{ include "redis.name" . }}" fi echo "\nreplica-announce-port 6379" >> /etc/redis/redis.conf + SELF_PORT=6379 {{ end }} - if [ "$(hostname)" = "{{ include "redis.name" . }}-0" ]; then - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} + # Discovery: ask sentinel who the master is, retrying forever. A + # redis cluster without reachable sentinels isn't manageable + # regardless, so blocking startup until they respond is acceptable. + # On cold start, sentinels respond from their static config + # (mymaster -> pod-0); after any failover with sentinel persistence + # enabled, they respond from CONFIG REWRITE state (the real master). + # The pod whose announced identity matches sentinel's answer boots + # as master; everyone else replicaof's it. + SENTINEL_REPLICAS={{ .Values.sentinel.replicas }} + SIDX=0 + MASTER_HOST="" + MASTER_PORT="" + until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do + {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + S_HOST="{{ include "redis.sentinel.name" . }}-${SIDX}.{{ include "redis.sentinel.name" . }}" + S_PORT=$((26380 + SIDX)) + {{- else if and (hasKey .Values.sentinel "replicaDirect") .Values.sentinel.replicaDirect }} + S_HOST="replica-${SIDX}.{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" + S_PORT=26379 + {{- else }} + S_HOST="{{ include "redis.sentinel.name" . }}-${SIDX}.{{ include "redis.sentinel.name" . }}" + S_PORT=26379 + {{- end }} + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + INFO=$(redis-cli -h "$S_HOST" -p "$S_PORT" --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + else + INFO=$(redis-cli -h "$S_HOST" -p "$S_PORT" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) + fi + MASTER_HOST=$(echo "$INFO" | head -1) + MASTER_PORT=$(echo "$INFO" | tail -1) + if ! echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; then + echo "Sentinel at $S_HOST:$S_PORT not responding; trying next" + SIDX=$(( (SIDX + 1) % SENTINEL_REPLICAS )) + [ $SIDX -eq 0 ] && sleep 5 + fi + done + echo "Sentinel says master=$MASTER_HOST:$MASTER_PORT (self=$SELF_HOST:$SELF_PORT)" + + # exec replaces the shell with redis-server so SIGTERM from the + # platform routes directly to redis-server instead of being swallowed + # by /bin/sh. Without exec, shutdown waits the full grace period + # before SIGKILL since the shell doesn't forward signals to children. + if [ "$MASTER_HOST" = "$SELF_HOST" ] && [ "$MASTER_PORT" = "$SELF_PORT" ]; then + echo "Booting as master" + exec {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} else - {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} - SENTINEL_BASE="{{ include "redis.sentinel.name" . }}" - SENTINEL_REPLICA_COUNT={{ .Values.sentinel.replicas }} - SENTINEL_INDEX=0 - MASTER_HOST="" - MASTER_PORT="" - until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do - SENTINEL_HOST="${SENTINEL_BASE}-${SENTINEL_INDEX}.${SENTINEL_BASE}" - SENTINEL_PORT=$((26380 + SENTINEL_INDEX)) - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p $SENTINEL_PORT --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p $SENTINEL_PORT SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - fi - MASTER_HOST=$(echo "$MASTER_INFO" | head -1) - MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - if ! echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; then - echo "Waiting for sentinel at $SENTINEL_HOST:$SENTINEL_PORT to return master, trying next..." - SENTINEL_INDEX=$(( (SENTINEL_INDEX + 1) % SENTINEL_REPLICA_COUNT )) - [ $SENTINEL_INDEX -eq 0 ] && sleep 5 - fi - done - echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" + echo "Booting as replica of $MASTER_HOST:$MASTER_PORT" # Self-heal sentinel's view of the slave set. Without this, a - # scale-up race or a sentinel restart (which wipes CONFIG REWRITE - # state via the bootstrap config copy) can leave this replica - # invisible to sentinel and so orphaned at the next failover. - # SENTINEL RESET only refreshes sentinel's bookkeeping — no - # failover is triggered, no clients are disrupted. + # scale-up race or a sentinel restart (which can wipe known-replica + # entries on volume loss) can leave this replica invisible to + # sentinel and orphaned at the next failover. SENTINEL RESET only + # refreshes sentinel's bookkeeping — no failover is triggered. ( while true; do if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then @@ -115,97 +140,26 @@ spec: done sleep 3 IDX=0 - while [ $IDX -lt $SENTINEL_REPLICA_COUNT ]; do - S_HOST="${SENTINEL_BASE}-${IDX}.${SENTINEL_BASE}" - S_PORT=$((26380 + IDX)) - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - redis-cli -h $S_HOST -p $S_PORT --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true - else - redis-cli -h $S_HOST -p $S_PORT SENTINEL RESET mymaster >/dev/null 2>&1 || true - fi - IDX=$((IDX + 1)) - done - ) & - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT - {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} - SENTINEL_HOST="{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" - MASTER_HOST="" - MASTER_PORT="" - until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - fi - MASTER_HOST=$(echo "$MASTER_INFO" | head -1) - MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } - done - echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" - # Self-heal sentinel's view of the slave set (see notes in the - # publicAccess branch above for rationale). - ( - while true; do - if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then - redis-cli -p 6379 --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping >/dev/null 2>&1 && break - else - redis-cli -p 6379 ping >/dev/null 2>&1 && break - fi - sleep 2 - done - sleep 3 - IDX=0 - while [ $IDX -lt {{ .Values.sentinel.replicas }} ]; do - S_HOST="replica-${IDX}.{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - redis-cli -h $S_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true - else - redis-cli -h $S_HOST -p 26379 SENTINEL RESET mymaster >/dev/null 2>&1 || true - fi - IDX=$((IDX + 1)) - done - ) & - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT - {{- else }} - SENTINEL_HOST="{{ include "redis.sentinel.name" . }}" - MASTER_HOST="" - MASTER_PORT="" - until echo "$MASTER_PORT" | grep -qE '^[0-9]+$'; do - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - else - MASTER_INFO=$(redis-cli -h $SENTINEL_HOST -p 26379 SENTINEL get-master-addr-by-name mymaster 2>/dev/null) - fi - MASTER_HOST=$(echo "$MASTER_INFO" | head -1) - MASTER_PORT=$(echo "$MASTER_INFO" | tail -1) - echo "$MASTER_PORT" | grep -qE '^[0-9]+$' || { echo "Waiting for sentinel at $SENTINEL_HOST to return master, retrying in 5s..."; sleep 5; } - done - echo "Sentinel returned master: $MASTER_HOST:$MASTER_PORT" - # Self-heal sentinel's view of the slave set (see notes in the - # publicAccess branch above for rationale). - ( - while true; do - if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then - redis-cli -p 6379 --no-auth-warning -a "$CUSTOM_REDIS_PASSWORD" ping >/dev/null 2>&1 && break - else - redis-cli -p 6379 ping >/dev/null 2>&1 && break - fi - sleep 2 - done - sleep 3 - IDX=0 - while [ $IDX -lt {{ .Values.sentinel.replicas }} ]; do - S_HOST="{{ include "redis.sentinel.name" . }}-${IDX}.{{ include "redis.sentinel.name" . }}" + while [ $IDX -lt $SENTINEL_REPLICAS ]; do + {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + RS_HOST="{{ include "redis.sentinel.name" . }}-${IDX}.{{ include "redis.sentinel.name" . }}" + RS_PORT=$((26380 + IDX)) + {{- else if and (hasKey .Values.sentinel "replicaDirect") .Values.sentinel.replicaDirect }} + RS_HOST="replica-${IDX}.{{ include "redis.sentinel.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local" + RS_PORT=26379 + {{- else }} + RS_HOST="{{ include "redis.sentinel.name" . }}-${IDX}.{{ include "redis.sentinel.name" . }}" + RS_PORT=26379 + {{- end }} if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - redis-cli -h $S_HOST -p 26379 --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true + redis-cli -h $RS_HOST -p $RS_PORT --no-auth-warning -a "$CUSTOM_SENTINEL_PASSWORD" SENTINEL RESET mymaster >/dev/null 2>&1 || true else - redis-cli -h $S_HOST -p 26379 SENTINEL RESET mymaster >/dev/null 2>&1 || true + redis-cli -h $RS_HOST -p $RS_PORT SENTINEL RESET mymaster >/dev/null 2>&1 || true fi IDX=$((IDX + 1)) done ) & - {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof $MASTER_HOST $MASTER_PORT - {{- end }} + exec {{ .Values.redis.serverCommand }} /etc/redis/redis.conf {{ .Values.redis.extraArgs }} --dir {{ .Values.redis.dataDir }} --replicaof "$MASTER_HOST" "$MASTER_PORT" fi command: /bin/sh cpu: {{ .Values.redis.resources.cpu }} @@ -242,6 +196,31 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 4 + # Explicit liveness probe — keep it permissive (just "is the process up?") + # so the platform doesn't kill a pod mid-resync. cpln defaults the liveness + # probe to whatever the readiness probe is, and our readiness probe + # intentionally returns failure during full resync (master_link_status:up + # AND master_sync_in_progress:0). Without this override, a slave doing a + # full resync would be killed before it could finish. + livenessProbe: + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + exec: + command: + - /bin/bash + - "-c" + - |- + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((6380 + POD_ID)) + exec 3<>/dev/tcp/127.0.0.1/$PORT + {{- else }} + tcpSocket: + port: 6379 + {{- end }} + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 inheritEnv: false ports: {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled (gt (.Values.redis.replicas | int) 0) }} @@ -283,6 +262,13 @@ spec: multiZone: enabled: false {{- end }} + # Parallel pod management. Default OrderedReady serializes replica boots, + # which forces sentinels and redis pods to wait for each peer to be Ready + # before the next starts and produces extended cold-start +sdown noise. + # Parallel boots all replicas at once; publishNotReadyAddresses keeps peer + # DNS resolvable during the simultaneous cold start so the cluster can form. + rolloutOptions: + scalingPolicy: Parallel {{- if .Values.redis.requestRetryPolicy }} requestRetryPolicy: {{ toYaml .Values.redis.requestRetryPolicy | indent 4 }} diff --git a/redis/versions/3.3.0/templates/workload-sentinel.yaml b/redis/versions/3.3.0/templates/workload-sentinel.yaml index b2282b2d..82ab5787 100644 --- a/redis/versions/3.3.0/templates/workload-sentinel.yaml +++ b/redis/versions/3.3.0/templates/workload-sentinel.yaml @@ -26,43 +26,72 @@ spec: {{- else }} mkdir -p /etc/sentinel {{- end }} - cp /config/sentinel.conf /etc/sentinel/sentinel.conf - if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then - echo "\nsentinel auth-pass mymaster $CUSTOM_REDIS_PASSWORD" >> /etc/sentinel/sentinel.conf - fi + # bootstrap_sentinel_conf: write /etc/sentinel/sentinel.conf only if + # sentinel hasn't already taken ownership of the file. The marker we + # check is the `sentinel myid ` directive — sentinel writes this + # via CONFIG REWRITE within milliseconds of its first successful + # startup, and the chart's static template never contains it. So: + # - marker present → previous boot got far enough that sentinel + # owns the file. Preserve everything (current master after any + # failover, known replicas, peer sentinels). + # - marker absent → file missing, empty, or written but sentinel + # never reached its first CONFIG REWRITE. Re-run bootstrap; + # idempotent against the static config so safe to re-run. + # When sentinel.persistence is disabled this still runs every boot + # (ephemeral filesystem, no marker survives) — same behavior as the + # pre-marker chart. The check only changes behavior when /etc/sentinel + # is backed by a persistent volume. + bootstrap_sentinel_conf() { + if [ -s /etc/sentinel/sentinel.conf ] && grep -q "^sentinel myid " /etc/sentinel/sentinel.conf; then + echo "sentinel.conf already bootstrapped; preserving sentinel-managed state" + return 0 + fi + echo "Bootstrapping sentinel.conf from static config" + cp /config/sentinel.conf /etc/sentinel/sentinel.conf - if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then - echo "\nrequirepass $CUSTOM_SENTINEL_PASSWORD" >> /etc/sentinel/sentinel.conf - fi + if [ -n "$CUSTOM_REDIS_PASSWORD" ]; then + echo "\nsentinel auth-pass mymaster $CUSTOM_REDIS_PASSWORD" >> /etc/sentinel/sentinel.conf + fi - {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} - POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) - PORT=$((26380 + POD_ID)) - echo "\nport $PORT" >> /etc/sentinel/sentinel.conf - echo "\nsentinel announce-ip {{ .Values.sentinel.publicAccess.address }}" >> /etc/sentinel/sentinel.conf - echo "\nsentinel announce-port $PORT" >> /etc/sentinel/sentinel.conf - {{ else }} - echo "\nport 26379" >> /etc/sentinel/sentinel.conf - POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) - LOCATION=${CPLN_LOCATION##*/} - CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" - if [ -n "$REPLICA_DIRECT" ]; then - echo "\nsentinel announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/sentinel/sentinel.conf - else - echo "\nsentinel announce-ip ${HOSTNAME}.{{ include "redis.sentinel.name" . }}" >> /etc/sentinel/sentinel.conf - fi - echo "\nsentinel announce-port 26379" >> /etc/sentinel/sentinel.conf - {{ end }} - {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} - echo "sentinel monitor mymaster {{ .Values.redis.publicAccess.address }} 6380 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf - {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} - echo "sentinel monitor mymaster replica-0.{{ include "redis.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf - {{- else }} - echo "sentinel monitor mymaster {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf - {{- end }} + if [ -n "$CUSTOM_SENTINEL_PASSWORD" ]; then + echo "\nrequirepass $CUSTOM_SENTINEL_PASSWORD" >> /etc/sentinel/sentinel.conf + fi + + {{- if and (hasKey .Values.sentinel "publicAccess") .Values.sentinel.publicAccess.enabled }} + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + PORT=$((26380 + POD_ID)) + echo "\nport $PORT" >> /etc/sentinel/sentinel.conf + echo "\nsentinel announce-ip {{ .Values.sentinel.publicAccess.address }}" >> /etc/sentinel/sentinel.conf + echo "\nsentinel announce-port $PORT" >> /etc/sentinel/sentinel.conf + {{ else }} + echo "\nport 26379" >> /etc/sentinel/sentinel.conf + POD_ID=$(echo "$POD_NAME" | rev | cut -d'-' -f 1 | rev) + LOCATION=${CPLN_LOCATION##*/} + CPLN_WORKLOAD_NAME="${CPLN_WORKLOAD##*/}" + if [ -n "$REPLICA_DIRECT" ]; then + echo "\nsentinel announce-ip replica-${POD_ID}.${CPLN_WORKLOAD_NAME}.${LOCATION}.${CPLN_GVC}.cpln.local" >> /etc/sentinel/sentinel.conf + else + echo "\nsentinel announce-ip ${HOSTNAME}.{{ include "redis.sentinel.name" . }}" >> /etc/sentinel/sentinel.conf + fi + echo "\nsentinel announce-port 26379" >> /etc/sentinel/sentinel.conf + {{ end }} + {{- if and (hasKey .Values.redis "publicAccess") .Values.redis.publicAccess.enabled }} + echo "sentinel monitor mymaster {{ .Values.redis.publicAccess.address }} 6380 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- else if and (hasKey .Values.redis "replicaDirect") .Values.redis.replicaDirect }} + echo "sentinel monitor mymaster replica-0.{{ include "redis.name" . }}.${LOCATION}.${CPLN_GVC}.cpln.local 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- else }} + echo "sentinel monitor mymaster {{ include "redis.name" . }}-0.{{ include "redis.name" . }} 6379 ${REDIS_SENTINEL_QUORUM}" >> /etc/sentinel/sentinel.conf + {{- end }} + } + + bootstrap_sentinel_conf - redis-sentinel /etc/sentinel/sentinel.conf + # exec replaces the shell with redis-sentinel so SIGTERM from the + # platform routes directly to redis-sentinel for clean shutdown. + # Without exec, the shell swallows the signal and sentinel only dies + # when the platform sends SIGKILL after the grace period expires. + exec redis-sentinel /etc/sentinel/sentinel.conf command: /bin/sh cpu: {{ .Values.sentinel.resources.cpu }} memory: {{ .Values.sentinel.resources.memory }} @@ -158,6 +187,10 @@ spec: multiZone: enabled: false {{- end }} + # See workload-redis.yaml for rationale. Parallel boots all sentinels at + # once instead of waiting for each to be Ready in sequence. + rolloutOptions: + scalingPolicy: Parallel {{- if .Values.sentinel.requestRetryPolicy }} requestRetryPolicy: {{ toYaml .Values.sentinel.requestRetryPolicy | indent 4 }} diff --git a/redis/versions/3.3.0/values.yaml b/redis/versions/3.3.0/values.yaml index 737dda09..6dff2f11 100644 --- a/redis/versions/3.3.0/values.yaml +++ b/redis/versions/3.3.0/values.yaml @@ -120,6 +120,9 @@ sentinel: # - resource-exhausted # - retriable-status-codes requestRetryPolicy: {} + # Sentinel persistence preserves the post-failover master across restarts via + # CONFIG REWRITE. Replicas query sentinel at startup, so persisted state lets + # them rejoin the real master rather than redis-0. Recommended ON in prod. persistence: enabled: false volumes: From d4dd4feff89e980123ea3fffaa09e55fcf67f2cc Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Thu, 7 May 2026 13:45:28 -0400 Subject: [PATCH 35/58] cdc-pipeline: use kafka v4.0.0 --- cdc-pipeline/versions/1.0.0/Chart.yaml | 4 ++-- cdc-pipeline/versions/1.0.0/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdc-pipeline/versions/1.0.0/Chart.yaml b/cdc-pipeline/versions/1.0.0/Chart.yaml index 8c1625e7..f788ff3a 100644 --- a/cdc-pipeline/versions/1.0.0/Chart.yaml +++ b/cdc-pipeline/versions/1.0.0/Chart.yaml @@ -13,7 +13,7 @@ dependencies: version: 2.2.0 repository: "oci://ghcr.io/controlplane-com/templates" - name: kafka - version: 3.4.0 + version: 4.0.0 repository: "oci://ghcr.io/controlplane-com/templates" - name: debezium-server version: 1.1.0 @@ -21,6 +21,6 @@ dependencies: annotations: created: "2026-04-13" - lastModified: "2026-04-13" + lastModified: "2026-05-07" category: "event-streaming" createsGvc: false diff --git a/cdc-pipeline/versions/1.0.0/README.md b/cdc-pipeline/versions/1.0.0/README.md index 44d7108b..22262ac3 100644 --- a/cdc-pipeline/versions/1.0.0/README.md +++ b/cdc-pipeline/versions/1.0.0/README.md @@ -82,5 +82,5 @@ INSERT INTO debezium_heartbeat VALUES (1, now()); | Component | Version | |-----------|---------| | PostgreSQL HA | 2.2.0 (Patroni, PostgreSQL 17) | -| Kafka | 3.4.0 (Apache Kafka 3.9.1, KRaft) | +| Kafka | 4.0.0 (Apache Kafka 3.9.1, KRaft) | | Debezium Server | 1.1.0 (Debezium 3.0) | From 75eef36826f3d6126d176b105d5824467d109a3a Mon Sep 17 00:00:00 2001 From: Kyle Cupp Date: Thu, 7 May 2026 15:31:04 -0400 Subject: [PATCH 36/58] kafka: fix pipeline --- kafka/versions/4.0.0/.helmignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 kafka/versions/4.0.0/.helmignore diff --git a/kafka/versions/4.0.0/.helmignore b/kafka/versions/4.0.0/.helmignore deleted file mode 100644 index 7d101009..00000000 --- a/kafka/versions/4.0.0/.helmignore +++ /dev/null @@ -1 +0,0 @@ -values.yaml \ No newline at end of file From 335ea6825bf4e79098ede4ba3834a5427362c9ea Mon Sep 17 00:00:00 2001 From: Jakob Nagel Date: Mon, 11 May 2026 11:28:08 -0400 Subject: [PATCH 37/58] ESS: gcp sync all secrets --- ess/versions/1.6.0/Chart.yaml | 17 ++ ess/versions/1.6.0/README.md | 222 +++++++++++++++++++++ ess/versions/1.6.0/templates/_helpers.tpl | 39 ++++ ess/versions/1.6.0/templates/identity.yaml | 5 + ess/versions/1.6.0/templates/policy.yaml | 10 + ess/versions/1.6.0/templates/secret.yaml | 9 + ess/versions/1.6.0/templates/workload.yaml | 61 ++++++ ess/versions/1.6.0/values.yaml | 86 ++++++++ 8 files changed, 449 insertions(+) create mode 100644 ess/versions/1.6.0/Chart.yaml create mode 100644 ess/versions/1.6.0/README.md create mode 100644 ess/versions/1.6.0/templates/_helpers.tpl create mode 100644 ess/versions/1.6.0/templates/identity.yaml create mode 100644 ess/versions/1.6.0/templates/policy.yaml create mode 100644 ess/versions/1.6.0/templates/secret.yaml create mode 100644 ess/versions/1.6.0/templates/workload.yaml create mode 100644 ess/versions/1.6.0/values.yaml diff --git a/ess/versions/1.6.0/Chart.yaml b/ess/versions/1.6.0/Chart.yaml new file mode 100644 index 00000000..f01e943f --- /dev/null +++ b/ess/versions/1.6.0/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: ess +description: External Secret Syncer for Control Plane +type: application +version: 1.6.0 +appVersion: v1.4.0 + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2025-03-12" + lastModified: "2026-05-11" + category: "secrets" + createsGvc: false \ No newline at end of file diff --git a/ess/versions/1.6.0/README.md b/ess/versions/1.6.0/README.md new file mode 100644 index 00000000..6d52c20e --- /dev/null +++ b/ess/versions/1.6.0/README.md @@ -0,0 +1,222 @@ +## External Secret Syncer (ESS) + +### Overview + +Creates an application that continuously syncs secrets from external providers into Control Plane secrets on a configurable schedule. Supported providers: **HashiCorp Vault**, **AWS Secrets Manager**, **AWS Parameter Store**, **Doppler**, **GCP Secret Manager**, **1Password**, and **1Password Connect**. + +--- + +### How It Works + +ESS runs as a workload on Control Plane. Your provider configuration and secrets list are stored in a Control Plane secret and mounted into the workload as `sync.yaml`. On startup, ESS schedules a polling loop for each configured secret. At each interval, it fetches the latest value from the external provider and creates or updates the corresponding Control Plane secret via the API. + +ESS tags every secret it manages with `syncer.cpln.io/source` (set to the workload path). This prevents two ESS instances from accidentally overwriting each other's secrets. An hourly cleanup job also deletes any Control Plane secrets that ESS owns but that have been removed from your `sync.yaml` config. + +--- + +### Patch Notes + +This version of ESS fixes a bug preventing the cleanup from running + +### Configuring `values.yaml` + +#### Top-level fields + +| Field | Description | +|---|---| +| `image` | The ESS container image. Do not change unless upgrading. | +| `resources.cpu` / `resources.memory` | Resource limits for the workload container. | +| `port` | Port for the ESS HTTP admin API (default: `3004`). Used for health checks and manual sync triggers. | +| `allowedIp` | List of CIDRs allowed to reach the ESS admin API externally. Replace the placeholder with your IP, or use `0.0.0.0/0` to allow all. | +| `essConfig` | The full sync configuration — providers and secrets (see below). | + +--- + +#### `essConfig.providers` + +Each provider entry requires a unique `name` and exactly one provider block. An optional `syncInterval` sets the default interval for all secrets using that provider. + +**Vault** +```yaml +- name: my-vault + vault: + address: https://my-vault.com:8200 # required + token: # required + syncInterval: 1m # optional — overrides global default +``` + +**AWS Parameter Store** +```yaml +- name: my-aws-ssm + awsParameterStore: + region: us-east-1 + accessKeyId: # optional if using an IAM-linked identity + secretAccessKey: # optional if using an IAM-linked identity +``` + +**AWS Secrets Manager** +```yaml +- name: my-aws-secrets-manager + awsSecretsManager: + region: us-east-1 + accessKeyId: + secretAccessKey: +``` + +**Doppler** +```yaml +- name: my-doppler + doppler: + accessToken: # use a Doppler service token (dp.st....) +``` + +**GCP Secret Manager** +```yaml +- name: my-gcp + gcpSecretManager: + projectId: 123456789876 + credentials: # optional — omit to use Application Default Credentials + clientEmail: + privateKey: +``` + +**1Password** +```yaml +- name: my-1password + onePassword: + serviceAccountToken: + integrationName: my-ess # optional + integrationVersion: 1.0.0 # optional +``` + +**1Password Connect** +```yaml +- name: my-1password-connect + onePasswordConnect: + serverURL: https://my-connect-server.example.com # required + token: # required +``` + +--- + +#### `essConfig.secrets` + +Each secret entry syncs one value (or a set of values) from a provider into a Control Plane secret. + +| Field | Description | +|---|---| +| `name` | Name of the Control Plane secret to create or update. | +| `provider` | Must match a provider `name` defined above. | +| `syncInterval` | Optional. Overrides the provider-level and global default for this specific secret. | + +Each secret must use exactly one of the following sync types: + +--- + +##### `opaque` — Single value (stored as a Control Plane `opaque` secret) + +Shorthand (path only, no fallback): +```yaml +- name: my-secret + provider: my-vault + opaque: /v1/secret/data/myapp +``` + +With options: +```yaml +- name: my-secret + provider: my-vault + opaque: + path: /v1/secret/data/myapp # path to fetch + parse: data.password # optional — extract a key from a JSON/YAML response + default: fallback-value # optional — used if fetch fails + encoding: base64 # optional — base64-decode the fetched value +``` + +> **Note:** If you use the shorthand form (`opaque: /some/path`) with no `default`, a fetch failure causes the sync to fail with no fallback. + +--- + +##### `dictionary` — Multiple values (stored as a Control Plane `dictionary` secret) + +Each key in the dictionary is fetched independently: +```yaml +- name: my-secret + provider: my-vault + dictionary: + PORT: + path: /v1/secret/data/app + parse: data.port + default: 5432 + PASSWORD: + path: /v1/secret/data/app + parse: data.password + USERNAME: + path: /v1/secret/data/app + parse: data.username + default: "no username" +``` + +Each key supports `path`, `parse`, `default`, and `encoding` — the same options as `opaque`. A failure on one key does not block others. + +--- + +##### `dictionaryFromProject` — Sync an entire project (Doppler or GCP Secret Manager) + +Syncs all secrets from a provider project in one operation, stored as a Control Plane `dictionary` secret. The expected shape depends on the provider. + +**Doppler** — specify a `project/config` path: +```yaml +- name: my-doppler-config + provider: my-doppler + dictionaryFromProject: + path: my-project/dev # format: "project/config" — exactly two segments +``` + +**GCP Secret Manager** — set to `true` to pull every accessible secret from the project configured on the provider: +```yaml +- name: my-gcp-config + provider: my-gcp + dictionaryFromProject: true +``` + +Each fetched secret's latest version becomes one key in the resulting dictionary. Secrets with no accessible latest version (no versions, disabled, or destroyed) are skipped. + +> **Note:** `dictionaryFromProject` is only valid with the Doppler or GCP Secret Manager providers. Doppler requires the `{ path: ... }` object form; GCP requires the `true` form. Mixing them (or using either with another provider) causes ESS to exit at startup. + +--- + +#### Doppler Path Formats + +| Sync type | Path format | Example | +|---|---|---| +| `opaque` or `dictionary` key | `project/config/SECRET_NAME` | `my-app/production/DATABASE_URL` | +| `dictionaryFromProject` | `project/config` | `my-app/production` | + +--- + +#### Sync Interval Format + +Intervals use the format `hms`. All parts are optional but at least one is required. + +Examples: `10s`, `5m`, `1h`, `1h30m`, `1h30m10s` + +Priority (highest wins): +1. Secret-level `syncInterval` +2. Provider-level `syncInterval` +3. Global default (`300s`) + +--- + +### Important Notes + +- **Conflict protection:** If a Control Plane secret already exists and is managed by a different ESS instance, the sync for that secret will fail. Two ESS instances cannot manage the same secret. +- **Secret type changes:** Changing a secret from `opaque` to `dictionary` (or vice versa) causes ESS to delete the existing secret and recreate it. There is a brief window where the secret does not exist. +- **Cleanup:** ESS runs an hourly job that deletes Control Plane secrets it owns but that no longer appear in `sync.yaml`. Removing a secret from the config will eventually result in its deletion from Control Plane. +- **Doppler `parse`:** The `parse` field only works when the Doppler secret's value is JSON or YAML. Using `parse` on a plain string secret throws an error. +- **`sync.yaml` hot reload:** ESS watches its config file and automatically restarts when changes are detected (every ~5 seconds). No workload restart is needed after updating the config secret. + +### Resources + +- [ESS Documentation](https://docs.controlplane.com/template-catalog/templates/external-secret-syncer) +- [Image Source Code](https://github.com/controlplane-com/external-secret-syncer) \ No newline at end of file diff --git a/ess/versions/1.6.0/templates/_helpers.tpl b/ess/versions/1.6.0/templates/_helpers.tpl new file mode 100644 index 00000000..95668c35 --- /dev/null +++ b/ess/versions/1.6.0/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* Resource Naming */}} + +{{/* +ESS Workload Name +*/}} +{{- define "ess.name" -}} +{{- printf "%s-ess" .Release.Name }} +{{- end }} + +{{/* +ESS Identity Name +*/}} +{{- define "ess.identity.name" -}} +{{- printf "%s-ess-identity" .Release.Name }} +{{- end }} + +{{/* +ESS Policy Name +*/}} +{{- define "ess.policy.name" -}} +{{- printf "%s-ess-policy" .Release.Name }} +{{- end }} + +{{/* +ESS Secret Config Name +*/}} +{{- define "ess.secret.name" -}} +{{- printf "%s-ess-config" .Release.Name }} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels +*/}} +{{- define "ess.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/ess/versions/1.6.0/templates/identity.yaml b/ess/versions/1.6.0/templates/identity.yaml new file mode 100644 index 00000000..e7176ee7 --- /dev/null +++ b/ess/versions/1.6.0/templates/identity.yaml @@ -0,0 +1,5 @@ +kind: identity +gvc: {{ .Values.global.cpln.gvc }} +name: {{ include "ess.identity.name" . }} +description: ESS identity +tags: {{- include "ess.tags" . | nindent 4 }} diff --git a/ess/versions/1.6.0/templates/policy.yaml b/ess/versions/1.6.0/templates/policy.yaml new file mode 100644 index 00000000..cba2f1dd --- /dev/null +++ b/ess/versions/1.6.0/templates/policy.yaml @@ -0,0 +1,10 @@ +kind: policy +name: {{ include "ess.policy.name" . }} +description: ESS policy +bindings: + - permissions: + - manage + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "ess.identity.name" . }} +target: all +targetKind: secret diff --git a/ess/versions/1.6.0/templates/secret.yaml b/ess/versions/1.6.0/templates/secret.yaml new file mode 100644 index 00000000..764bc110 --- /dev/null +++ b/ess/versions/1.6.0/templates/secret.yaml @@ -0,0 +1,9 @@ +kind: secret +name: {{ include "ess.secret.name" . }} +description: ESS config +tags: {{- include "ess.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: | +{{- toYaml .Values.essConfig | nindent 4 }} \ No newline at end of file diff --git a/ess/versions/1.6.0/templates/workload.yaml b/ess/versions/1.6.0/templates/workload.yaml new file mode 100644 index 00000000..a4106066 --- /dev/null +++ b/ess/versions/1.6.0/templates/workload.yaml @@ -0,0 +1,61 @@ +kind: workload +name: {{ include "ess.name" . }} +description: External Secret Syncer +tags: {{- include "ess.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: ess + cpu: {{ .Values.resources.cpu | quote }} + image: {{ .Values.image }} + inheritEnv: false + memory: {{ .Values.resources.memory | quote }} + ports: + - number: {{ .Values.port }} + protocol: http + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + path: /about + port: {{ .Values.port }} + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumes: + - path: /usr/src/app/sync.yaml + recoveryPolicy: retain + uri: cpln://secret/{{ include "ess.secret.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 3 + metric: cpu + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: + {{- toYaml .Values.allowedIp | nindent 8 }} + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: none + inboundAllowWorkload: [] + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "ess.identity.name" . }} + loadBalancer: + direct: + enabled: false + ports: [] + supportDynamicTags: false diff --git a/ess/versions/1.6.0/values.yaml b/ess/versions/1.6.0/values.yaml new file mode 100644 index 00000000..a445bd60 --- /dev/null +++ b/ess/versions/1.6.0/values.yaml @@ -0,0 +1,86 @@ +image: ghcr.io/controlplane-com/cpln-build/external-secret-syncer:v1.4.0 + +resources: + cpu: 200m + memory: 256Mi + +port: 3004 + +allowedIp: + - 1.2.3.4 # Replace with your IP + +essConfig: + providers: + - name: my-vault + vault: + address: https://my-vault.com:8200 + token: + syncInterval: 1m + - name: my-aws-ssm + awsParameterStore: + region: us-east-1 + accessKeyId: # alternatively configure identity to natively use AWS permissions + secretAccessKey: # alternatively configure identity to natively use AWS permissions + # - name: my-aws-secrets-manager + # awsSecretsManager: + # region: us-east-1 + # accessKeyId: + # secretAccessKey: + # - name: my-1password + # onePassword: + # serviceAccountToken: + # integrationName: my-ess + # integrationVersion: 1.0.0 + # - name: my-1password-connect + # onePasswordConnect: + # serverURL: https://my-connect-server.example.com + # token: + # - name: my-doppler + # doppler: + # accessToken: + # - name: my-gcp + # gcpSecretManager: + # projectId: 123456789876 + # credentials: + # clientEmail: + # privateKey: + secrets: + - name: auth + provider: my-vault + syncInterval: 20s + dictionary: + PORT: + path: /v1/secret/data/app + parse: data.port + default: 5432 + PASSWORD: + path: /v1/secret/data/app + parse: data.password + USERNAME: + default: "no username" + path: /v1/secret/data/app + parse: data.username + - name: ssm + provider: my-aws + syncInterval: 20s + opaque: /example/app + # - name: secrets-manager + # provider: my-aws-secrets-manager + # dictionary: + # PASSWORD: + # path: /example/app + # parse: password + # - name: doppler-secret + # provider: my-doppler + # opaque: /project/config/SECRET_NAME + # - name: doppler-project + # provider: my-doppler + # dictionaryFromProject: + # path: project/config # syncs all secrets from a Doppler project+config + # - name: gcp + # provider: my-gcp + # opaque: database-password + # - name: gcp-project + # provider: my-gcp + # dictionaryFromProject: true # syncs all secrets from the GCP project + From 191fbefec90652df60e174edbe6c3a74a0f000ff Mon Sep 17 00:00:00 2001 From: Hakan Date: Tue, 12 May 2026 10:38:16 -0700 Subject: [PATCH 38/58] shorten cdc desc --- cdc-pipeline/versions/1.0.0/Chart.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cdc-pipeline/versions/1.0.0/Chart.yaml b/cdc-pipeline/versions/1.0.0/Chart.yaml index f788ff3a..c53dc01b 100644 --- a/cdc-pipeline/versions/1.0.0/Chart.yaml +++ b/cdc-pipeline/versions/1.0.0/Chart.yaml @@ -1,9 +1,6 @@ apiVersion: v2 name: cdc-pipeline -description: >- - Change Data Capture pipeline with PostgreSQL HA, Kafka, and Debezium Server. - Automatically coordinates database WAL settings, Kafka SASL credentials, - and cross-service DNS for a production-ready CDC streaming setup. +description: CDC pipeline with PostgreSQL HA, Kafka, and Debezium. Auto-configured WAL, SASL credentials, and cross-service DNS. type: application version: 1.0.0 appVersion: "1.0" From a5845d1112474eafffdf19154bf177a60a87c9d0 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Tue, 12 May 2026 16:20:57 -0700 Subject: [PATCH 39/58] added pgbouncer and probes, fixed init bugs, added single location support, updated readme (#255) * init 1.4.0 * init script now runs in foreground and runs exec * removed extra wait * added liveness and readiness probes * added prestop hook and removed probes temporarily * readded probes * added pgbouncer * added cpln-common, removed tag * added probes, added pgbouncer, fixed startup bugs, supports single location, added cpln-common * updated README --- cockroach/versions/1.4.0/Chart.yaml | 18 ++ cockroach/versions/1.4.0/README.md | 164 +++++++++++++++ .../versions/1.4.0/templates/_helpers.tpl | 86 ++++++++ cockroach/versions/1.4.0/templates/gvc.yaml | 11 ++ .../versions/1.4.0/templates/identity.yaml | 24 +++ .../versions/1.4.0/templates/policy.yaml | 17 ++ .../templates/secret-pgbouncer-startup.yaml | 55 ++++++ .../versions/1.4.0/templates/secret.yaml | 186 ++++++++++++++++++ .../versions/1.4.0/templates/volumeset.yaml | 18 ++ .../1.4.0/templates/workload-backup.yaml | 64 ++++++ .../1.4.0/templates/workload-cockroach.yaml | 94 +++++++++ .../1.4.0/templates/workload-pgbouncer.yaml | 62 ++++++ cockroach/versions/1.4.0/values.yaml | 74 +++++++ 13 files changed, 873 insertions(+) create mode 100644 cockroach/versions/1.4.0/Chart.yaml create mode 100644 cockroach/versions/1.4.0/README.md create mode 100644 cockroach/versions/1.4.0/templates/_helpers.tpl create mode 100644 cockroach/versions/1.4.0/templates/gvc.yaml create mode 100644 cockroach/versions/1.4.0/templates/identity.yaml create mode 100644 cockroach/versions/1.4.0/templates/policy.yaml create mode 100644 cockroach/versions/1.4.0/templates/secret-pgbouncer-startup.yaml create mode 100644 cockroach/versions/1.4.0/templates/secret.yaml create mode 100644 cockroach/versions/1.4.0/templates/volumeset.yaml create mode 100644 cockroach/versions/1.4.0/templates/workload-backup.yaml create mode 100644 cockroach/versions/1.4.0/templates/workload-cockroach.yaml create mode 100644 cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml create mode 100644 cockroach/versions/1.4.0/values.yaml diff --git a/cockroach/versions/1.4.0/Chart.yaml b/cockroach/versions/1.4.0/Chart.yaml new file mode 100644 index 00000000..b802b8be --- /dev/null +++ b/cockroach/versions/1.4.0/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: cockroach +description: Distributed PostgreSQL-compatible database for Control Plane + +type: application +version: 1.4.0 +appVersion: "25.4.0" + +annotations: + created: "2025-08-25" + lastModified: "2026-05-06" + category: "database" + createsGvc: true + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" \ No newline at end of file diff --git a/cockroach/versions/1.4.0/README.md b/cockroach/versions/1.4.0/README.md new file mode 100644 index 00000000..3b12d1ec --- /dev/null +++ b/cockroach/versions/1.4.0/README.md @@ -0,0 +1,164 @@ +# CockroachDB + +CockroachDB is a distributed SQL database built on a transactional and strongly-consistent key-value store. It provides automatic replication, distribution, and survivability across multiple locations with minimal latency and maximum throughput. CockroachDB offers ACID transactions, horizontal scalability, and built-in fault tolerance, making it ideal for applications requiring global data distribution and high availability. + +## Configuration + +To configure your CockroachDB cluster across multiple locations, update the `gvc.locations` section in the `values.yaml` file. + +**Note**: While CockroachDB can run on 1 location, a minimum of 3 locations and 3 replicas per location is recommended for high resilience. + +### Volume Storage + +Configure initial storage capacity and optional autoscaling for the CockroachDB data volume in `values.yaml`: + +```yaml +volumeset: + capacity: 10 # initial capacity in GiB (minimum is 10) + autoscaling: + enabled: false + maxCapacity: 100 # maximum capacity in GiB + minFreePercentage: 10 # scale when free space drops below this percentage + scalingFactor: 1.2 # multiply current capacity by this factor when scaling +``` + +### Database Initialization + +To create a database with a user on initialization, configure the `database` section in your `values.yaml` file. The database and user are created automatically on first deploy only — they are not re-created on restarts. + +### Internal Access Configuration + +To specify which workloads can access this CockroachDB cluster internally, configure the `internal_access` section in your `values.yaml` file: + +**Access Types:** +- `same-gvc`: Allow access from all workloads in the same GVC +- `same-org`: Allow access from all workloads in the same organization +- `workload-list`: Allow access only from specific workloads listed in `outside_workloads` and can be used in conjunction with `same-gvc` + +Once deployed, CockroachDB will be available on port 26257. CockroachDB is configured in `--insecure` mode because Control Plane handles mTLS for all inter-workload communication. Connect using the internal hostname: + +```bash +cockroach sql --insecure --host=-cockroach..cpln.local:26257 +``` + +### Admin Dashboard + +The CockroachDB admin UI runs on port 8080 but is not exposed externally. Access it via port forward and open `http://localhost:8080` in your browser. + +The cluster automatically handles data distribution and replication across your configured locations. + +**Note on GVC Naming** + +- This template creates a GVC with a default name defined in the `values.yaml`. If you plan to deploy multiple instances of this template, you **must assign a unique GVC name** for each deployment. + +### Multi-Region Survivability + +On first deploy, the cluster automatically configures the database with all configured locations as regions and sets the survival goal to `REGION`, meaning the cluster can tolerate the loss of an entire location without downtime. To verify: + +```sql +SHOW SURVIVAL GOAL FROM DATABASE mydb; +``` + +**Note**: A production CockroachDB setup can survive a location outage cleanly, but rolling out or restarting replicas in the remaining locations during that outage exceeds the cluster's fault tolerance and will cause a brief period of downtime for ranges on those restarting nodes. + +## PgBouncer Connection Pooling (Optional) + +PgBouncer multiplexes application connections into a smaller pool of real database connections, reducing overhead and protecting CockroachDB from connection exhaustion under high concurrency. It connects to all CockroachDB nodes across all locations, so failover and load distribution are handled transparently. + +When enabled, PgBouncer becomes the primary connection endpoint. Connect to `{release-name}-pgbouncer.{gvc}.cpln.local:5432` instead of the CockroachDB workload directly. + +```yaml +pgbouncer: + enabled: true + poolMode: transaction # options: session, transaction, statement + defaultPoolSize: 25 # real CockroachDB connections per PgBouncer pod + maxClientConn: 250 # max app connections per PgBouncer pod + maxDbConnections: 100 # hard cap on total CockroachDB connections regardless of how many PgBouncer pods are running + minReplicas: 2 + maxReplicas: 4 +``` + +**Pool modes:** +- `transaction` — connection held only for the duration of a transaction. Best for most web and API workloads. Not compatible with session-level features like `SET` variables, temporary tables, or advisory locks. +- `session` — connection held for the entire client session. Compatible with all features but provides less connection reuse. +- `statement` — connection returned after every statement. Transactions are not supported. Rarely used. + +**`maxDbConnections`** is a hard cap on the total number of real CockroachDB connections PgBouncer will open, shared across all PgBouncer pods. Set it to a value your cluster can safely handle regardless of how many PgBouncer pods are running. + +**Scaling:** PgBouncer autoscales on RPS between `minReplicas` and `maxReplicas`. Increase `maxReplicas` for high-throughput workloads where PgBouncer becomes the bottleneck before CockroachDB does. + +## Application Retry Logic + +**Your application must implement retry logic on database connections.** PgBouncer routes around failed CockroachDB nodes, but transient errors are still surfaced to the application during failover events such as a location outage or rolling restarts — while PgBouncer cycles through backends and Raft leader elections complete. Without retries, these transient errors will propagate directly to the client. + +## Backing Up + +Set your desired backup schedule in the values file and configure your AWS S3 or GCS bucket. You can also set a prefix where your backups will be stored in the bucket. + +Set `backup.location` to the region closest to your storage bucket to minimize cross-region transfer latency. CockroachDB nodes upload backup data directly to cloud storage using their own workload identity — the backup job only triggers the SQL command. + +### AWS S3 + +For the backup job to have access to an S3 bucket, ensure the following prerequisites are completed in your AWS account before installing: + +1. Create your bucket. Update `aws.bucket` to include its name and `aws.region` to include its region. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Update `aws.cloudAccountName`. + +3. Create a new AWS IAM policy with the following JSON (replace `YOUR_BUCKET_NAME`): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetObjectVersion", + "s3:DeleteObjectVersion" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + } + ] +} +``` + +4. Set `aws.policyName` to match the policy created in step 3. + +### GCS + +For the backup job to have access to a GCS bucket, ensure the following prerequisites are completed in your GCP account before installing: + +1. Create your bucket. Update `gcp.bucket` to include its name. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Update `gcp.cloudAccountName`. + +**Important**: You must add the `Storage Admin` role to the created GCP service account. + +### Restoring a Backup + +Backups are stored at `BUCKET/PREFIX/`. To restore, run `cockroach sql` from a machine with access to the bucket and network access to the cluster. + +**AWS S3** +```sh +cockroach sql --insecure \ + --host="WORKLOAD_INTERNAL_HOSTNAME:26257" \ + --execute="RESTORE FROM LATEST IN 's3://BUCKET_NAME/PREFIX?AUTH=implicit&AWS_REGION=BUCKET_REGION';" +``` + +**GCS** +```sh +cockroach sql --insecure \ + --host="WORKLOAD_INTERNAL_HOSTNAME:26257" \ + --execute="RESTORE FROM LATEST IN 'gs://BUCKET_NAME/PREFIX?AUTH=implicit';" +``` + +### Supported External Services +- [CockroachDB Documentation](https://www.cockroachlabs.com/docs/stable/) diff --git a/cockroach/versions/1.4.0/templates/_helpers.tpl b/cockroach/versions/1.4.0/templates/_helpers.tpl new file mode 100644 index 00000000..1d68cd9f --- /dev/null +++ b/cockroach/versions/1.4.0/templates/_helpers.tpl @@ -0,0 +1,86 @@ +{{/* Resource Naming */}} + +{{/* +Cockroach Workload Name +*/}} +{{- define "cockroach.name" -}} +{{- printf "%s-cockroach" .Release.Name }} +{{- end }} + +{{/* +Cockroach Secret Database Config Name +*/}} +{{- define "cockroach.secretDatabase.name" -}} +{{- printf "%s-cockroach-config" .Release.Name }} +{{- end }} + +{{/* +Cockroach Secret Startup Name +*/}} +{{- define "cockroach.secretStartup.name" -}} +{{- printf "%s-cockroach-startup" .Release.Name }} +{{- end }} + +{{/* +Cockroach Identity Name +*/}} +{{- define "cockroach.identity.name" -}} +{{- printf "%s-cockroach-identity" .Release.Name }} +{{- end }} + +{{/* +Cockroach Policy Name +*/}} +{{- define "cockroach.policy.name" -}} +{{- printf "%s-cockroach-policy" .Release.Name }} +{{- end }} + +{{/* +Cockroach Volume Set Name +*/}} +{{- define "cockroach.volume.name" -}} +{{- printf "%s-cockroach-vs" .Release.Name }} +{{- end }} + +{{/* +Cockroach Backup Workload Name +*/}} +{{- define "cockroach.backup.name" -}} +{{- printf "%s-cockroach-backup" .Release.Name }} +{{- end }} + +{{/* +Cockroach PgBouncer Workload Name +*/}} +{{- define "cockroach.pgbouncer.name" -}} +{{- printf "%s-cockroach-pgbouncer" .Release.Name }} +{{- end }} + +{{/* +Cockroach PgBouncer Startup Secret Name +*/}} +{{- define "cockroach.pgbouncer.secretStartup.name" -}} +{{- printf "%s-cockroach-pgbouncer-startup" .Release.Name }} +{{- end }} + + +{{/* Validation */}} + +{{/* +Validate that gvc.locations has at least 1 entry +*/}} +{{- define "cockroach.validateLocations" -}} +{{- if lt (len .Values.gvc.locations) 1 -}} +{{- fail "gvc.locations must contain at least 1 location" -}} +{{- end -}} +{{- end -}} + + +{{/* Labeling */}} + +{{/* +Common labels - delegated to cpln-common +*/}} +{{- define "cockroach.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} \ No newline at end of file diff --git a/cockroach/versions/1.4.0/templates/gvc.yaml b/cockroach/versions/1.4.0/templates/gvc.yaml new file mode 100644 index 00000000..f86826ec --- /dev/null +++ b/cockroach/versions/1.4.0/templates/gvc.yaml @@ -0,0 +1,11 @@ +kind: gvc +name: {{ .Values.gvc.name }} +description: {{ .Values.gvc.name }} +tags: {{- include "cockroach.tags" . | nindent 4 }} +spec: + endpointNamingFormat: org + staticPlacement: + locationLinks: + {{- range .Values.gvc.locations }} + - //location/{{ .name }} + {{- end }} diff --git a/cockroach/versions/1.4.0/templates/identity.yaml b/cockroach/versions/1.4.0/templates/identity.yaml new file mode 100644 index 00000000..10dbc1a6 --- /dev/null +++ b/cockroach/versions/1.4.0/templates/identity.yaml @@ -0,0 +1,24 @@ +--- +kind: identity +gvc: {{ .Values.gvc.name }} +name: {{ include "cockroach.identity.name" . }} +description: CockroachDB identity +tags: {{- include "cockroach.tags" . | nindent 4 }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.backup.aws.cloudAccountName }} + policyRefs: + - cpln-connector + - aws::ReadOnlyAccess + - {{ .Values.backup.aws.policyName | quote }} +{{- end }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "gcp") }} +gcp: + bindings: + - resource: //storage.googleapis.com/projects/_/buckets/{{ .Values.backup.gcp.bucket }} + roles: + - roles/storage.objectAdmin + cloudAccountLink: //cloudaccount/{{ .Values.backup.gcp.cloudAccountName }} + scopes: + - https://www.googleapis.com/auth/cloud-platform +{{- end }} \ No newline at end of file diff --git a/cockroach/versions/1.4.0/templates/policy.yaml b/cockroach/versions/1.4.0/templates/policy.yaml new file mode 100644 index 00000000..811adaff --- /dev/null +++ b/cockroach/versions/1.4.0/templates/policy.yaml @@ -0,0 +1,17 @@ +--- +kind: policy +name: {{ include "cockroach.policy.name" . }} +description: CockroachDB policy +tags: {{- include "cockroach.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.gvc.name }}/identity/{{ include "cockroach.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "cockroach.secretStartup.name" . }} + - //secret/{{ include "cockroach.secretDatabase.name" . }} + {{- if .Values.pgbouncer.enabled }} + - //secret/{{ include "cockroach.pgbouncer.secretStartup.name" . }} + {{- end }} diff --git a/cockroach/versions/1.4.0/templates/secret-pgbouncer-startup.yaml b/cockroach/versions/1.4.0/templates/secret-pgbouncer-startup.yaml new file mode 100644 index 00000000..e040ffd6 --- /dev/null +++ b/cockroach/versions/1.4.0/templates/secret-pgbouncer-startup.yaml @@ -0,0 +1,55 @@ +{{- if .Values.pgbouncer.enabled }} +{{- $hosts := list }} +{{- $workload := include "cockroach.name" . }} +{{- $gvc := .Values.gvc.name }} +{{- range .Values.gvc.locations }} +{{- $loc := .name }} +{{- $reps := int .replicas }} +{{- range until $reps }} +{{- $hosts = append $hosts (printf "replica-%d.%s.%s.%s.cpln.local" . $workload $loc $gvc) }} +{{- end }} +{{- end }} +--- +kind: secret +name: {{ include "cockroach.pgbouncer.secretStartup.name" . }} +description: PgBouncer startup script +tags: {{- include "cockroach.tags" . | nindent 2 }} +type: opaque +data: + encoding: plain + payload: |- + #!/bin/sh + set -eu + + DB_NAME="{{ .Values.database.name }}" + DB_HOST="{{ join "," $hosts }}" + + echo "Starting PgBouncer, connecting to ${DB_HOST}" + + printf '"root" ""\n"{{ .Values.database.user }}" ""\n' > /tmp/userlist.txt + + cat > /tmp/pgbouncer.ini <&1) + EXIT_CODE=$? + set -e + + echo "Init attempt $ATTEMPT: $OUTPUT" + + ALREADY_INITIALIZED=$(echo "$OUTPUT" | tr -d '\r' | grep -qiE "already.*initialized" && echo "true" || echo "false") + + if [[ $EXIT_CODE -eq 0 ]]; then + echo "Cluster initialization succeeded." + SUCCESS=true + FRESH_INIT=true + break + elif [[ "$ALREADY_INITIALIZED" == "true" ]]; then + echo "Cluster already initialized — skipping init." + SUCCESS=true + FRESH_INIT=false + break + fi + + echo "Init not ready yet — retrying in 3s..." + sleep 3 + ((ATTEMPT++)) + done + + if [[ "$SUCCESS" == true && "$FRESH_INIT" == true ]]; then + echo "Proceeding to database/user creation..." + cockroach sql --insecure --host="$SELF_FQDN:26257" <&1) + ADD_EXIT_CODE=$? + set -e + + if [[ $ADD_EXIT_CODE -eq 0 ]]; then + echo "Region $loc added successfully." + ADD_REGION_SUCCESS=true + break + fi + + echo "Attempt $attempt failed to add region $loc: $ADD_OUTPUT — retrying in 5s..." + sleep 5 + done + + if [[ "$ADD_REGION_SUCCESS" == false ]]; then + echo "ERROR: Failed to add region $loc after 10 attempts." + fi + fi + done + + echo "Setting survival goal..." + cockroach sql --insecure --host="$SELF_FQDN:26257" \ + --execute="ALTER DATABASE {{ .Values.database.name | quote }} SURVIVE REGION FAILURE;" + + echo "Multi-region configuration complete." + else + echo "Single/dual location deployment — skipping multi-region configuration." + fi + + elif [[ "$SUCCESS" == true && "$FRESH_INIT" == false ]]; then + echo "Existing cluster detected — skipping region & DB configuration." + else + echo "ERROR: Failed to initialize cluster after $MAX_ATTEMPTS attempts." + fi + ) & + fi + + # Replace shell with CockroachDB — receives SIGTERM, container exits if CockroachDB crashes + exec cockroach start \ + --insecure \ + --listen-addr=0.0.0.0:26257 \ + --http-addr=0.0.0.0:8080 \ + --advertise-addr="$SELF_FQDN" \ + --join="$JOIN_HOSTS" \ + --cluster-name={{ include "cockroach.name" . }} \ + --locality=region=${LOCATION} +--- +kind: secret +name: {{ include "cockroach.secretDatabase.name" . }} +description: CockroachDB config +tags: {{- include "cockroach.tags" . | nindent 4 }} +type: dictionary +data: + db: {{ .Values.database.name | quote }} + user: {{ .Values.database.user | quote }} diff --git a/cockroach/versions/1.4.0/templates/volumeset.yaml b/cockroach/versions/1.4.0/templates/volumeset.yaml new file mode 100644 index 00000000..ee6c6d31 --- /dev/null +++ b/cockroach/versions/1.4.0/templates/volumeset.yaml @@ -0,0 +1,18 @@ +kind: volumeset +name: {{ include "cockroach.volume.name" . }} +description: CockroachDB volumeset +gvc: {{ .Values.gvc.name }} +tags: {{- include "cockroach.tags" . | nindent 4 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.volumeset.capacity }} + {{- if .Values.volumeset.autoscaling.enabled }} + autoscaling: + maxCapacity: {{ .Values.volumeset.autoscaling.maxCapacity }} + minFreePercentage: {{ .Values.volumeset.autoscaling.minFreePercentage }} + scalingFactor: {{ .Values.volumeset.autoscaling.scalingFactor }} + {{- end }} + performanceClass: general-purpose-ssd + snapshots: + createFinalSnapshot: true + retentionDuration: 7d diff --git a/cockroach/versions/1.4.0/templates/workload-backup.yaml b/cockroach/versions/1.4.0/templates/workload-backup.yaml new file mode 100644 index 00000000..ed18e4f6 --- /dev/null +++ b/cockroach/versions/1.4.0/templates/workload-backup.yaml @@ -0,0 +1,64 @@ +{{- if .Values.backup.enabled }} +--- +kind: workload +name: {{ include "cockroach.backup.name" . }} +description: CockroachDB Backup +tags: {{- include "cockroach.tags" . | nindent 4 }} +gvc: {{ .Values.gvc.name }} +spec: + type: cron + containers: + - name: backup-cockroach + cpu: {{ .Values.backup.resources.cpu | quote }} + memory: {{ .Values.backup.resources.memory | quote }} + image: {{ .Values.backup.image }} + inheritEnv: false + env: + - name: BACKUP_PROVIDER + value: {{ .Values.backup.provider }} + - name: COCKROACH_HOST + value: {{ include "cockroach.name" . }}.{{ .Values.gvc.name }}.cpln.local + {{- if eq .Values.backup.provider "aws" }} + - name: AWS_BUCKET + value: {{ .Values.backup.aws.bucket }} + - name: AWS_REGION + value: {{ .Values.backup.aws.region }} + - name: AWS_PREFIX + value: {{ .Values.backup.aws.prefix | quote }} + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + - name: GCP_BUCKET + value: {{ .Values.backup.gcp.bucket }} + - name: GCP_PREFIX + value: {{ .Values.backup.gcp.prefix | quote }} + {{- end }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 1 + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + suspend: true + timeoutSeconds: 3600 + localOptions: + - location: //location/{{ .Values.backup.location }} + suspend: false + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + job: + activeDeadlineSeconds: {{ .Values.backup.activeDeadlineSeconds }} + concurrencyPolicy: Forbid + historyLimit: 5 + restartPolicy: Never + schedule: {{ .Values.backup.schedule }} + supportDynamicTags: false +{{- end }} diff --git a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml new file mode 100644 index 00000000..10f076b3 --- /dev/null +++ b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml @@ -0,0 +1,94 @@ +{{- include "cockroach.validateLocations" . -}} +--- +kind: workload +name: {{ include "cockroach.name" . }} +description: CockroachDB cluster +tags: {{- include "cockroach.tags" . | nindent 2 }} +spec: + type: stateful + containers: + - name: cockroach + cpu: {{ .Values.resources.cpu | quote }} + memory: {{ .Values.resources.memory | quote }} + image: {{ .Values.image }} + command: "/bin/bash" + args: + - "/cockroach/start.sh" + inheritEnv: false + env: + - name: DB_NAME + value: cpln://secret/{{ include "cockroach.secretDatabase.name" . }}.db + - name: DB_USER + value: cpln://secret/{{ include "cockroach.secretDatabase.name" . }}.user + ports: + - number: 8080 + protocol: tcp + - number: 26257 + protocol: tcp + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - "cockroach node drain --insecure --host=localhost:26257 --self || true" + livenessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - path: /cockroach/cockroach-data + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "cockroach.volume.name" . }} + - path: /cockroach/start.sh + uri: cpln://secret/{{ include "cockroach.secretStartup.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: {{ (index .Values.gvc.locations 0).replicas | int }} + metric: disabled + minScale: {{ (index .Values.gvc.locations 0).replicas | int }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 10 + firewallConfig: + external: + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: {{ .Values.internal_access.type }} + {{- if .Values.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + {{- end }} + identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "cockroach.identity.name" . }} + loadBalancer: + replicaDirect: true + localOptions: + {{- range $location := .Values.gvc.locations }} + - autoscaling: + maxConcurrency: 0 + maxScale: {{ if eq ($location.replicas | int) 0 }}1{{ else }}{{ $location.replicas | int }}{{ end }} + metric: disabled + minScale: {{ if eq ($location.replicas | int) 0 }}0{{ else }}{{ $location.replicas | int }}{{ end }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + location: //location/{{ $location.name }} + suspend: {{ if eq ($location.replicas | int) 0 }}true{{ else }}false{{ end }} + timeoutSeconds: 10 + {{- end }} + supportDynamicTags: false diff --git a/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml b/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml new file mode 100644 index 00000000..87dc562d --- /dev/null +++ b/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml @@ -0,0 +1,62 @@ +{{- if .Values.pgbouncer.enabled }} +--- +kind: workload +name: {{ include "cockroach.pgbouncer.name" . }} +description: PgBouncer Connection Pooler for CockroachDB +tags: + {{- include "cockroach.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: pgbouncer + image: {{ .Values.pgbouncer.image }} + cpu: {{ .Values.pgbouncer.resources.cpu | quote }} + minCpu: {{ .Values.pgbouncer.resources.minCpu | quote }} + memory: {{ .Values.pgbouncer.resources.memory | quote }} + minMemory: {{ .Values.pgbouncer.resources.minMemory | quote }} + inheritEnv: false + command: "/bin/sh" + args: + - "/pgbouncer/start.sh" + volumes: + - path: /pgbouncer/start.sh + uri: cpln://secret/{{ include "cockroach.pgbouncer.secretStartup.name" . }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - "psql -h 127.0.0.1 -p 5432 -U root -d {{ .Values.database.name }} -c 'SELECT 1' -q 2>/dev/null" + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 4 + ports: + - number: 5432 + protocol: tcp + identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "cockroach.identity.name" . }} + defaultOptions: + autoscaling: + metric: rps + minScale: {{ .Values.pgbouncer.minReplicas }} + maxScale: {{ .Values.pgbouncer.maxReplicas }} + maxConcurrency: 0 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + timeoutSeconds: 5 + firewallConfig: + internal: + inboundAllowType: {{ .Values.internal_access.type }} + {{- if .Values.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + {{- end }} + external: + inboundAllowCIDR: [] + outboundAllowCIDR: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false +{{- end }} diff --git a/cockroach/versions/1.4.0/values.yaml b/cockroach/versions/1.4.0/values.yaml new file mode 100644 index 00000000..7166ff47 --- /dev/null +++ b/cockroach/versions/1.4.0/values.yaml @@ -0,0 +1,74 @@ +gvc: + name: cockroach-gvc + locations: + - name: aws-us-west-2 + replicas: 3 + - name: aws-us-east-2 + replicas: 3 + - name: aws-eu-central-1 + replicas: 3 + +image: cockroachdb/cockroach:v25.4.0 + +resources: + cpu: 2 + memory: 4Gi + +database: + name: mydb + user: myuser + +volumeset: + capacity: 10 # Initial capacity in GiB (minimum is 10) + autoscaling: + enabled: false # Set to true to enable autoscaling + maxCapacity: 100 # Maximum capacity in GiB when autoscaling is enabled + minFreePercentage: 10 # Minimum free percentage to trigger scaling when autoscaling is enabled + scalingFactor: 1.2 # Scaling factor to determine how much to scale up when autoscaling is triggered + +internal_access: + type: same-gvc # options: same-gvc, same-org, workload-list + workloads: # Note: can only be used if type is same-gvc or workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + +pgbouncer: + enabled: false + image: edoburu/pgbouncer:v1.25.1-p0 + poolMode: transaction # options: session, transaction, statement + defaultPoolSize: 25 # number of real CockroachDB connections PgBouncer maintains per pod + maxClientConn: 250 # maximum number of client connections PgBouncer accepts per pod + maxDbConnections: 100 # hard cap on total CockroachDB connections regardless of how many PgBouncer pods are running + minReplicas: 2 + maxReplicas: 4 + serverCheckDelay: 30 # seconds between idle server connection health checks (default 30) + serverConnectTimeout: 2 # seconds before giving up on a new server connection (default 15) + serverLoginRetry: 0 # seconds before retrying a failed server login; 0 = no caching of failures (default 15) + clientLoginTimeout: 10 # seconds before rejecting a client waiting for login (default 60) + queryWaitTimeout: 10 # seconds before rejecting a logged-in client waiting for a server connection (default 120) + resources: + cpu: 200m + minCpu: 100m + memory: 1Gi + minMemory: 128Mi + +backup: + enabled: false + image: controlplanecorporation/cockroach-backup:1.0 + schedule: "0 2 * * *" + activeDeadlineSeconds: 14400 # hard kill after 4 hours if backup hangs + location: aws-us-east-2 # run the backup job in the same region as your storage bucket + resources: + cpu: 500m + memory: 512Mi + provider: aws # options: aws, gcp + aws: + bucket: my-backup-bucket + region: us-east-1 + cloudAccountName: my-backup-cloudaccount + policyName: my-backup-policy + prefix: cockroach/backups + gcp: + bucket: my-backup-bucket + cloudAccountName: my-backup-cloudaccount + prefix: cockroach/backups From bbbceb21cee24f125feae7b0af4ce3cef4dd369f Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Wed, 13 May 2026 13:40:28 -0700 Subject: [PATCH 40/58] added firewall for pgbouncer settings --- .../versions/1.4.0/templates/workload-cockroach.yaml | 6 ++++++ .../versions/1.4.0/templates/workload-pgbouncer.yaml | 6 +++--- cockroach/versions/1.4.0/values.yaml | 11 +++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml index 10f076b3..5d2834ca 100644 --- a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml +++ b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml @@ -69,10 +69,16 @@ spec: outboundAllowCIDR: - 0.0.0.0/0 internal: + {{- if .Values.pgbouncer.enabled }} + inboundAllowType: workload-list + inboundAllowWorkload: + - //gvc/{{ .Values.gvc.name }}/workload/{{ include "cockroach.pgbouncer.name" . }} + {{- else }} inboundAllowType: {{ .Values.internal_access.type }} {{- if .Values.internal_access.workloads }} inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} {{- end }} + {{- end }} identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "cockroach.identity.name" . }} loadBalancer: replicaDirect: true diff --git a/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml b/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml index 87dc562d..d03b82e3 100644 --- a/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml +++ b/cockroach/versions/1.4.0/templates/workload-pgbouncer.yaml @@ -47,9 +47,9 @@ spec: timeoutSeconds: 5 firewallConfig: internal: - inboundAllowType: {{ .Values.internal_access.type }} - {{- if .Values.internal_access.workloads }} - inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} + inboundAllowType: {{ .Values.pgbouncer.internal_access.type }} + {{- if .Values.pgbouncer.internal_access.workloads }} + inboundAllowWorkload: {{ .Values.pgbouncer.internal_access.workloads | toYaml | nindent 8 }} {{- end }} external: inboundAllowCIDR: [] diff --git a/cockroach/versions/1.4.0/values.yaml b/cockroach/versions/1.4.0/values.yaml index 7166ff47..490cbc25 100644 --- a/cockroach/versions/1.4.0/values.yaml +++ b/cockroach/versions/1.4.0/values.yaml @@ -27,13 +27,12 @@ volumeset: scalingFactor: 1.2 # Scaling factor to determine how much to scale up when autoscaling is triggered internal_access: - type: same-gvc # options: same-gvc, same-org, workload-list - workloads: # Note: can only be used if type is same-gvc or workload-list - #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + type: same-gvc # options: same-gvc, same-org, workload-list; used for CockroachDB when pgbouncer is disabled + workloads: # Note: can only be used if type is workload-list #- //gvc/GVC_NAME/workload/WORKLOAD_NAME pgbouncer: - enabled: false + enabled: true image: edoburu/pgbouncer:v1.25.1-p0 poolMode: transaction # options: session, transaction, statement defaultPoolSize: 25 # number of real CockroachDB connections PgBouncer maintains per pod @@ -46,6 +45,10 @@ pgbouncer: serverLoginRetry: 0 # seconds before retrying a failed server login; 0 = no caching of failures (default 15) clientLoginTimeout: 10 # seconds before rejecting a client waiting for login (default 60) queryWaitTimeout: 10 # seconds before rejecting a logged-in client waiting for a server connection (default 120) + internal_access: + type: same-gvc # options: same-gvc, same-org, workload-list; controls who can connect to PgBouncer + workloads: # Note: can only be used if type is workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME resources: cpu: 200m minCpu: 100m From 9e7c56aa0887d8537170917e4df01ecd5047e863 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Wed, 13 May 2026 14:39:58 -0700 Subject: [PATCH 41/58] added cockroach to its firewall list when pgbouncer is enabled (#256) --- cockroach/versions/1.4.0/templates/workload-cockroach.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml index 5d2834ca..c26355a7 100644 --- a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml +++ b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml @@ -72,6 +72,7 @@ spec: {{- if .Values.pgbouncer.enabled }} inboundAllowType: workload-list inboundAllowWorkload: + - //gvc/{{ .Values.gvc.name }}/workload/{{ include "cockroach.name" . }} - //gvc/{{ .Values.gvc.name }}/workload/{{ include "cockroach.pgbouncer.name" . }} {{- else }} inboundAllowType: {{ .Values.internal_access.type }} From 2783615c46566918932c9893a5b963a751e6c213 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Wed, 13 May 2026 16:10:15 -0700 Subject: [PATCH 42/58] updated headers patch to not touch other settings (#257) * init 1.1.1 * added cockroach to its firewall list when pgbouncer is enabled (#256) * updated patch script * updated patch to be surgical on headers setting --- coraza/versions/1.1.1/Chart.yaml | 18 ++++ coraza/versions/1.1.1/README.md | 53 +++++++++++ coraza/versions/1.1.1/templates/_helpers.tpl | 46 ++++++++++ coraza/versions/1.1.1/templates/identity.yaml | 6 ++ coraza/versions/1.1.1/templates/policy.yaml | 14 +++ .../1.1.1/templates/secret-custom-rules.yaml | 15 ++++ .../1.1.1/templates/secret-startup.yaml | 88 +++++++++++++++++++ coraza/versions/1.1.1/templates/workload.yaml | 82 +++++++++++++++++ coraza/versions/1.1.1/values.yaml | 16 ++++ 9 files changed, 338 insertions(+) create mode 100644 coraza/versions/1.1.1/Chart.yaml create mode 100644 coraza/versions/1.1.1/README.md create mode 100644 coraza/versions/1.1.1/templates/_helpers.tpl create mode 100644 coraza/versions/1.1.1/templates/identity.yaml create mode 100644 coraza/versions/1.1.1/templates/policy.yaml create mode 100644 coraza/versions/1.1.1/templates/secret-custom-rules.yaml create mode 100644 coraza/versions/1.1.1/templates/secret-startup.yaml create mode 100644 coraza/versions/1.1.1/templates/workload.yaml create mode 100644 coraza/versions/1.1.1/values.yaml diff --git a/coraza/versions/1.1.1/Chart.yaml b/coraza/versions/1.1.1/Chart.yaml new file mode 100644 index 00000000..e3bb23ea --- /dev/null +++ b/coraza/versions/1.1.1/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: coraza +description: Coraza web application firewall (WAF) for Control Plane + +type: application +version: 1.1.1 +appVersion: "20241018" + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" + +annotations: + created: "2025-10-23" + lastModified: "2026-05-13" + category: "security" + createsGvc: false \ No newline at end of file diff --git a/coraza/versions/1.1.1/README.md b/coraza/versions/1.1.1/README.md new file mode 100644 index 00000000..6789b964 --- /dev/null +++ b/coraza/versions/1.1.1/README.md @@ -0,0 +1,53 @@ +## Coraza WAF App + +Creates a Coraza Web Application Firewall (WAF) with OWASP Core Rule Set (CRS) integration that proxies traffic to a target workload, providing comprehensive security filtering and protection. + +### Configuration + +The following values can be configured in your values file: + +- `targetWorkload`: The internal name of the workload to proxy traffic to (`WORKLOAD_NAME.GVC_NAME.cpln.local`) +- `targetPort`: The port of the target workload to proxy traffic to +- `WAFPort`: The port on the WAF workload to expose to the internet +- `resources`: Reserved resources for the workload +- `multiZone`: Deploys replicas across multiple zones +- `diskBodyInspection`: When `true` (default), request bodies exceeding the 512KB in-memory limit are buffered to disk at `/tmp/coraza` for full inspection up to 12.5MB. When `false`, all body inspection is kept in memory — bodies up to 12.5MB are held in memory rather than spilling to disk, which avoids disk I/O but increases memory pressure on large requests. + +### Logging + +All Coraza logging is currently sent to `/dev/stdout` to be readable in the Control Plane built-in logging interface. Logging can be redirected by using the existing environment variables in the workload configuration. + +### Advanced Configuration + +Coraza configuration is largely specified through environment variables and can be customized by the user once installed. You can modify these environment variables in the workload configuration to adjust Coraza's behavior, logging levels, and security policies according to your specific requirements. + +### Usage + +The Coraza WAF will act as a reverse proxy, filtering incoming requests before forwarding them to your target workload. Configure the `targetWorkload` and `targetPort` values to point to your application, then the WAF will be accessible on the specified `WAFPort`. + +**Important**: The target workload must be configured with internal access set to `same-gvc`, `same-org`, or specifically allow this workload in order for the WAF to reach it. + +### Security Features + +Coraza provides web application firewall capabilities including: +- Automatic integration of OWASP Core Rule Set (CRS) for comprehensive protection +- Request filtering and validation +- Protection against common web attacks +- Custom rule configuration +- Traffic monitoring and logging + +### Custom Rules + +After installation, you can add custom rules by editing the created secret with the suffix `coraza-custom-rules`. The secret contains an example rule that blocks requests containing "attack" in the URI: + +``` +SecRule REQUEST_URI "@rx attack" "id:1001,phase:1,deny,msg:'Blocked attack attempt'" +``` + +**Note**: After modifying the custom rules secret, you must restart the workload replicas for the changes to take effect. See the Coraza and CRS documentation below for instructions on creating custom rules. + +## Additional Resources + +- [OWASP Coraza Docs](https://coraza.io/docs/tutorials/introduction/) +- [OWASP CRS Docs](https://coreruleset.org/docs/) +- [Coraza Caddy README](https://github.com/coreruleset/coraza-crs-docker#) \ No newline at end of file diff --git a/coraza/versions/1.1.1/templates/_helpers.tpl b/coraza/versions/1.1.1/templates/_helpers.tpl new file mode 100644 index 00000000..be9ee0b8 --- /dev/null +++ b/coraza/versions/1.1.1/templates/_helpers.tpl @@ -0,0 +1,46 @@ +{{/* Resource Naming */}} + +{{/* +Coraza Workload Name +*/}} +{{- define "coraza.name" -}} +{{- printf "%s-coraza-waf" .Release.Name }} +{{- end }} + +{{/* +Coraza Secret Custom Rules Name +*/}} +{{- define "coraza.secretRules.name" -}} +{{- printf "%s-coraza-custom-rules" .Release.Name }} +{{- end }} + +{{/* +Coraza Secret Startup Name +*/}} +{{- define "coraza.secretStartup.name" -}} +{{- printf "%s-coraza-startup" .Release.Name }} +{{- end }} + +{{/* +Coraza Identity Name +*/}} +{{- define "coraza.identity.name" -}} +{{- printf "%s-coraza-identity" .Release.Name }} +{{- end }} + +{{/* +Coraza Policy Name +*/}} +{{- define "coraza.policy.name" -}} +{{- printf "%s-coraza-policy" .Release.Name }} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Common labels +*/}} +{{- define "coraza.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} diff --git a/coraza/versions/1.1.1/templates/identity.yaml b/coraza/versions/1.1.1/templates/identity.yaml new file mode 100644 index 00000000..aa2418c8 --- /dev/null +++ b/coraza/versions/1.1.1/templates/identity.yaml @@ -0,0 +1,6 @@ +--- +kind: identity +gvc: {{ .Values.global.cpln.gvc }} +name: {{ include "coraza.identity.name" . }} +description: Coraza identity +tags: {{- include "coraza.tags" . | nindent 4 }} \ No newline at end of file diff --git a/coraza/versions/1.1.1/templates/policy.yaml b/coraza/versions/1.1.1/templates/policy.yaml new file mode 100644 index 00000000..c560cae9 --- /dev/null +++ b/coraza/versions/1.1.1/templates/policy.yaml @@ -0,0 +1,14 @@ +--- +kind: policy +name: {{ include "coraza.policy.name" . }} +description: Coraza WAF policy +tags: {{- include "coraza.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "coraza.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "coraza.secretStartup.name" . }} + - //secret/{{ include "coraza.secretRules.name" . }} \ No newline at end of file diff --git a/coraza/versions/1.1.1/templates/secret-custom-rules.yaml b/coraza/versions/1.1.1/templates/secret-custom-rules.yaml new file mode 100644 index 00000000..0e025824 --- /dev/null +++ b/coraza/versions/1.1.1/templates/secret-custom-rules.yaml @@ -0,0 +1,15 @@ +--- +kind: secret +name: {{ include "coraza.secretRules.name" . }} +description: Coraza WAF custom rules +tags: {{- include "coraza.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + # Add your custom rules here + + # Example Rule: block requests containing "attack" in the URI + # Test by querying the endpoint with /attack + + SecRule REQUEST_URI "@rx attack" "id:1001,phase:1,deny,msg:'Blocked attack attempt'" \ No newline at end of file diff --git a/coraza/versions/1.1.1/templates/secret-startup.yaml b/coraza/versions/1.1.1/templates/secret-startup.yaml new file mode 100644 index 00000000..50c37dda --- /dev/null +++ b/coraza/versions/1.1.1/templates/secret-startup.yaml @@ -0,0 +1,88 @@ +--- +kind: secret +name: {{ include "coraza.secretStartup.name" . }} +description: Coraza WAF startup script +tags: {{- include "coraza.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: > + #!/bin/sh + + set -u + + # Script sends patch request to Caddy admin API to configure proper header for reverse proxy to target workload + + # Caddy admin API URL and port + + ADMIN_URL="127.0.0.1" + + ADMIN_PORT="2019" + + ROUTE_PATH="/config/apps/http/servers/srv0/routes/0/handle/1" + + PATCH_FILE="/tmp/proxy_patch.json" + + + {{ if .Values.diskBodyInspection }} + mkdir -p /tmp/coraza + {{ end }} + + echo "[INFO] Waiting for Caddy admin API..." + + until wget -q --spider "http://${ADMIN_URL}:${ADMIN_PORT}/config/" + 2>/dev/null; do + sleep 1 + done + + echo "[INFO] Caddy admin API is up." + + + # Create patch file with proper header + + cat > "${PATCH_FILE}" <<'EOF' + + { + "handler": "reverse_proxy", + "headers": { + "request": { + "set": { + "Host": ["{{ .Values.targetWorkload }}"] + } + } + }, + "trusted_proxies": ["192.168.0.0/16","172.16.0.0/12","10.0.0.0/8","127.0.0.1/8","fd00::/8","::1"], + "upstreams": [ + { "dial": "{{ .Values.targetWorkload }}:{{ .Values.targetPort }}" } + ] + } + + EOF + + + echo "[INFO] Applying PATCH via netcat..." + + LEN=$(wc -c < "${PATCH_FILE}") + + # Requests patch command to admin API using netcat + + RESPONSE=$( + { + printf "PATCH %s HTTP/1.1\r\nHost: %s:%s\r\nContent-Type: application/json\r\nContent-Length: %s\r\nConnection: close\r\n\r\n" \ + "${ROUTE_PATH}" "${ADMIN_URL}" "${ADMIN_PORT}" "${LEN}" + cat "${PATCH_FILE}" + } | nc ${ADMIN_URL} ${ADMIN_PORT} || true + ) + + + echo "$RESPONSE" + + + if echo "$RESPONSE" | grep -q "200 OK"; then + echo "[INFO] Patch succeeded." + else + echo "[WARN] Patch may have failed — see response above." + fi + + + rm -f "${PATCH_FILE}" \ No newline at end of file diff --git a/coraza/versions/1.1.1/templates/workload.yaml b/coraza/versions/1.1.1/templates/workload.yaml new file mode 100644 index 00000000..868d2e55 --- /dev/null +++ b/coraza/versions/1.1.1/templates/workload.yaml @@ -0,0 +1,82 @@ +kind: workload +name: {{ include "coraza.name" . }} +description: Coraza Web Application Firewall (WAF) +tags: {{- include "coraza.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: coraza-crs + cpu: {{ .Values.resources.cpu | quote }} + env: + - name: ACCESSLOG + value: /dev/stdout + - name: BACKEND + value: http://{{ .Values.targetWorkload }}:{{ .Values.targetPort }} + - name: CORAZA_AUDIT_LOG + value: /dev/stdout + - name: CORAZA_AUDIT_LOG_PARTS + value: ABDEFHIJZ + - name: CORAZA_DEBUG_LOG + value: /dev/stdout + - name: CORAZA_DEBUG_LOGLEVEL + value: '1' + - name: CORAZA_RULE_ENGINE + value: 'On' + {{- if not .Values.diskBodyInspection }} + - name: CORAZA_REQ_BODY_NOFILES_LIMIT + value: '13107200' + {{- end }} + - name: PORT + value: "{{ .Values.WAFPort }}" + image: {{ .Values.image }} + inheritEnv: false + lifecycle: + postStart: + exec: + command: + - /bin/sh + - /opt/coraza/startup.sh + memory: {{ .Values.resources.memory | quote }} + ports: + - number: {{ .Values.WAFPort }} + protocol: http + volumes: + - path: /opt/coraza/rules.d/custom.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "coraza.secretRules.name" . }} + - path: /opt/coraza/startup.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "coraza.secretStartup.name" . }} + defaultOptions: + multiZone: + enabled: {{ .Values.multiZone }} + autoscaling: + maxConcurrency: 0 + maxScale: 3 + metric: cpu + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: + - 0.0.0.0/0 + inboundBlockedCIDR: [] + outboundAllowCIDR: [] + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "coraza.identity.name" . }} + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false + supportDynamicTags: false diff --git a/coraza/versions/1.1.1/values.yaml b/coraza/versions/1.1.1/values.yaml new file mode 100644 index 00000000..a28eee54 --- /dev/null +++ b/coraza/versions/1.1.1/values.yaml @@ -0,0 +1,16 @@ +image: ghcr.io/coreruleset/coraza-crs@sha256:eed7280e0de4820507b500b1ee10de820c175165d5cce329609bf34f32977af8 # Coraza GitHub image + +# MUST BE CHANGED +targetWorkload: my-workload.my-gvc.cpln.local # Workload internal name of the workload to proxy traffic to + +targetPort: 8080 # Port of the workload to proxy traffic to + +WAFPort: 80 # Port on the WAF workload to expose to the internet + +resources: + cpu: 50m + memory: 128Mi + +multiZone: false + +diskBodyInspection: true # When true, request bodies exceeding the in-memory limit are buffered to disk for inspection. Disable to keep all body inspection in memory. \ No newline at end of file From 25f61062cb8edf327010f6b80c023377e6e13dc0 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Wed, 13 May 2026 17:42:28 -0700 Subject: [PATCH 43/58] reverted to single location --- cassandra/versions/1.0.0/templates/gvc.yaml | 11 -------- .../versions/1.0.0/templates/identity.yaml | 1 - .../versions/1.0.0/templates/policy.yaml | 2 +- .../1.0.0/templates/secret-config.yaml | 8 +++--- .../versions/1.0.0/templates/secret-init.yaml | 24 +++++++---------- .../versions/1.0.0/templates/volumeset.yaml | 1 - .../1.0.0/templates/workload-cassandra.yaml | 26 +++---------------- cassandra/versions/1.0.0/values.yaml | 8 +----- 8 files changed, 19 insertions(+), 62 deletions(-) delete mode 100644 cassandra/versions/1.0.0/templates/gvc.yaml diff --git a/cassandra/versions/1.0.0/templates/gvc.yaml b/cassandra/versions/1.0.0/templates/gvc.yaml deleted file mode 100644 index 744088fb..00000000 --- a/cassandra/versions/1.0.0/templates/gvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: gvc -name: {{ .Values.gvc.name }} -description: {{ .Values.gvc.name }} -tags: {{- include "cassandra.tags" . | nindent 2 }} -spec: - endpointNamingFormat: org - staticPlacement: - locationLinks: - {{- range .Values.gvc.locations }} - - //location/{{ .name }} - {{- end }} diff --git a/cassandra/versions/1.0.0/templates/identity.yaml b/cassandra/versions/1.0.0/templates/identity.yaml index 9513b667..505afc81 100644 --- a/cassandra/versions/1.0.0/templates/identity.yaml +++ b/cassandra/versions/1.0.0/templates/identity.yaml @@ -1,5 +1,4 @@ kind: identity name: {{ include "cassandra.identity.name" . }} description: {{ include "cassandra.workload.name" . }} identity -gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} diff --git a/cassandra/versions/1.0.0/templates/policy.yaml b/cassandra/versions/1.0.0/templates/policy.yaml index 8b98159d..51bfc516 100644 --- a/cassandra/versions/1.0.0/templates/policy.yaml +++ b/cassandra/versions/1.0.0/templates/policy.yaml @@ -5,7 +5,7 @@ bindings: - permissions: - reveal principalLinks: - - //gvc/{{ .Values.gvc.name }}/identity/{{ include "cassandra.identity.name" . }} + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} targetKind: secret targetLinks: - //secret/{{ include "cassandra.secret.init.name" . }} diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml index 2034f1f4..447c3819 100644 --- a/cassandra/versions/1.0.0/templates/secret-config.yaml +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -7,13 +7,13 @@ data: cluster_name: '{{ .Values.clusterName }}' # Networking — replaced at pod startup by the init script. - # listen_address binds the gossip port to the pod IP; Envoy's inbound proxy - # forwards to the original destination (pod IP) via ORIGINAL_DST. + # listen_address and broadcast_address both use the pod's own FQDN (resolves + # to pod IP). Within a single location all pods share the same cluster network. listen_address: LISTEN_ADDRESS_PLACEHOLDER - broadcast_address: BROADCAST_ADDRESS_PLACEHOLDER + broadcast_address: LISTEN_ADDRESS_PLACEHOLDER storage_port: 9043 rpc_address: 0.0.0.0 - broadcast_rpc_address: BROADCAST_ADDRESS_PLACEHOLDER + broadcast_rpc_address: LISTEN_ADDRESS_PLACEHOLDER # Seeds — replaced at pod startup by the init script seed_provider: diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index f23e6c8f..42c90d64 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -21,26 +21,23 @@ data: MY_IP=$(grep -E "^[0-9]" /etc/hosts | grep "${HOSTNAME}" | awk '{print $1}') REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') LOCATION=$(basename "${CPLN_LOCATION}") + WORKLOAD_NAME=$(basename "${CPLN_WORKLOAD}") + GVC_NAME=$(basename "${CPLN_GVC}") echo "HOSTNAME: ${HOSTNAME}" echo "MY_FQDN: ${MY_FQDN}" echo "MY_IP: ${MY_IP}" echo "REPLICA_INDEX: ${REPLICA_INDEX}" echo "LOCATION: ${LOCATION}" + echo "WORKLOAD_NAME: ${WORKLOAD_NAME}" + echo "GVC_NAME: ${GVC_NAME}" - # Broadcast address: replicaDirect cpln.local FQDN for this replica. - # Resolves to a stable VIP that Envoy routes cross-location on port 9043. - BROADCAST_ADDR="replica-${REPLICA_INDEX}.${CASSANDRA_WORKLOAD}.${LOCATION}.${CASSANDRA_GVC}.cpln.local" - echo "BROADCAST_ADDR: ${BROADCAST_ADDR}" - - # Seeds: replicaDirect FQDNs for all replicas in all locations. - # Stable across redeploys — no pod IPs needed. + # Seeds: replicaDirect FQDNs for all replicas in this location. + # These resolve to stable per-replica VIPs via CP's DNS — no pod IPs needed. SEEDS="" - {{- range .Values.gvc.locations }} - for i in $(seq 0 $(( {{ .replicas | int }} - 1 ))); do - SEEDS="${SEEDS}replica-${i}.${CASSANDRA_WORKLOAD}.{{ .name }}.${CASSANDRA_GVC}.cpln.local," + for i in $(seq 0 $(( {{ .Values.replicas | int }} - 1 ))); do + SEEDS="${SEEDS}replica-${i}.${WORKLOAD_NAME}.${LOCATION}.${GVC_NAME}.cpln.local," done - {{- end }} SEEDS="${SEEDS%,}" echo "SEEDS: ${SEEDS}" @@ -58,9 +55,8 @@ data: # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) cp /cassandra-config/cassandra.yaml /etc/cassandra/cassandra.yaml - sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml - sed -i "s|BROADCAST_ADDRESS_PLACEHOLDER|${BROADCAST_ADDR}|g" /etc/cassandra/cassandra.yaml - sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml + sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml + sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml echo "cassandra.yaml written. Starting Cassandra..." # nodetool drain is run via the workload preStop hook before SIGTERM reaches diff --git a/cassandra/versions/1.0.0/templates/volumeset.yaml b/cassandra/versions/1.0.0/templates/volumeset.yaml index 12af3e3f..9f077cc1 100644 --- a/cassandra/versions/1.0.0/templates/volumeset.yaml +++ b/cassandra/versions/1.0.0/templates/volumeset.yaml @@ -1,7 +1,6 @@ kind: volumeset name: {{ include "cassandra.volumeset.name" . }} description: {{ include "cassandra.workload.name" . }} data -gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} spec: initialCapacity: {{ .Values.volumes.data.initialCapacity }} diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 7a92cbe9..68a6ec61 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -1,7 +1,6 @@ kind: workload name: {{ include "cassandra.workload.name" . }} description: Cassandra cluster -gvc: {{ .Values.gvc.name }} tags: {{- include "cassandra.tags" . | nindent 2 }} spec: type: stateful @@ -17,10 +16,6 @@ spec: env: - name: MAX_HEAP_SIZE value: {{ .Values.jvmHeapSize | quote }} - - name: CASSANDRA_WORKLOAD - value: {{ include "cassandra.workload.name" . | quote }} - - name: CASSANDRA_GVC - value: {{ .Values.gvc.name | quote }} lifecycle: preStop: exec: @@ -66,37 +61,22 @@ spec: defaultOptions: autoscaling: maxConcurrency: 0 - maxScale: {{ (index .Values.gvc.locations 0).replicas | int }} + maxScale: {{ .Values.replicas | int }} metric: disabled - minScale: {{ (index .Values.gvc.locations 0).replicas | int }} + minScale: {{ .Values.replicas | int }} scaleToZeroDelay: 300 target: 95 capacityAI: false debug: false suspend: false timeoutSeconds: 60 - localOptions: - {{- range .Values.gvc.locations }} - - location: //location/{{ .name }} - autoscaling: - maxConcurrency: 0 - maxScale: {{ .replicas | int }} - metric: disabled - minScale: {{ .replicas | int }} - scaleToZeroDelay: 300 - target: 95 - capacityAI: false - debug: false - suspend: false - timeoutSeconds: 60 - {{- end }} firewallConfig: internal: inboundAllowType: {{ .Values.internal_access.type }} {{- if .Values.internal_access.workloads }} inboundAllowWorkload: {{ .Values.internal_access.workloads | toYaml | nindent 8 }} {{- end }} - identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "cassandra.identity.name" . }} + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} loadBalancer: direct: enabled: false diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 754fcf1c..62d040a0 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,10 +1,4 @@ -gvc: - name: cassandra-gvc - locations: - - name: aws-us-east-2 - replicas: 1 - - name: aws-us-west-2 - replicas: 1 +replicas: 3 image: cassandra:5.0 cpu: 2 From 5857affbe6870b4e895e49492da648d32acc6159 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Wed, 13 May 2026 17:59:04 -0700 Subject: [PATCH 44/58] updated hostname in script --- cassandra/versions/1.0.0/templates/secret-init.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 42c90d64..d7c89689 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -32,11 +32,13 @@ data: echo "WORKLOAD_NAME: ${WORKLOAD_NAME}" echo "GVC_NAME: ${GVC_NAME}" - # Seeds: replicaDirect FQDNs for all replicas in this location. - # These resolve to stable per-replica VIPs via CP's DNS — no pod IPs needed. + # Seeds: stable per-replica headless-service FQDNs (K8s stateful-set DNS). + # Strip the pod-specific prefix from MY_FQDN to get the base domain suffix, + # then construct the FQDN for every replica without hard-coding the namespace hash. + FQDN_BASE="${MY_FQDN#${HOSTNAME}.}" SEEDS="" for i in $(seq 0 $(( {{ .Values.replicas | int }} - 1 ))); do - SEEDS="${SEEDS}replica-${i}.${WORKLOAD_NAME}.${LOCATION}.${GVC_NAME}.cpln.local," + SEEDS="${SEEDS}${WORKLOAD_NAME}-${i}.${FQDN_BASE}," done SEEDS="${SEEDS%,}" echo "SEEDS: ${SEEDS}" From 902e261968ffa20d53bdf5f02f2213c374b9ed3d Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 14 May 2026 11:59:44 -0700 Subject: [PATCH 45/58] added auth + database credentials and replication factor --- .../versions/1.0.0/templates/_helpers.tpl | 4 ++ .../1.0.0/templates/secret-config.yaml | 3 + .../1.0.0/templates/secret-credentials.yaml | 7 +++ .../versions/1.0.0/templates/secret-init.yaml | 63 ++++++++++++++++--- .../1.0.0/templates/workload-cassandra.yaml | 3 + cassandra/versions/1.0.0/values.yaml | 8 +++ 6 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 cassandra/versions/1.0.0/templates/secret-credentials.yaml diff --git a/cassandra/versions/1.0.0/templates/_helpers.tpl b/cassandra/versions/1.0.0/templates/_helpers.tpl index 65d8c8a0..5fab7974 100644 --- a/cassandra/versions/1.0.0/templates/_helpers.tpl +++ b/cassandra/versions/1.0.0/templates/_helpers.tpl @@ -24,6 +24,10 @@ {{- printf "%s-cassandra-data" .Release.Name }} {{- end }} +{{- define "cassandra.secret.credentials.name" -}} +{{- printf "%s-cassandra-credentials" .Release.Name }} +{{- end }} + {{/* Labeling */}} diff --git a/cassandra/versions/1.0.0/templates/secret-config.yaml b/cassandra/versions/1.0.0/templates/secret-config.yaml index 447c3819..19fe8c90 100644 --- a/cassandra/versions/1.0.0/templates/secret-config.yaml +++ b/cassandra/versions/1.0.0/templates/secret-config.yaml @@ -34,3 +34,6 @@ data: saved_caches_directory: /var/lib/cassandra/saved_caches endpoint_snitch: GossipingPropertyFileSnitch + + authenticator: PasswordAuthenticator + authorizer: CassandraAuthorizer diff --git a/cassandra/versions/1.0.0/templates/secret-credentials.yaml b/cassandra/versions/1.0.0/templates/secret-credentials.yaml new file mode 100644 index 00000000..1c861941 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/secret-credentials.yaml @@ -0,0 +1,7 @@ +kind: secret +name: {{ include "cassandra.secret.credentials.name" . }} +type: dictionary +data: + username: {{ .Values.username }} + password: {{ .Values.password }} + keyspace: {{ .Values.keyspaceName }} diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index d7c89689..13a65f3a 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -60,12 +60,61 @@ data: sed -i "s|LISTEN_ADDRESS_PLACEHOLDER|${MY_FQDN}|g" /etc/cassandra/cassandra.yaml sed -i "s|SEEDS_PLACEHOLDER|${SEEDS}|g" /etc/cassandra/cassandra.yaml - echo "cassandra.yaml written. Starting Cassandra..." - # nodetool drain is run via the workload preStop hook before SIGTERM reaches - # this process, so exec here is safe — Cassandra stays PID 1. - if [ "$(id -u)" = "0" ]; then - chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true - exec gosu cassandra cassandra -f + echo "cassandra.yaml written." + + if [ "${REPLICA_INDEX}" = "0" ]; then + # Replica 0 runs bootstrap after Cassandra is ready. + # exec cannot be used here because bootstrap runs post-startup, so we + # background Cassandra, trap SIGTERM for forwarding, and wait on its PID. + _stop() { kill -TERM "${CASS_PID}" 2>/dev/null; wait "${CASS_PID}" 2>/dev/null || true; } + trap _stop TERM INT + if [ "$(id -u)" = "0" ]; then + chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true + gosu cassandra cassandra -f & + else + cassandra -f & + fi + CASS_PID=$! + + echo "Waiting for Cassandra to accept CQL connections..." + ATTEMPTS=0 + until cqlsh 127.0.0.1 9042 -u cassandra -p cassandra -e "SELECT now() FROM system.local" > /dev/null 2>&1; do + ATTEMPTS=$(( ATTEMPTS + 1 )) + if [ "${ATTEMPTS}" -ge 60 ]; then + echo "ERROR: Cassandra did not become CQL-ready after 300 seconds" + exit 1 + fi + sleep 5 + done + echo "CQL ready." + + BOOTSTRAP_FLAG="/var/lib/cassandra/.bootstrapped" + if [ -f "${BOOTSTRAP_FLAG}" ]; then + echo "Bootstrap flag found — skipping first-time initialisation." + else + echo "Running first-time bootstrap..." + + BOOTSTRAP_CQL=/tmp/cassandra-bootstrap.cql + printf "ALTER USER cassandra WITH PASSWORD '%s';\n" "{{ .Values.superuserPassword }}" > "${BOOTSTRAP_CQL}" + printf "ALTER KEYSPACE system_auth WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %s};\n" "{{ .Values.replicationFactor | int }}" >> "${BOOTSTRAP_CQL}" + printf "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %s};\n" "{{ .Values.keyspaceName }}" "{{ .Values.replicationFactor | int }}" >> "${BOOTSTRAP_CQL}" + printf "CREATE ROLE IF NOT EXISTS '%s' WITH PASSWORD = '%s' AND LOGIN = true;\n" "{{ .Values.username }}" "{{ .Values.password }}" >> "${BOOTSTRAP_CQL}" + printf "GRANT ALL PERMISSIONS ON KEYSPACE %s TO '%s';\n" "{{ .Values.keyspaceName }}" "{{ .Values.username }}" >> "${BOOTSTRAP_CQL}" + + cqlsh 127.0.0.1 9042 -u cassandra -p cassandra -f "${BOOTSTRAP_CQL}" + rm -f "${BOOTSTRAP_CQL}" + + touch "${BOOTSTRAP_FLAG}" + echo "Bootstrap complete: keyspace '{{ .Values.keyspaceName }}', user '{{ .Values.username }}' created." + fi + + wait "${CASS_PID}" else - exec cassandra -f + # Non-replica-0: exec directly so Cassandra is PID 1. + if [ "$(id -u)" = "0" ]; then + chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true + exec gosu cassandra cassandra -f + else + exec cassandra -f + fi fi diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 68a6ec61..4515c81e 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -1,3 +1,6 @@ +{{- if gt (.Values.replicationFactor | int) (.Values.replicas | int) }} +{{- fail (printf "replicationFactor (%d) cannot exceed replicas (%d)" (.Values.replicationFactor | int) (.Values.replicas | int)) }} +{{- end }} kind: workload name: {{ include "cassandra.workload.name" . }} description: Cassandra cluster diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 62d040a0..a6fe5730 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,4 +1,12 @@ replicas: 3 +# replicationFactor must not exceed replicas +replicationFactor: 3 + +# IMPORTANT: Change all credentials before deploying to production +superuserPassword: supersecretpassword +username: username +password: password +keyspaceName: mydatabase image: cassandra:5.0 cpu: 2 From 9438723ed46d6c7897bc2f4e1ac1d57cf9cf3bd3 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 14 May 2026 13:40:22 -0700 Subject: [PATCH 46/58] moved wait for repair system_auth --- .../versions/1.0.0/templates/secret-init.yaml | 38 ++++++++++++++++--- cassandra/versions/1.0.0/values.yaml | 4 +- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 13a65f3a..1cab841c 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -92,18 +92,44 @@ data: if [ -f "${BOOTSTRAP_FLAG}" ]; then echo "Bootstrap flag found — skipping first-time initialisation." else + # Wait for all replicas to join before writing any auth data. + # This ensures token ranges are finalised so writes land on the correct + # long-term owners and are not redistributed inconsistently after the fact. + echo "Waiting for all {{ .Values.replicas | int }} replicas to join before bootstrapping..." + WAIT_ATTEMPTS=0 + until [ "$(nodetool status 2>/dev/null | grep -c '^UN')" -eq "{{ .Values.replicas | int }}" ]; do + WAIT_ATTEMPTS=$(( WAIT_ATTEMPTS + 1 )) + if [ "${WAIT_ATTEMPTS}" -ge 60 ]; then + echo "WARN: Timed out waiting for all replicas — proceeding with available nodes" + break + fi + sleep 10 + done + echo "Running first-time bootstrap..." + # Detect which cassandra password to use in case a previous partial bootstrap + # already ran ALTER USER before the script exited. + CQLSH_PASS="cassandra" + if ! cqlsh 127.0.0.1 9042 -u cassandra -p cassandra -e "SELECT now() FROM system.local" > /dev/null 2>&1; then + CQLSH_PASS="{{ .Values.superuserPassword }}" + echo "Default cassandra password no longer valid — using configured superuser password." + fi + BOOTSTRAP_CQL=/tmp/cassandra-bootstrap.cql - printf "ALTER USER cassandra WITH PASSWORD '%s';\n" "{{ .Values.superuserPassword }}" > "${BOOTSTRAP_CQL}" - printf "ALTER KEYSPACE system_auth WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %s};\n" "{{ .Values.replicationFactor | int }}" >> "${BOOTSTRAP_CQL}" - printf "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %s};\n" "{{ .Values.keyspaceName }}" "{{ .Values.replicationFactor | int }}" >> "${BOOTSTRAP_CQL}" - printf "CREATE ROLE IF NOT EXISTS '%s' WITH PASSWORD = '%s' AND LOGIN = true;\n" "{{ .Values.username }}" "{{ .Values.password }}" >> "${BOOTSTRAP_CQL}" - printf "GRANT ALL PERMISSIONS ON KEYSPACE %s TO '%s';\n" "{{ .Values.keyspaceName }}" "{{ .Values.username }}" >> "${BOOTSTRAP_CQL}" + printf "ALTER USER cassandra WITH PASSWORD '%s';\n" "{{ .Values.superuserPassword }}" > "${BOOTSTRAP_CQL}" + printf "ALTER KEYSPACE system_auth WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %s};\n" "{{ .Values.replicas | int }}" >> "${BOOTSTRAP_CQL}" + printf "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %s};\n" "{{ .Values.keyspaceName }}" "{{ .Values.replicationFactor | int }}" >> "${BOOTSTRAP_CQL}" + printf "CREATE ROLE IF NOT EXISTS '%s' WITH PASSWORD = '%s' AND LOGIN = true;\n" "{{ .Values.username }}" "{{ .Values.password }}" >> "${BOOTSTRAP_CQL}" + printf "GRANT ALL PERMISSIONS ON KEYSPACE %s TO '%s';\n" "{{ .Values.keyspaceName }}" "{{ .Values.username }}" >> "${BOOTSTRAP_CQL}" - cqlsh 127.0.0.1 9042 -u cassandra -p cassandra -f "${BOOTSTRAP_CQL}" + cqlsh 127.0.0.1 9042 -u cassandra -p "${CQLSH_PASS}" -f "${BOOTSTRAP_CQL}" rm -f "${BOOTSTRAP_CQL}" + nodetool repair system_auth \ + && echo "system_auth repair complete." \ + || echo "WARN: system_auth repair did not complete — repair cron will handle it" + touch "${BOOTSTRAP_FLAG}" echo "Bootstrap complete: keyspace '{{ .Values.keyspaceName }}', user '{{ .Values.username }}' created." fi diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index a6fe5730..c3bc5d85 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,6 +1,6 @@ -replicas: 3 +replicas: 4 # replicationFactor must not exceed replicas -replicationFactor: 3 +replicationFactor: 2 # IMPORTANT: Change all credentials before deploying to production superuserPassword: supersecretpassword From 16f289bc3966242ca89e48f85ef8dfbb4869f954 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 14 May 2026 14:12:54 -0700 Subject: [PATCH 47/58] reset default values --- cassandra/versions/1.0.0/templates/secret-init.yaml | 3 ++- cassandra/versions/1.0.0/values.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 1cab841c..96f62d56 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -78,7 +78,8 @@ data: echo "Waiting for Cassandra to accept CQL connections..." ATTEMPTS=0 - until cqlsh 127.0.0.1 9042 -u cassandra -p cassandra -e "SELECT now() FROM system.local" > /dev/null 2>&1; do + until cqlsh 127.0.0.1 9042 -u cassandra -p cassandra -e "SELECT now() FROM system.local" > /dev/null 2>&1 \ + || cqlsh 127.0.0.1 9042 -u cassandra -p "{{ .Values.superuserPassword }}" -e "SELECT now() FROM system.local" > /dev/null 2>&1; do ATTEMPTS=$(( ATTEMPTS + 1 )) if [ "${ATTEMPTS}" -ge 60 ]; then echo "ERROR: Cassandra did not become CQL-ready after 300 seconds" diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index c3bc5d85..b44cb218 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -1,6 +1,6 @@ -replicas: 4 +replicas: 3 # replicationFactor must not exceed replicas -replicationFactor: 2 +replicationFactor: 1 # IMPORTANT: Change all credentials before deploying to production superuserPassword: supersecretpassword @@ -9,8 +9,8 @@ password: password keyspaceName: mydatabase image: cassandra:5.0 -cpu: 2 -memory: 8Gi +cpu: 1 +memory: 4Gi # JVM heap: leave ~50% of container memory for off-heap (bloom filters, page cache, etc.) # Cassandra 5.x uses G1GC — only MAX_HEAP_SIZE is valid; HEAP_NEWSIZE is ignored. jvmHeapSize: 2G From 301c560c904506b8cb4b116f6034fe82ded27228 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 14 May 2026 14:35:49 -0700 Subject: [PATCH 48/58] moved validation to helpers file, added repair cron job --- .../versions/1.0.0/templates/_helpers.tpl | 13 ++++ .../versions/1.0.0/templates/secret-init.yaml | 4 ++ .../1.0.0/templates/workload-cassandra.yaml | 6 +- .../1.0.0/templates/workload-repair.yaml | 63 +++++++++++++++++++ cassandra/versions/1.0.0/values.yaml | 5 ++ 5 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 cassandra/versions/1.0.0/templates/workload-repair.yaml diff --git a/cassandra/versions/1.0.0/templates/_helpers.tpl b/cassandra/versions/1.0.0/templates/_helpers.tpl index 5fab7974..a910dc60 100644 --- a/cassandra/versions/1.0.0/templates/_helpers.tpl +++ b/cassandra/versions/1.0.0/templates/_helpers.tpl @@ -28,6 +28,19 @@ {{- printf "%s-cassandra-credentials" .Release.Name }} {{- end }} +{{- define "cassandra.workload.repair.name" -}} +{{- printf "%s-cassandra-repair" .Release.Name }} +{{- end }} + + +{{/* Validation */}} + +{{- define "cassandra.validate" -}} +{{- if gt (.Values.replicationFactor | int) (.Values.replicas | int) }} +{{- fail (printf "replicationFactor (%d) cannot exceed replicas (%d)" (.Values.replicationFactor | int) (.Values.replicas | int)) }} +{{- end }} +{{- end }} + {{/* Labeling */}} diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 96f62d56..30dee799 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -68,6 +68,9 @@ data: # background Cassandra, trap SIGTERM for forwarding, and wait on its PID. _stop() { kill -TERM "${CASS_PID}" 2>/dev/null; wait "${CASS_PID}" 2>/dev/null || true; } trap _stop TERM INT + # Set HOSTNAME to FQDN so cassandra-env.sh advertises the full FQDN as the + # JMX/RMI callback address, making remote nodetool (repair cron) reachable. + export HOSTNAME="${MY_FQDN}" if [ "$(id -u)" = "0" ]; then chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true gosu cassandra cassandra -f & @@ -138,6 +141,7 @@ data: wait "${CASS_PID}" else # Non-replica-0: exec directly so Cassandra is PID 1. + export HOSTNAME="${MY_FQDN}" if [ "$(id -u)" = "0" ]; then chown -R cassandra:cassandra /var/lib/cassandra 2>/dev/null || true exec gosu cassandra cassandra -f diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 4515c81e..56264bdb 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -1,6 +1,4 @@ -{{- if gt (.Values.replicationFactor | int) (.Values.replicas | int) }} -{{- fail (printf "replicationFactor (%d) cannot exceed replicas (%d)" (.Values.replicationFactor | int) (.Values.replicas | int)) }} -{{- end }} +{{- include "cassandra.validate" . }} kind: workload name: {{ include "cassandra.workload.name" . }} description: Cassandra cluster @@ -19,6 +17,8 @@ spec: env: - name: MAX_HEAP_SIZE value: {{ .Values.jvmHeapSize | quote }} + - name: LOCAL_JMX + value: "no" lifecycle: preStop: exec: diff --git a/cassandra/versions/1.0.0/templates/workload-repair.yaml b/cassandra/versions/1.0.0/templates/workload-repair.yaml new file mode 100644 index 00000000..0d4d15a3 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/workload-repair.yaml @@ -0,0 +1,63 @@ +{{- if .Values.repair.enabled }} +kind: workload +name: {{ include "cassandra.workload.repair.name" . }} +description: Scheduled full Cassandra cluster repair +tags: {{- include "cassandra.tags" . | nindent 2 }} +spec: + type: cron + containers: + - name: repair + image: {{ .Values.image }} + cpu: 250m + memory: 256Mi + inheritEnv: false + command: /bin/bash + args: + - '-c' + - | + set -euo pipefail + + GVC_NAME=$(basename "${CPLN_GVC}") + # Cassandra workload name is baked in at render time so the repair job + # always targets the correct stateful workload within this release. + CASSANDRA_WORKLOAD="{{ include "cassandra.workload.name" . }}" + + # Per-replica CP internal DNS: {workload}-{n}.{workload}.{gvc}.cpln.local + FQDN_BASE="${CASSANDRA_WORKLOAD}.${GVC_NAME}.cpln.local" + + echo "Starting full repair on all {{ .Values.replicas | int }} replicas..." + FAILED=0 + for i in $(seq 0 $(( {{ .Values.replicas | int }} - 1 ))); do + HOST="${CASSANDRA_WORKLOAD}-${i}.${FQDN_BASE}" + echo "--- Repairing replica ${i} (${HOST}) ---" + nodetool -h "${HOST}" -p 7199 repair \ + && echo "Replica ${i} repair complete." \ + || { echo "WARN: Replica ${i} repair failed."; FAILED=$(( FAILED + 1 )); } + done + + if [ "${FAILED}" -gt 0 ]; then + echo "WARN: ${FAILED} replica(s) failed repair — will retry on next scheduled run." + else + echo "All replicas repaired successfully." + fi + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 1 + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + suspend: false + firewallConfig: + internal: + inboundAllowType: {{ .Values.internal_access.type }} + job: + schedule: {{ .Values.repair.schedule | quote }} + concurrencyPolicy: Forbid + restartPolicy: Never + historyLimit: 5 +{{- end }} diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index b44cb218..547534d2 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -26,3 +26,8 @@ volumes: internal_access: type: same-gvc workloads: + +repair: + enabled: true + # Cron schedule for full cluster repair (must run within gc_grace_seconds = 10 days) + schedule: "0 2 * * 0" From 051292b979a6cb3be5a84e061d32e5822f7c4e85 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 14 May 2026 16:21:22 -0700 Subject: [PATCH 49/58] updated cron job --- .../versions/1.0.0/templates/secret-init.yaml | 7 +++++++ .../1.0.0/templates/workload-cassandra.yaml | 2 ++ .../1.0.0/templates/workload-repair.yaml | 18 +++++++++--------- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-init.yaml b/cassandra/versions/1.0.0/templates/secret-init.yaml index 30dee799..e842d8ba 100644 --- a/cassandra/versions/1.0.0/templates/secret-init.yaml +++ b/cassandra/versions/1.0.0/templates/secret-init.yaml @@ -54,6 +54,13 @@ data: printf 'dc=%s\nrack=rack1\nprefer_local=true\n' "${LOCATION}" > /etc/cassandra/cassandra-rackdc.properties echo "cassandra-rackdc.properties written: dc=${LOCATION} rack=rack1 prefer_local=true" + # JMX credentials required for remote nodetool (LOCAL_JMX=no enables auth) + printf 'cassandra %s\n' "{{ .Values.superuserPassword }}" > /etc/cassandra/jmxremote.password + chmod 400 /etc/cassandra/jmxremote.password + printf 'cassandra readwrite\n' > /etc/cassandra/jmxremote.access + chmod 644 /etc/cassandra/jmxremote.access + chown cassandra:cassandra /etc/cassandra/jmxremote.password /etc/cassandra/jmxremote.access 2>/dev/null || true + # Copy the mounted config template and replace placeholders with runtime values. # (Heredocs cannot be used inside a Helm YAML payload — << is a YAML merge key.) cp /cassandra-config/cassandra.yaml /etc/cassandra/cassandra.yaml diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 56264bdb..dff38cfe 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -31,6 +31,8 @@ spec: memory: {{ .Values.memory | quote }} inheritEnv: false ports: + - number: 7199 + protocol: tcp - number: 9042 protocol: tcp - number: 9043 diff --git a/cassandra/versions/1.0.0/templates/workload-repair.yaml b/cassandra/versions/1.0.0/templates/workload-repair.yaml index 0d4d15a3..3613b1b2 100644 --- a/cassandra/versions/1.0.0/templates/workload-repair.yaml +++ b/cassandra/versions/1.0.0/templates/workload-repair.yaml @@ -1,7 +1,7 @@ {{- if .Values.repair.enabled }} kind: workload name: {{ include "cassandra.workload.repair.name" . }} -description: Scheduled full Cassandra cluster repair +description: Scheduled Cassandra cluster repair job tags: {{- include "cassandra.tags" . | nindent 2 }} spec: type: cron @@ -17,26 +17,26 @@ spec: - | set -euo pipefail - GVC_NAME=$(basename "${CPLN_GVC}") - # Cassandra workload name is baked in at render time so the repair job - # always targets the correct stateful workload within this release. CASSANDRA_WORKLOAD="{{ include "cassandra.workload.name" . }}" - # Per-replica CP internal DNS: {workload}-{n}.{workload}.{gvc}.cpln.local - FQDN_BASE="${CASSANDRA_WORKLOAD}.${GVC_NAME}.cpln.local" + # The svc.cluster.local hostnames resolve directly to pod IPs, bypassing + # the CP service mesh proxy that blocks JMX (port 7199) connections. + # The namespace hash is extracted from the pod's DNS search domain. + K8S_NAMESPACE=$(awk '/^search/{print $2}' /etc/resolv.conf | cut -d'.' -f1) echo "Starting full repair on all {{ .Values.replicas | int }} replicas..." FAILED=0 for i in $(seq 0 $(( {{ .Values.replicas | int }} - 1 ))); do - HOST="${CASSANDRA_WORKLOAD}-${i}.${FQDN_BASE}" + HOST="${CASSANDRA_WORKLOAD}-${i}.${CASSANDRA_WORKLOAD}.${K8S_NAMESPACE}.svc.cluster.local" echo "--- Repairing replica ${i} (${HOST}) ---" - nodetool -h "${HOST}" -p 7199 repair \ + nodetool -u cassandra -pw "{{ .Values.superuserPassword }}" -h "${HOST}" -p 7199 repair \ && echo "Replica ${i} repair complete." \ || { echo "WARN: Replica ${i} repair failed."; FAILED=$(( FAILED + 1 )); } done if [ "${FAILED}" -gt 0 ]; then - echo "WARN: ${FAILED} replica(s) failed repair — will retry on next scheduled run." + echo "ERROR: ${FAILED} replica(s) failed repair." + exit 1 else echo "All replicas repaired successfully." fi From 5ee560ae851e376685de3b62b086b0b261597b6b Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 15 May 2026 08:30:06 -0700 Subject: [PATCH 50/58] added multizone --- cassandra/versions/1.0.0/templates/workload-cassandra.yaml | 2 ++ cassandra/versions/1.0.0/values.yaml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index dff38cfe..fbb59075 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -73,6 +73,8 @@ spec: target: 95 capacityAI: false debug: false + multiZone: + enabled: {{ .Values.multiZone.enabled }} suspend: false timeoutSeconds: 60 firewallConfig: diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 547534d2..9576cc83 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -23,6 +23,9 @@ volumes: minFreePercentage: 20 scalingFactor: 1.5 +multiZone: + enabled: false + internal_access: type: same-gvc workloads: From b4e92bcc20f17333c6ea240eef1fdf1991b7f09d Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Fri, 15 May 2026 14:36:01 -0700 Subject: [PATCH 51/58] updated backup image and functionality with pgbouncer, added multizone support (#258) * updated backup image, added logic to point to pgbouncer * added multizone for cockroach * updated backup image, added multizone per local option --- .gitignore | 1 + .../versions/1.4.0/templates/workload-backup.yaml | 12 ++++++++++++ .../versions/1.4.0/templates/workload-cockroach.yaml | 4 ++++ cockroach/versions/1.4.0/values.yaml | 12 +++++++----- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b12730e2..4c1b874d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_STORE Chart.lock +charts/ .claude \ No newline at end of file diff --git a/cockroach/versions/1.4.0/templates/workload-backup.yaml b/cockroach/versions/1.4.0/templates/workload-backup.yaml index ed18e4f6..2c9d2d55 100644 --- a/cockroach/versions/1.4.0/templates/workload-backup.yaml +++ b/cockroach/versions/1.4.0/templates/workload-backup.yaml @@ -17,7 +17,19 @@ spec: - name: BACKUP_PROVIDER value: {{ .Values.backup.provider }} - name: COCKROACH_HOST + {{- if .Values.pgbouncer.enabled }} + value: {{ include "cockroach.pgbouncer.name" . }}.{{ .Values.gvc.name }}.cpln.local + {{- else }} value: {{ include "cockroach.name" . }}.{{ .Values.gvc.name }}.cpln.local + {{- end }} + - name: COCKROACH_PORT + {{- if .Values.pgbouncer.enabled }} + value: "5432" + {{- else }} + value: "26257" + {{- end }} + - name: COCKROACH_DB + value: {{ .Values.database.name | quote }} {{- if eq .Values.backup.provider "aws" }} - name: AWS_BUCKET value: {{ .Values.backup.aws.bucket }} diff --git a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml index c26355a7..01c2ccc1 100644 --- a/cockroach/versions/1.4.0/templates/workload-cockroach.yaml +++ b/cockroach/versions/1.4.0/templates/workload-cockroach.yaml @@ -62,6 +62,8 @@ spec: target: 100 capacityAI: false debug: false + multiZone: + enabled: {{ .Values.multiZone }} suspend: false timeoutSeconds: 10 firewallConfig: @@ -95,6 +97,8 @@ spec: capacityAI: false debug: false location: //location/{{ $location.name }} + multiZone: + enabled: {{ $.Values.multiZone }} suspend: {{ if eq ($location.replicas | int) 0 }}true{{ else }}false{{ end }} timeoutSeconds: 10 {{- end }} diff --git a/cockroach/versions/1.4.0/values.yaml b/cockroach/versions/1.4.0/values.yaml index 490cbc25..accbbed6 100644 --- a/cockroach/versions/1.4.0/values.yaml +++ b/cockroach/versions/1.4.0/values.yaml @@ -1,15 +1,17 @@ gvc: name: cockroach-gvc locations: - - name: aws-us-west-2 - replicas: 3 - - name: aws-us-east-2 + - name: aws-us-east-1 replicas: 3 - name: aws-eu-central-1 replicas: 3 + - name: aws-us-west-2 + replicas: 3 image: cockroachdb/cockroach:v25.4.0 +multiZone: false + resources: cpu: 2 memory: 4Gi @@ -57,10 +59,10 @@ pgbouncer: backup: enabled: false - image: controlplanecorporation/cockroach-backup:1.0 + image: ghcr.io/controlplane-com/backup-images/cockroach-backup:1.1 schedule: "0 2 * * *" activeDeadlineSeconds: 14400 # hard kill after 4 hours if backup hangs - location: aws-us-east-2 # run the backup job in the same region as your storage bucket + location: aws-us-east-1 # run the backup job in the same region as your storage bucket resources: cpu: 500m memory: 512Mi From 4b2f46a4392b832d582b2d5901f6b0f256eee3bb Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 15 May 2026 15:04:05 -0700 Subject: [PATCH 52/58] init backup --- .../versions/1.0.0/templates/_helpers.tpl | 31 +++++++++ .../versions/1.0.0/templates/identity.yaml | 18 +++++ .../versions/1.0.0/templates/policy.yaml | 1 + .../1.0.0/templates/secret-credentials.yaml | 7 ++ .../1.0.0/templates/workload-backup.yaml | 65 +++++++++++++++++++ cassandra/versions/1.0.0/values.yaml | 24 +++++++ 6 files changed, 146 insertions(+) create mode 100644 cassandra/versions/1.0.0/templates/workload-backup.yaml diff --git a/cassandra/versions/1.0.0/templates/_helpers.tpl b/cassandra/versions/1.0.0/templates/_helpers.tpl index a910dc60..8dc8cf0c 100644 --- a/cassandra/versions/1.0.0/templates/_helpers.tpl +++ b/cassandra/versions/1.0.0/templates/_helpers.tpl @@ -32,6 +32,10 @@ {{- printf "%s-cassandra-repair" .Release.Name }} {{- end }} +{{- define "cassandra.workload.backup.name" -}} +{{- printf "%s-cassandra-backup" .Release.Name }} +{{- end }} + {{/* Validation */}} @@ -39,6 +43,33 @@ {{- if gt (.Values.replicationFactor | int) (.Values.replicas | int) }} {{- fail (printf "replicationFactor (%d) cannot exceed replicas (%d)" (.Values.replicationFactor | int) (.Values.replicas | int)) }} {{- end }} +{{- if .Values.backup.enabled }} + {{- if not (or (eq .Values.backup.type "logical") (eq .Values.backup.type "physical")) }} + {{- fail (printf "backup.type must be 'logical' or 'physical', got: %s" .Values.backup.type) }} + {{- end }} + {{- if not (or (eq .Values.backup.provider "aws") (eq .Values.backup.provider "gcp")) }} + {{- fail (printf "backup.provider must be 'aws' or 'gcp', got: %s" .Values.backup.provider) }} + {{- end }} + {{- if eq .Values.backup.provider "aws" }} + {{- if not .Values.backup.aws.cloudAccountName }} + {{- fail "backup.aws.cloudAccountName is required when backup.provider is aws" }} + {{- end }} + {{- if not .Values.backup.aws.policyName }} + {{- fail "backup.aws.policyName is required when backup.provider is aws" }} + {{- end }} + {{- if not .Values.backup.aws.bucket }} + {{- fail "backup.aws.bucket is required when backup.provider is aws" }} + {{- end }} + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + {{- if not .Values.backup.gcp.cloudAccountName }} + {{- fail "backup.gcp.cloudAccountName is required when backup.provider is gcp" }} + {{- end }} + {{- if not .Values.backup.gcp.bucket }} + {{- fail "backup.gcp.bucket is required when backup.provider is gcp" }} + {{- end }} + {{- end }} +{{- end }} {{- end }} diff --git a/cassandra/versions/1.0.0/templates/identity.yaml b/cassandra/versions/1.0.0/templates/identity.yaml index 505afc81..4a4229f8 100644 --- a/cassandra/versions/1.0.0/templates/identity.yaml +++ b/cassandra/versions/1.0.0/templates/identity.yaml @@ -2,3 +2,21 @@ kind: identity name: {{ include "cassandra.identity.name" . }} description: {{ include "cassandra.workload.name" . }} identity tags: {{- include "cassandra.tags" . | nindent 2 }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.backup.aws.cloudAccountName }} + policyRefs: + - cpln-connector + - aws::ReadOnlyAccess + - {{ .Values.backup.aws.policyName | quote }} +{{- end }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "gcp") }} +gcp: + bindings: + - resource: //storage.googleapis.com/projects/_/buckets/{{ .Values.backup.gcp.bucket }} + roles: + - roles/storage.objectAdmin + cloudAccountLink: //cloudaccount/{{ .Values.backup.gcp.cloudAccountName }} + scopes: + - https://www.googleapis.com/auth/cloud-platform +{{- end }} diff --git a/cassandra/versions/1.0.0/templates/policy.yaml b/cassandra/versions/1.0.0/templates/policy.yaml index 51bfc516..40708455 100644 --- a/cassandra/versions/1.0.0/templates/policy.yaml +++ b/cassandra/versions/1.0.0/templates/policy.yaml @@ -10,3 +10,4 @@ targetKind: secret targetLinks: - //secret/{{ include "cassandra.secret.init.name" . }} - //secret/{{ include "cassandra.secret.config.name" . }} + - //secret/{{ include "cassandra.secret.credentials.name" . }} diff --git a/cassandra/versions/1.0.0/templates/secret-credentials.yaml b/cassandra/versions/1.0.0/templates/secret-credentials.yaml index 1c861941..12e460ed 100644 --- a/cassandra/versions/1.0.0/templates/secret-credentials.yaml +++ b/cassandra/versions/1.0.0/templates/secret-credentials.yaml @@ -5,3 +5,10 @@ data: username: {{ .Values.username }} password: {{ .Values.password }} keyspace: {{ .Values.keyspaceName }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} + backup-bucket: {{ .Values.backup.aws.bucket | quote }} + aws-region: {{ .Values.backup.aws.region | quote }} +{{- end }} +{{- if and .Values.backup.enabled (eq .Values.backup.provider "gcp") }} + backup-bucket: {{ .Values.backup.gcp.bucket | quote }} +{{- end }} diff --git a/cassandra/versions/1.0.0/templates/workload-backup.yaml b/cassandra/versions/1.0.0/templates/workload-backup.yaml new file mode 100644 index 00000000..3e409454 --- /dev/null +++ b/cassandra/versions/1.0.0/templates/workload-backup.yaml @@ -0,0 +1,65 @@ +{{- if and .Values.backup.enabled (eq .Values.backup.type "logical") }} +kind: workload +name: {{ include "cassandra.workload.backup.name" . }} +description: Scheduled Cassandra logical backup job +tags: {{- include "cassandra.tags" . | nindent 2 }} +spec: + type: cron + containers: + - name: backup + image: {{ .Values.backup.image }} + cpu: {{ .Values.backup.resources.cpu | quote }} + memory: {{ .Values.backup.resources.memory | quote }} + inheritEnv: false + env: + - name: BACKUP_TYPE + value: logical + - name: BACKUP_PROVIDER + value: {{ .Values.backup.provider | quote }} + - name: BACKUP_BUCKET + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.backup-bucket + {{- if eq .Values.backup.provider "aws" }} + - name: AWS_REGION + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.aws-region + - name: BACKUP_PREFIX + value: {{ .Values.backup.aws.prefix | quote }} + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + - name: BACKUP_PREFIX + value: {{ .Values.backup.gcp.prefix | quote }} + {{- end }} + - name: CASSANDRA_HOST + value: {{ include "cassandra.workload.name" . }}.{{ .Values.global.cpln.gvc }}.cpln.local + - name: CASSANDRA_PORT + value: "9042" + - name: CASSANDRA_USER + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.username + - name: CASSANDRA_PASSWORD + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.password + - name: CASSANDRA_KEYSPACE + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.keyspace + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} + defaultOptions: + autoscaling: + maxConcurrency: 0 + maxScale: 1 + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 95 + capacityAI: false + debug: false + suspend: false + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: none + job: + schedule: {{ .Values.backup.schedule | quote }} + concurrencyPolicy: Forbid + restartPolicy: Never + historyLimit: 5 +{{- end }} diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 9576cc83..02aef548 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -30,6 +30,30 @@ internal_access: type: same-gvc workloads: +backup: + enabled: false + type: logical # options: logical, physical + image: + schedule: "0 2 * * *" # daily at 2am UTC + + resources: + cpu: 250m + memory: 256Mi + + provider: aws # options: aws, gcp + + aws: + bucket: my-backup-bucket + region: us-east-1 + cloudAccountName: my-cloud-account + policyName: my-s3-policy + prefix: cassandra/backups + + gcp: + bucket: my-backup-bucket + cloudAccountName: my-cloud-account + prefix: cassandra/backups + repair: enabled: true # Cron schedule for full cluster repair (must run within gc_grace_seconds = 10 days) From 835412596237302b5797aa08a736943e66340781 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 15 May 2026 15:55:26 -0700 Subject: [PATCH 53/58] logical backup working state --- cassandra/versions/1.0.0/templates/secret-credentials.yaml | 1 + cassandra/versions/1.0.0/templates/workload-backup.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cassandra/versions/1.0.0/templates/secret-credentials.yaml b/cassandra/versions/1.0.0/templates/secret-credentials.yaml index 12e460ed..486ad1a6 100644 --- a/cassandra/versions/1.0.0/templates/secret-credentials.yaml +++ b/cassandra/versions/1.0.0/templates/secret-credentials.yaml @@ -5,6 +5,7 @@ data: username: {{ .Values.username }} password: {{ .Values.password }} keyspace: {{ .Values.keyspaceName }} + superuser-password: {{ .Values.superuserPassword }} {{- if and .Values.backup.enabled (eq .Values.backup.provider "aws") }} backup-bucket: {{ .Values.backup.aws.bucket | quote }} aws-region: {{ .Values.backup.aws.region | quote }} diff --git a/cassandra/versions/1.0.0/templates/workload-backup.yaml b/cassandra/versions/1.0.0/templates/workload-backup.yaml index 3e409454..9cd943e0 100644 --- a/cassandra/versions/1.0.0/templates/workload-backup.yaml +++ b/cassandra/versions/1.0.0/templates/workload-backup.yaml @@ -33,9 +33,9 @@ spec: - name: CASSANDRA_PORT value: "9042" - name: CASSANDRA_USER - value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.username + value: cassandra - name: CASSANDRA_PASSWORD - value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.password + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.superuser-password - name: CASSANDRA_KEYSPACE value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.keyspace identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "cassandra.identity.name" . }} From 2d9d3da493a1c20aa1d0cb7a61ac6704c6a7a82d Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 15 May 2026 15:58:46 -0700 Subject: [PATCH 54/58] physical backup added --- .../1.0.0/templates/workload-cassandra.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index fbb59075..8a9129a8 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -63,6 +63,38 @@ spec: - path: /scripts/cassandra-init.sh recoveryPolicy: retain uri: 'cpln://secret/{{ include "cassandra.secret.init.name" . }}' +{{- if and .Values.backup.enabled (eq .Values.backup.type "physical") }} + - name: backup + image: {{ .Values.backup.image }} + cpu: {{ .Values.backup.resources.cpu | quote }} + memory: {{ .Values.backup.resources.memory | quote }} + inheritEnv: false + env: + - name: BACKUP_TYPE + value: physical + - name: BACKUP_SCHEDULE + value: {{ .Values.backup.schedule | quote }} + - name: BACKUP_PROVIDER + value: {{ .Values.backup.provider | quote }} + - name: BACKUP_BUCKET + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.backup-bucket + {{- if eq .Values.backup.provider "aws" }} + - name: AWS_REGION + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.aws-region + - name: BACKUP_PREFIX + value: {{ .Values.backup.aws.prefix | quote }} + {{- end }} + {{- if eq .Values.backup.provider "gcp" }} + - name: BACKUP_PREFIX + value: {{ .Values.backup.gcp.prefix | quote }} + {{- end }} + - name: CASSANDRA_JMX_PASSWORD + value: cpln://secret/{{ include "cassandra.secret.credentials.name" . }}.superuser-password + volumes: + - path: /var/lib/cassandra + recoveryPolicy: retain + uri: 'cpln://volumeset/{{ include "cassandra.volumeset.name" . }}' +{{- end }} defaultOptions: autoscaling: maxConcurrency: 0 From d17f81f81b530a0cf83d05358d427fff4bdb51db Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 15 May 2026 17:43:47 -0700 Subject: [PATCH 55/58] opened firewall for backup and restore on cassandra workload --- cassandra/versions/1.0.0/templates/workload-cassandra.yaml | 6 ++++++ cassandra/versions/1.0.0/values.yaml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml index 8a9129a8..bf901424 100644 --- a/cassandra/versions/1.0.0/templates/workload-cassandra.yaml +++ b/cassandra/versions/1.0.0/templates/workload-cassandra.yaml @@ -110,6 +110,12 @@ spec: suspend: false timeoutSeconds: 60 firewallConfig: + {{- if and .Values.backup.enabled (eq .Values.backup.type "physical") }} + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + {{- end }} internal: inboundAllowType: {{ .Values.internal_access.type }} {{- if .Values.internal_access.workloads }} diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 02aef548..105573c5 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -45,7 +45,7 @@ backup: aws: bucket: my-backup-bucket region: us-east-1 - cloudAccountName: my-cloud-account + cloudAccountName: my-backup-cloudaccount policyName: my-s3-policy prefix: cassandra/backups From 8e4cd9731ca11ab7e639e57df9e1b9c82d1ce276 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 15 May 2026 17:55:37 -0700 Subject: [PATCH 56/58] added readme --- cassandra/versions/1.0.0/README.md | 228 +++++++++++++++++++++++++++ cassandra/versions/1.0.0/values.yaml | 2 +- 2 files changed, 229 insertions(+), 1 deletion(-) diff --git a/cassandra/versions/1.0.0/README.md b/cassandra/versions/1.0.0/README.md index e69de29b..b48284dd 100644 --- a/cassandra/versions/1.0.0/README.md +++ b/cassandra/versions/1.0.0/README.md @@ -0,0 +1,228 @@ +# Cassandra + +This app deploys a Cassandra 5.0 cluster in a single location. Each node runs as a stateful replica with its own persistent volume, forming a peer-to-peer cluster that distributes and replicates data across nodes according to the configured replication factor. The template includes optional scheduled backups (logical or physical) and periodic anti-entropy repair. + +## Architecture + +- **Cassandra cluster**: Multi-node cluster deployed in a single location where each node owns a slice of the token ring and replicates data to peers +- **Per-node volumes**: Each node gets its own persistent volume so SSTable data survives restarts +- **Repair** (optional): Scheduled cron job that runs `nodetool repair` across all nodes to keep data consistent +- **Backup** (optional): Logical (`cqlsh COPY TO`) or physical (`nodetool snapshot`) backup to S3 or GCS + +## Configuration + +### Core Settings + +```yaml +replicas: 3 # Number of Cassandra nodes +replicationFactor: 3 # Copies of each partition stored across the cluster + # Must not exceed replicas + +superuserPassword: supersecretpassword # Built-in cassandra superuser password +username: username # Application user +password: password # Application user password +keyspaceName: mydatabase # Keyspace created on startup + +image: cassandra:5.0 +cpu: 1 +memory: 4Gi +jvmHeapSize: 2G # Set to ~50% of memory — Cassandra needs the rest for off-heap cache +clusterName: my-cassandra +``` + +**Volume** — set the initial storage capacity and optionally enable autoscaling: + +```yaml +volumes: + data: + initialCapacity: 10 # GiB + autoscaling: + maxCapacity: 100 + minFreePercentage: 20 + scalingFactor: 1.5 +``` + +Configure which workloads can reach Cassandra: + +```yaml +internal_access: + type: same-gvc # Options: same-gvc, same-org, workload-list + workloads: + # Uncomment and specify workloads if using workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME +``` + +- `same-gvc`: Allow access from all workloads in the same GVC +- `same-org`: Allow access from all workloads in the org +- `workload-list`: Allow access only from specified workloads + +## Connecting + +Each Cassandra replica is reachable via its own DNS name: + +``` +Host: {release-name}-cassandra-{n}.{gvc}.cpln.local +Port: 9042 (CQL, native transport) +Username: {username} +Password: {password} +Keyspace: {keyspaceName} +``` + +For example, with `release-name=my-app` and `gvc=production`: +- Replica 0: `my-app-cassandra-0.production.cpln.local:9042` +- Replica 1: `my-app-cassandra-1.production.cpln.local:9042` + +Configure your Cassandra driver with all replica addresses as contact points so it can discover the full topology and perform token-aware routing. + +## Replicas vs Replication Factor + +These are two separate settings that work together: + +- **`replicas`** — how many Cassandra nodes are deployed. More nodes means more capacity and better throughput, as the token ring is split across more nodes. +- **`replicationFactor`** — how many copies of each partition are stored across the cluster. A replication factor of 3 means every row exists on 3 different nodes, so the cluster can survive 2 node failures without data loss (with `QUORUM` consistency). + +`replicationFactor` must not exceed `replicas` — you cannot store 3 copies of data across only 2 nodes. + +## Multi-Zone + +When `multiZone.enabled: true`, Control Plane spreads replicas across availability zones within the location: + +```yaml +multiZone: + enabled: true +``` + +With a replication factor of 3 across 3 zones, each zone holds one copy of every partition. The cluster survives a complete zone outage with no data loss, provided your client uses `LOCAL_QUORUM` consistency (reads and writes succeed with responses from the surviving 2 zones). + +Verify your selected location supports multi-zone before enabling this option. + +## Repair + +Cassandra uses eventual consistency — when nodes miss writes during downtime, data can drift out of sync. `nodetool repair` runs an anti-entropy process that compares and reconciles data across all replicas. Repair must complete across all nodes at least once within `gc_grace_seconds` (default: 10 days) to prevent deleted data from reappearing. + +The template includes a scheduled repair cron job: + +```yaml +repair: + enabled: true + schedule: "0 2 * * 0" # Weekly, Sunday at 2am UTC +``` + +The default weekly schedule satisfies the 10-day `gc_grace_seconds` requirement with margin. Do not disable repair in production or increase the interval beyond 10 days. + +Repair can be resource-intensive on large datasets. If it impacts query performance, consider running it during low-traffic windows or increasing node resources. + +## Backing Up + +Two backup modes are available: + +- **Logical** — exports keyspace tables as CSVs using `cqlsh COPY TO`, then uploads to cloud storage. Runs as a standalone cron workload on schedule. Suitable for smaller datasets or when portability matters. +- **Physical** — creates SSTable snapshots using `nodetool snapshot` and syncs them to cloud storage. Runs as a sidecar container on each Cassandra replica. Faster and more space-efficient for large datasets, but backups are per-node and must be restored node-by-node. + +Set `backup.enabled: true`, choose a `type`, set `backup.provider`, and fill in the corresponding cloud block: + +```yaml +backup: + enabled: true + type: logical # logical or physical + image: ghcr.io/controlplane-com/backup-images/cassandra-backup:5.0 + schedule: "0 2 * * *" # daily at 2am UTC + + resources: + cpu: 250m + memory: 256Mi + + provider: aws # aws or gcp + + aws: + bucket: my-backup-bucket + region: us-east-1 + cloudAccountName: my-backup-cloudaccount + policyName: my-s3-policy + prefix: cassandra/backups + + gcp: + bucket: my-backup-bucket + cloudAccountName: my-cloud-account + prefix: cassandra/backups +``` + +### AWS S3 + +1. Create your S3 bucket. Set `aws.bucket` and `aws.region` to match. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Set `aws.cloudAccountName` to match. + +3. Create an AWS IAM policy with the following JSON (replace `YOUR_BUCKET_NAME`): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetObjectVersion", + "s3:DeleteObjectVersion" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + } + ] +} +``` + +4. Set `aws.policyName` to the name of the policy created in step 3. + +### GCS + +1. Create your GCS bucket. Set `gcp.bucket` to match. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Set `gcp.cloudAccountName` to match. + +**Important**: Add the `Storage Admin` role to the GCP service account created for the Cloud Account. + +## Restoring a Backup + +### Logical Restore + +Exec into the backup cron workload and run `restore.sh` with the timestamp of the backup you want to restore: + +```bash +RESTORE_TIMESTAMP=2026-05-15T02-00-00Z /usr/local/bin/restore.sh +``` + +The timestamp format matches the backup filename in your bucket (e.g. `cassandra/backups/2026-05-15T02-00-00Z/`). + +The script downloads the CSVs for the configured keyspace and replays them into Cassandra using `cqlsh COPY FROM`. Existing rows with matching primary keys are overwritten; rows not in the backup are left in place. + +### Physical Restore + +Physical backups are per-node — each replica backed up its own SSTable slice. To restore, exec into the **backup sidecar container** (not the cassandra container) on each replica that needs to be restored and run: + +```bash +RESTORE_TIMESTAMP=2026-05-15T02-00-00Z /usr/local/bin/restore.sh +``` + +The script downloads the snapshot files for that replica from `{prefix}/{timestamp}/{hostname}/`, writes them to the shared volume, then calls `nodetool import` to load the SSTables into the live Cassandra instance without a restart. + +**Important**: Repeat this on every replica. Because each node owns a different token range, restoring only one replica leaves the cluster with incomplete data. + +## Important Notes + +- **Minimum replicas for production**: Use at least 3 replicas with a replication factor of 3 so the cluster can survive a node failure while still achieving quorum +- **JVM heap**: Set `jvmHeapSize` to approximately 50% of `memory` — Cassandra relies heavily on off-heap memory for bloom filters, row cache, and OS page cache +- **gc_grace_seconds**: The default is 10 days. Ensure repair runs at least once within this window on all nodes, or deleted data may reappear after a node recovers from downtime +- **Scaling up**: Adding replicas after initial deployment does not automatically rebalance data. Run `nodetool rebuild` on new nodes and then `nodetool cleanup` on existing nodes after scaling +- **Multi-zone**: Verify your selected location supports multi-zone before enabling + +## Supported External Services + +- [Cassandra Documentation](https://cassandra.apache.org/doc/latest/) +- [Cassandra Driver Documentation](https://docs.datastax.com/en/developer/driver-matrix/doc/common/driverMatrix.html) diff --git a/cassandra/versions/1.0.0/values.yaml b/cassandra/versions/1.0.0/values.yaml index 105573c5..debbcdc0 100644 --- a/cassandra/versions/1.0.0/values.yaml +++ b/cassandra/versions/1.0.0/values.yaml @@ -33,7 +33,7 @@ internal_access: backup: enabled: false type: logical # options: logical, physical - image: + image: ghcr.io/controlplane-com/backup-images/cassandra-backup:5.0 schedule: "0 2 * * *" # daily at 2am UTC resources: From aa11a346f59c20455c65166c8417c6493764d4e5 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Mon, 18 May 2026 13:28:22 -0700 Subject: [PATCH 57/58] updated readme --- cassandra/versions/1.0.0/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cassandra/versions/1.0.0/README.md b/cassandra/versions/1.0.0/README.md index b48284dd..8913d436 100644 --- a/cassandra/versions/1.0.0/README.md +++ b/cassandra/versions/1.0.0/README.md @@ -68,11 +68,7 @@ Password: {password} Keyspace: {keyspaceName} ``` -For example, with `release-name=my-app` and `gvc=production`: -- Replica 0: `my-app-cassandra-0.production.cpln.local:9042` -- Replica 1: `my-app-cassandra-1.production.cpln.local:9042` - -Configure your Cassandra driver with all replica addresses as contact points so it can discover the full topology and perform token-aware routing. +Provide multiple node hostnames as contact points in your application so it can discover the full cluster topology. ## Replicas vs Replication Factor From 4328d1512f1f16ca7099d15235a9e47bec36c262 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Mon, 18 May 2026 13:46:00 -0700 Subject: [PATCH 58/58] changed createGvc to false --- cassandra/versions/1.0.0/Chart.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cassandra/versions/1.0.0/Chart.yaml b/cassandra/versions/1.0.0/Chart.yaml index 9073d1b9..6fe2b3ca 100644 --- a/cassandra/versions/1.0.0/Chart.yaml +++ b/cassandra/versions/1.0.0/Chart.yaml @@ -6,10 +6,10 @@ version: 1.0.0 appVersion: "5.0" annotations: - created: "2026-04-21" - lastModified: "2026-04-28" + created: "2026-05-18" + lastModified: "2026-05-18" category: "database" - createsGvc: true + createsGvc: false dependencies: - name: cpln-common