From 8362cc35691ff64ce5ad401f43ca141910255b76 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Thu, 9 Jun 2016 17:51:39 -0400 Subject: [PATCH] Porting kafka binder specific code from spring cloud streams --- .gitignore | 24 + .mvn/jvm.config | 1 + .mvn/maven.config | 1 + .mvn/wrapper/maven-wrapper.jar | Bin 0 -> 49502 bytes .mvn/wrapper/maven-wrapper.properties | 1 + .settings.xml | 66 ++ LICENSE | 201 +++++ README.adoc | 2 + mvnw | 234 +++++ mvnw.cmd | 234 +++++ pom.xml | 94 ++ spring-cloud-starter-stream-kafka/pom.xml | 25 + .../main/resources/META-INF/spring.provides | 1 + spring-cloud-stream-binder-kafka/pom.xml | 143 ++++ .../kafka/KafkaBinderHealthIndicator.java | 95 ++ .../binder/kafka/KafkaBindingProperties.java | 43 + .../binder/kafka/KafkaConsumerProperties.java | 83 ++ .../kafka/KafkaExtendedBindingProperties.java | 60 ++ .../kafka/KafkaMessageChannelBinder.java | 800 +++++++++++++++++ .../binder/kafka/KafkaProducerProperties.java | 68 ++ .../binder/kafka/WindowingOffsetManager.java | 262 ++++++ .../config/KafkaBinderConfiguration.java | 79 ++ .../KafkaBinderConfigurationProperties.java | 243 ++++++ .../stream/binder/kafka/package-info.java | 20 + .../main/resources/META-INF/spring.binders | 2 + .../stream/binder/kafka/KafkaBinderTests.java | 808 ++++++++++++++++++ .../stream/binder/kafka/KafkaTestBinder.java | 89 ++ .../kafka/RawKafkaPartitionTestSupport.java | 38 + .../binder/kafka/RawModeKafkaBinderTests.java | 262 ++++++ 29 files changed, 3979 insertions(+) create mode 100644 .gitignore create mode 100644 .mvn/jvm.config create mode 100644 .mvn/maven.config create mode 100644 .mvn/wrapper/maven-wrapper.jar create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100644 .settings.xml create mode 100644 LICENSE create mode 100755 mvnw create mode 100644 mvnw.cmd create mode 100644 pom.xml create mode 100644 spring-cloud-starter-stream-kafka/pom.xml create mode 100644 spring-cloud-starter-stream-kafka/src/main/resources/META-INF/spring.provides create mode 100644 spring-cloud-stream-binder-kafka/pom.xml create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingProperties.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaConsumerProperties.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExtendedBindingProperties.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaProducerProperties.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/WindowingOffsetManager.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfigurationProperties.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/package-info.java create mode 100644 spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders create mode 100644 spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java create mode 100644 spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java create mode 100644 spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java create mode 100644 spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawModeKafkaBinderTests.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..42bdaef55 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +/application.yml +/application.properties +asciidoctor.css +*~ +.#* +*# +target/ +build/ +bin/ +_site/ +.classpath +.project +.settings +.springBeans +.DS_Store +*.sw* +*.iml +*.ipr +*.iws +.idea/* +.factorypath +dump.rdb +.apt_generated +artifacts diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 000000000..0e7dabeff --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 000000000..3b8cf46e1 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1 @@ +-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..5fd4d5023f1463b5ba3970e33c460c1eb26d748d GIT binary patch literal 49502 zcmb@tV|1n6wzeBvGe*U>ZQHh;%-Bg)Y}={WHY%yuwkkF%MnzxVwRUS~wY|@J_gP;% z^VfXZ{5793?z><89(^dufT2xlYVOQnYG>@?lA@vQF|UF0&X7tk8BUf?wq2J& zZe&>>paKUg4@;fwk0yeUPvM$yk)=f>TSFFB^a8f|_@mbE#MaBnd5qf6;hXq}c%IeK zn7gB0Kldbedq-vl@2wxJi{$%lufroKUjQLSFmt|<;M8~<5otM5ur#Dgc@ivmwRiYZW(Oco7kb8DWmo|a{coqYMU2raB9r6e9viK6MI3c&%jp05-Tf*O#6@8Ra=egYy01 z-V!G;_omANEvU-8!*>*)lWka9M<+IkNsrsenbXOfLc6qrYe`;lpst;vfs*70$z9UM zq%L>pFCOr$X*|9&3L2h;?VA9-IU*iR6FiGlJ=b~DzE5s^thxXUs4%~*zD#K&k>wZAU8 zpaa!M+Z-zjkfGK15N!&o<3=cgbZV7%ex@j^)Q9V`q^i;Fsbkbe6eHJ;dx{QbdCCs1 zdxq^WxoPsr`eiK3D0Ep}k$ank-0G&+lY!ZHDZBYEx%% z2FyE?Lb0cflLB)kDIj;G=m`^UO<4h(RWdF-DT>p{1J5J90!K!AgC0)?jxPbm$KUjg zJED+#7xQmAmr`(S%BQTV-c97As~r3zD$E;3S)@}p5udA@m6pLgRL5h-;m>LvCq?&Q zokC7Vnk-zBEaa;=Y;6(LJHS>mOJV&%0YfRdUOqbKZy~b z(905jIW0Pg;y`Yv2t+RnDvL4yGEUX*tK)JT6TWn4ik~L)fX#tAV!d8)+A)qWtSjcr z7s|f%f;*%XW!jiRvv9ayj@f&dc|1tKDc{O3BWcLGsn-OYyXRLXEOEwP4k?c`nIut0 z?4S;eO@EoynmkxHq>QpDL1q^wOQxrl))2qya?dk05^5hK? z{P6;WKHUaHw9B0dd&|xw&CYN2fVrn};Gq<=Z^QZk3e~HzzY~JrnPCs0XwMp#B<9Gm zw0?7h#4EY%O-ub6mi&O2vcpIkuM?st;RtEpKSz^Xr#3WHhpsZd!gh|_jGQ`KA30T- zKlz9vgB;pY^}Uh??nQKSzk>2&J+Qi*r3DeX4^$%2ag9^x_YckA-f9p_;8ulh(8j9~ zes{O#{v!m%n^el(VryTF-C%xfJJ$rZj)|Y|8o&))q9CEwg2;Wz&xzyHD=@T_B%b}C z=8G^*4*J4#jUJn{7-3^U(_uUp6E8+GDt#le)nya-Q4kL5ZGiFxT4bF+mX`whcif*? z>CL&Ryn3HHT^^QmWYr<}Q1_Jj7fOh}cS8r+^R#at-CnNl3!1_$96&7nR}gh}))7a0J&z-_eI))+{RCt)r8|7|sV9o01^9nv?aePxMqwPP!x|sNmnn&6{K$K*mVX9lxSAmcqAV1(hKA-=coeTb*otxTOGYXsh zW$31^q7L@<#y~SUYoNKP1JK?4|FQNQb$i8mCG@WhX9i_^;@M2f#!nq7_K*M!4lGz1 z5tfADkO7BZDLgVQ?k7C)f;$eqjHI&zgxhf}x$8^ZEwFfm-qY=+M+fbS)9r8fFE5H9 zv{WPU35cR8%z;(W%5<>y+E&v84J4^Y##N!$B++RI`CZ1i3IW9Nau=*pSxW&^Ov-F> zex=&9XYLVcm1Y?am>2VC`%gMev9$#~; zYwxYvMfeKFsd!OBB@eOb2QNHFcsfKm;&z{OVEUiYmQ}~L@>$Ms@|Ptf3jQO-=Q;1+ zFCw+p+Z3lK_FmIAYnk2V;o915cDM}%Ht5RH%w}P>Yg9{h1mZ}~R6tUII4X7i4-2i% z2Uiw3_uHR!d~5(s;p6btI@-xhAkRg9K|n#}PNT9Dw9P>z$3>30lP1(=mcQ|tpyv3@ ze1qU!69OAx4s7$8r7Y-#5I`m!BXq`f!6C(BtUlG-oq+liqMCS_D@0nSFc%y+N6_Zh zi%L3LhF3zZP{d1)L&SXxPD(fp@T@J;jZeNaf$zl>vAh7=tI z2;wS^QyRdZm~)Ur&!af;8eB8*7(F96K^=WbC$)#TWvB~Awo5AtPf8Il4snD}Xsqd< z>cH+gcg72nTg5tl>oFbwdT{BDyy1=f=4~h~L$)UX;FXa;NdSlyF{(YLrx&VDp`pQI zh3pQtC=d8i1V6yUmFon*LQsNYWen?eO-gSZ4cvYcdEd0klSxcBYw+|5AyCv6TT96h z{7Yh9`h}biU?3oBFn=d8>Hn`1Q*w6rgeX^QbC-WFwjY}Int0;qUny4WMjIee@#0%l z>YAWLVCNo1lp$>9L$Tx`t!dp?>5Pfbhc*!*wzfWkj_x`Q?`3Jc@9r8uq~dgb+lgeh zlA`eUal3e2ZnWQSSYB>qy#85^>j7!=uO-hG5*erp22NaC81#Ytioc>r?D9$b_JiC+ zSp)8KR$%}FjFNRkeE#c5vKbXNJDBoO< z)73Jt7Y|3v45efud1xkg2GO3OwYfsuBV`f6S_D>Aoh2%=`1Y$bHP>0kBvTSowX57H z&1nbbx=IT>X^ScKYL&&{LNq~^UNgR|at`D;SxTYpLvnj_F*bGgNV2tEl1k$ccA&NW zmX(LV*>Op)BOgoric(98mIU)$eUa&jM5bKlnOrHm$p^v@u;W0J)!@XWg+#X=9En(-tiw!l?65rD=zzl(+%<)bI{ZN;SRco{jO;>7 zlSY|TIxuN|d#YHx^^~>iYj2V>cC>wQwWzGVI!6#epjJ6tl_`7tDY17WMKMB@s*Jr& zXOs*@>EwQ6s>M13eZEBJ#q0|;8jao{wK4keesH9?$OSk~_3#*x`8fAzQa7fprQ6(Z zi$}B%m81y*S)RxaX;wW!5{{EDw8)IE3XDRO1Y^%TMr}c|Y>WBAKT=b*K&uMT(?JSl zO>gVtl_bKQ$??TeWr7wYO+Vbl?CTQj?JrW&td`|#@;R2Gca9jq^p`{@)KY97o3}Af zfTh{pUUWD;P7sq=I!lA6;*hq0Nq`F56T)x$K?BMOk}tptYw(%$?*otp2N6IF3#GgqM46Cda!qzvGZcMgcGV`bY5ZIfOB6^;US#WgRai zq#vS8ZqPY953|eFw<-p2Cakx|z#_{4pG}mk{EANI{PnK*CUslvS8whko=OTe13|It z>{O2p=mmanR2-n>LQHaMo}noWCmjFO@7^z~`Y{V>O`@rT{yBS=VXsb}*Pi_zDqM3? zjCZqWR}fEzAkms+Hiq8~qRAFvo}dVW{1gcZ?v&PdX?UG*yS}zT9g7nZ!F1WRH}sHA zJ4~B2Br~8?uhbaX!3g+7=3fVM)q^wEzv**rk5e34==NRCV z3G$G5B!DICFslm)c){oesa_0muLxGoq`xYVNURl*NhE#v2>y9vDz&vJwrB`Q>DhN# zY2GnY!Y^8E%PU0}haXL$8a5QN1-&7NWuC~{62j| z2ozmFyx8GpOzj?&KK1JF28;E8H_p4N^LMm9K0y}!lCxcK79eFGTtGm?7jy?t94Q@X zli|our1#|>f*68fyA0bSn=YisYSl8HB(dFN4Y$qb7p4DR0YQt=^eEMnJkgiM48$>QV6x5*^a|D|t zMPDk}u<^YEYrt|H&hy)DRk%rDIb{LTo;h7=fp^J9Lr&`{9`8_pS*tQ_$KXB$2#5{h z-&yPbN-zInq{7aYZuaItS8-2Mb4OQe2jD*&)0~898E|HlAq`o!M&It@vvnj z_y@))>~_oR%S8OfmFTGYIat^#8_YKMqWLac<^}RZFDcJqvSJa>&6HaLS7p-$)QyL= zHrO|t75`d41Bp37RZtKR%g^%o@9C5Ce=CjuvVQ-KI#Uw2WWa>cho;jztUt~Le*_pT zkfA2iif9QFp;vhd)|A?tdAQ?9o~?EqgL;=)eKFQ{E^u?OIP}fl^5A;$^ZVutCIqj5 z&*i+G?!Px|5~~6zTYf>~uw*kM`5p&Hju&#w!7^An3*mQwTK22wC7p^OsvMjWf`$MY zLX|ZFV#+>Uq2!QyRD9cgbI9nswteMAMWtK(_=d%r?TLrx?_rkjbjI(rbK#T9Gn}J| z5ajow3ZErpw+%}YfVL-q^{r~##xJ^_ux2yO1!LJZXg)>F70STV=&Ruwp&XP^_?$h0 zn>$a?!>N+Kt$UXzg`e+szB}*uw)Z$uL6?>*!0IrE)SgV~#a?Qgg7HuTsu3ncrcs|l z=sQSMtr}S!sQ4SriKg=M`1Y|bC`XJ+J(YT)op!Q);kj0_e)YNVNw8SI|1f%9%X?i5>$lLE(Wfc$wY?(O985d5e*)UPtF!7gG3(Kd z-^=-%-wWCEK`r4oFh^{|;Ci%W^P>K%9dBNDqi%c$Q{iY#(zbwN7~pQI=SHd%WuV7Z zO?0P;Zc6yeN;)IbJIP0=>W)EgE!76jM^?IyQ*D(T})1NGmP z~YAb6T^#R6;)Ls;cV~LWk z33lcLpbSjxStw9Z>Nv&+rPOXxCGB=?ttZs?{OF7;GYlV&w7-82POb$XrogqFpLA2`j&MLZXr=IG>PAFSb2np~x;E_kV{ zsDwbK$?iYRn7$;mHYZhQn6P2#_hXAHd?;q~!Zy}%;@%wT3u|Sa-!WxxOE_fwyFv*Db@>X;Rl+fK1oP?55*dN0#2%SuikZ)y7Kx>`8*9d?}5 zKvXF7J5&Ey6{A8qUFxrFOh<$xdSWV^dw7z|`7RVZJhAwO72V zRrM_3*wI`^ycl7~>6KaCYBr#WGR>}B)Q(V%&$MhVrU>u~ql zjGeZF&>=_ld$oY!V}5}Gb> z*iP38KOav9RHY)0uITwgz99w- zJX-0BGCdY*$c7pi@>@-`2>#>}c(DHaI62ntpKz z`c01Z#u7WuMZ71!jl7hv5|o61+uv5nG?*dffEL~328P5HlKh2&RQ;9X@f>c1x<>v= zZWNSz3Ii~oyAsKCmbd}|$2%ZN&3gc9>(NV=Z4Fnz2F@)PPbx1wwVMsUn=-G=cqE3# zjY{G4OI~2o$|*iuswTg1=hcZK$C=0^rOt-aOwXuxU=*uT?yF00)6sE}ZAZyy*$ZTH zk!P*xILX#5RygHy{k?2((&pRQv9_Ew+wZ>KPho_o1-{~I*s1h8 zBse@ONdkk-8EG?r5qof}lwTxdmmEN|%qw(STW|PFsw1LD!h_Vjo;C4?@h|da4Y;*; zvApQ=T&=jWU39Uz=_yN@Bn0{{)yn8RZ2&X!<*KBv-7tcWdkF1Ij8D0mU zwbcs}0vDaLGd@xx%S_QZ1H)GTt`~>+#z}HXJTl9S!sd9seVJc|_wUMSdD$>k`K_RG zlq(fsnR@KM^;C}}&vG2t+}_nGPuI5ovg$6TYeMPIREGxP@2r~RKd@>gV`mq0XENsh z%IRZ-ZNP+4#J`o-yRpP;w@;CrSr3wiix3e9Qc|s(WapRq950P->g|JYC$A)$YrGeH zz5dKlAHAPJ>%?llqqB&#+#VU3sp=9>Xms1J;tSYN>LMwNtU68yr!})K4X>%^IrIDp z>SHy&6fJHybwS^BW>okFeaQp6wxaVP`hy;ZX#e+=w3c?PGD&_LmeqL8oZ*YaM1+#S z5WNAKo4+99JW(+qcMjh;+c%R#R?t;(aQ`2`C=bo((ERzgAwKKazXy*0wHN;v;P|f> zBW&?`h#_I^?Bc5GX7XP@|MOiw%&-#?EQ|w+FdCl_&qPN&s$|Z17UCF9oXS#N z)px6>zm&}0osTnCGI;AXsj`q=LpIsW4x}q~70uey5N_NpdJ*Gv^@$g@f2{EB>LP7Y zE5P`jZh1vHNgk7LfMT({jLCjRZa4ubW;UA#%<@Zj?efrPdm{W3J5UEFgm`YkVqz;AMFetZuM5uQpvORb1GDX`WZGwTrF z46+&sAri5QXCfGYpdgonWR5`>ZEa;?jrKvfNvXF<&l)1uU-3q#4X16R2~?P0yg3H` zfw82QWZo^cac+%(g^_6`+2>~Fvy{pOCGnj86+=-!N`GPWAjus1ejhn6f4|mDkU6EE z&u~;xfdRMkj=h;4d~~+4(>L8weT3cz9e@E11EH!tX<IC!@kS+dsIQA`HQ2vdoS zzSD0U?mb1M0@qXu{yhZk2Y6}2B-AvvYg|tRr6z*_*2l*VLiR6G;M{O^Znq~LI%=I_ zCEU{htx&Bo+69G`p|A@R>KlY1*;;!{aWq?Pc0Cu!mT-0S`!>3<@s%Ri;utYNQ+CXDj+LC5<*$4*$-mogGg^S~3JRv{ry zPJzKJg!XKb>P}yJVc^1V@T&MV{z;@DLhvV{dG?RogCcPkROivliSr58>5Zw&&A2?n z9`JOLU;eQGaOr6GB(u{t3!+$NaLge$x#M&*sg!J;m~rRc)Ij5|?KX_4WiM-eE%t8e zqUM7eZ~ZonavR;K4g2t$4Fj=UVyEHM7LPb%8#0?Ks{~?!qhx9)2^>rg8{0npLtFKR zJB)19TFiD^T7IUXA8wt!@n5gj&@OK~EO}MR6^qd?^-?%-0~b2K9RWh+_mSEQQWsLCFOt#JlAQMgNxvv-m z;sF*r;WZ*Wi@I|6pMN+|_rLYKlWwvpKZY9rA;fo8l8hFQGI?4#kt1-r4UL;nPF@{~ z2T~a@2>yD|GuU55boxoIIe_BFo2Vq&rs&2itv|B>OC*bIeOqMBRw~y5KRMwiVHc)` zIBdliiY?Ai7*+k#NZf3MW5!hya~RZ6r7k)b?HF0e(n`ZX=iCpT7St`FDwL@SGgKlq zNnnU*3IcnYDzJg{7V$cb`xeb4(s(({&%f69XMTw-JQErS%?X_}?&y&tvHw@>1v{#R z4J@(=el^kRI+jGa;4)l#v%-jM^$~0ulxh6-{w*4Lsa>Tuc z>ElR3uM~GUChI)c{TW${73A3$vs<&iH;e?4HjW2MvSz9tp9@69+`_@x{Qte^eFo5IlAi&zw$=t6u8K%8JtjRI88PFNM7R>DaCO3rgngmk zI-RMOyt@kr-gVra=tl^@J#tI7M$dird(?aU!`&1xcm~2;dHN(RCxh4H((f|orQ!BS zu;(3Vn+^doXaqlhnjBJj-)w?5{;EEZTMx+?G>Rp4U^g<_yw_blAkdbj=5YrNhZB9@ zNmW=-!yFx5?5aF^+6*1XI|s3lIn_eyh`uv%?liNzSC#z&z^R(mqEYL@TdWzgkf>g1 zedzs*={eJavn{8vF%4nf@et<@wkOPR>NiVuYtESbFXQ;sDz_;|ITVeoW|me5>jN5P z5--{13JT{3ktkAf9M;Jty)yectg#{+9sK{C;2CvPU81tB3{8S5>hK{EXdVe?fR?sd8m`V zPM*$)g$HKp0~9Xf6#z!YJ&g!%VkCMxkt>ofE!62?#-&%|95^)JJ9 zk;GlJdoH0HwtDF(_aTv}mt$?EyRyE6@pm5DG~Gj-2%3HcZT13e)$)z99bdK_WCx|Q zQNza(R)Z>ZKTn8oIdcw%c^pFaMpFZ4HOds!BODgSBWJJYW3I_WJvoEm4xsfs%#LZ6 zdPCk{5XJ>2f7Hj-i*9lTW6BKCIuy)3L!b3(uPoSgW1WA+OEYYBRgSsJq7wjHh%c8ymMs3FU%~cprqL*084p*^T3{J%Gwq`jB30n(&y6- zII8-_r-s5&CVtsoNZ9%On?7yn;oZG03-$wx^uRk9>b*ufh15|HHk|%=MA^ioyb9CYU$7y$4R|M5HvpiCTxKSU`LUg$+ zB3IBl&{qO}agqF~BFM6&11wMeR-#Rkuh_(^j+P4{;X_w|siva$5P`dykyhfAUD%e8 z+{G0|7(Q`_U91sMKFO^rHoCWfXi0$^ev)-187G}klYv@+Rf%uZ&T4-Uhh=)pcU6O1 znXc^c5)!$X+39|4`yNHuCj0wkm+K1VN0G3_EL?-ZH$p5Y*v6ec4MV zS~1~}ZUhl&i^4`Fa|zyH4I%rXp;D6{&@*^TPEX2;4aI$}H@*ROEyFfe^RZI%;T>X> z>WVSUmx@2gGBxkV&nfyPK=JI$HxRKUv(-*xA_C;lDxT|PgX*&YYdkrd5-*3E1OSXBs>35DLsHHp%zm+n0N(Yu{lMo>_t&d1Xy zfCxl=(CNNx>ze+7w)60mp>(M``Qn$aUrVb$cJAb6=Do7VgW`Qn2;v5{9tB)jP$_mB zn{Hb_sMs4yxK|!`PI7+zO68}{Iv)dpu!+ZZl)xuoVU(oFsm<3gT{j2c*ORl|Lt+?dR^M?0 znW6rNA)cR*ci;z?BaG(f(XynY_y+kTjj~T$9{N{>ITQ4-DmZ6{cOkoea9*LpYL{Apo0hSpLqJu z9`tjP&ei;%pn9QY>-$9=<73M#X;qGb+%Bt0x>=u`eDtthI+LWB9CdAO=ulZo9&Ohs2X8GW>b7#&U|py28KTvPBl#Nqv^{AgkVXrOyS z@%3)}$I&mJOYWoG$BBb)Kb~0ptDmBxHNH^i6B8FA7NR2HfTnjP?eDnoY4NS_aYg4P zGGPw11sAf^^fTkY#j@T#6Ll*^GVaPo-1;aS6_a}{r{tWZilzse2m zc?LS=B|EWxCD|!O%|%t3C@Rd7=rKJRsteAWRoDu|*Kx-QwYZQeYpGrZ_1J%mFM;*S*u=0 z%1OC9>kmCGqBBu#-1jVPRVW*BTv%3uPI8fO?JOZD#P_W^V+K7&KVB>hzZ@PdY*%Ezo;}|5Mk`Mo2m*_K%no*jDJGp(s9j;&U`Z>z zO#SEe)k!p$VE-j2xDoX$!;Up5%8x$c`GH$l+gTA*YQaE0jwCOA<*__2NkV){z_u2=4NQ zSk$(oj$%ygio?3V8T3IyGMYvPs`t{im2IoHs7or+>>MYvG%Q?PwOLqe%73uGh6Wn; zo>e7qI$9?%cVVkvQLOLKcU5n*`~qn8pzkdu=Z4#2VnhUy>S*;kT=NqA!dQtnE?wVg zOKobxJ|QCjk`!(2*~5NQx{{=Lr=)ndyn{V|&PxUa=xQXVU?#M24F8H%C*uvs(#Va0 zSkp}0EFYq0#9xp&$O?gIInc#^^_6Ol88W%)S5A@HeE0(SR&!Yl>u=*5JEoUViDR@2 zJBjTsp=Y44W`Nb2+*CcZCkwP(QChX1s)b09DEIZCKt1$q2~;&DJ9!{bQ1Y6&T_9u1 zZM8^im8Wf#FUO6tZqc7#`z0cN_JA>#U_b7he%?cCnlV2&47y5Fc)Z7bp5xGe1zNq9 zl1VaV-tsm3fY=oIX^SPl!P;9$o?**0brq#ShM~3CXhh^SK0oOKB9O>;q3G@ z&4&h$mLSgohc^5IC|H>IGfZvVQFUT>T$|U7{znY`56<5d)07oiv*2R0+-BGPPkWJ! zIOzKF+<5o2YLWP|SGCx8w@<>u6K1o`++xJ+6kaJrt<&0Haq zyUccgxI$sR07Vo9-pF);heBva;?&NcAzC*gSSG9B3c?A;IH9J zl$j%F4*8;F0;H2Cjo*kWz4{kSh?nX}23&&KL+U(#nOAuR`wn@uwUNkWEgb*ZShKPy z`aXTJT4f*Um4`iv2KOfzf-~`#pOfH8>is*xnLBDTyx2Xuc8Y2Od6z((P2AZK@b_96 z#0V6jdw>sEDJ#uNGV|EshD1g&bYZCzCZTZ)286HLHc8Eyy_HPi;d#%;Wx}d6tUUxq z_VB$+898z_{9-A<*v6VI7?(dC04o!8$>DQ$OdbrA_@<6auiBNp{Dw$Hs@@gcybIQT zAU7Pc5YEX&&9IZ~iDo&V`&8K$-4o$)g?wF8xdv1I8-n}1bc7tviIBqt z#iIl1Hn;W?>2&#bU#VZ1wxq(7z=Q15#0yoz)#|r`KSPKI-{aN%l61^?B4RMDt?Vk` z)G#K6vUN?C!t{Q<@O4$0(qI>$U@@TI2FVF;AhSSb5}LtXx&=k&8%MWM3wv;Xq0p~W z#ZX;QFv5G9-i6=+d;R7Dwi)ciIZ1_V!aw;K^etau+g0fOA2HXpV#LQZGzf?h#@}(o z|3w!sZ|&mp$;tmDiO=zef5C|Alz+@@4u5#yZ7yNpP=&`432%a{K#{;nsS!jwk-$Qs zZRty}+N`Y~)c8|$&ra{bOQWM2K7qa}4Y{ndK%dKp&{ zFCvX{PAy_C{xzS_-`0>JlPP7&5!5 zBQ$NQz^z#2y-VeIxnfY|RzU`w+1t6vwQ|wM)LlpuaUzYehGII;>2DYyR|~wC@l97s zgX=f*1qtfDyco%BHmN+o<2qoi`D67R+RM$$NN5-moE4kx3MCFfuip*45nComOZKQf z3!(8tkSdhY5+A%@Y=eVEZkXU3S6B2V-R$ZuRIXWhsrJg3g)p4vXY@RV60bKuG zT6T!enE<;(A{*HPQhae*(@_!maV~AWD4EOwq10tkCXq+HPoe_Pu?d4Kg=2ypcs?&f zLa>mEmPF4ucJ%i~fEsNIa{QmQU27%Abh|w(`q)s~He5$5WYQ_wNJX6Qop<=7;I1jd zNZak`}0lVm+^O!i;|Lwo}ofXuJ)*UtH4xaPm*R7?YS*<&D__=@Kki>{f_Z-XqM;Tj195+~@d;rx zh5pj8oMuupWa#E(%85**I~1Zat-Sa^_R11-CiKdd`8m(DGuzOm9lX$Dd!DX!_Al}d zS!-|}dWG80S;`jSKDH%Uv;-OJNeBI0Bp$z->{_>1KU%h&Af7nns(L=xRN1 zLvOP=*UWIr)_5G2+fCsUV7mV|D>-~_VnvZ3_>=9 z_bL6`eK%W*9eJ34&Puz^@^ZIyoF@%DTun#OOEdUEn8>N9q(}?5*?`o?!_<(i%yc`k zf!xXD6SQscHgPgiHt>x6{n{+}%azrfV4VHi#umyi0;11c816`E??2`$;Rc`)qA2H( z5L|{o=ut7Te=^~@cR0_#cah0?w0Me$&>}ga8xxy=?DDl#}S~Y z4o2n`%IyGjQEP%8qS|v(kFK&RCJbF1gsRVJ>ceSjU`LuYJu%C>SRV#l`)ShD&KKzv ztD<9l0lcW0UQ8xjv|1NXRrCZhZh3JFX_BNT@V|u9$o~8M=cjOX|5iBS|9PAGPvQLc z6sA~BTM(~!c&V=5<}ZIx}O7A;|&bd7vR_y)t+ z?Vm7kb^gJ88g;!fRfMTSvKaPozQz4WcYD8l#0WxQ${P%0A$pwhjXzyA0ZzErH{1@M z22-6b1SQ!SMNyqj_7MXE2cwcEm)W)YwB)ji`3Y^5ABx--A11WB3mBQB<7K!~``j&@ z8PKJ^KSa>#M(rar$h}aBFuNI9sB5uAquDlzKW+hYB&WKf9i&+q$j5P;sz2u$f`uHS zaX8$!@N2b81<<0w<{CpXzQGqSZRpfVb3R%bjsw-Kl}2UH>}1M?MLA#ojYaagiYL!P z$_@7yOl~PbidzJ8yx{Jz9&4NS99(R5R&lf~X_{xjXj|tuvPgvzbyC}#ABy^+H+FN0 z8p5U!{kxOvdv3fr35|Kb`J(eXzo*GvF6`_5GI)&6EW}&OGp=!8n`W0mr_o~Xq-t?% z_pDDfIW#L^DmX?q#mA%Jz-f86KG`^7V|1zdA#4#<=}91g$#@J`gOqMu+7H&yMdNIt zp02(*8z*i{Zu;#S#uP#q!6oNjQzC|?>fgzorE(d+S#iv4$if+$-4$8&eo zuSZJ1>R2HJ^3T9dr{tn+#JMGv#x@&C$EZapW9)uhp0`rDsISKrv`~3j)08JZlP&}HwA!z^~-?Ma(x0_AS{@r z8!(Z}5d8+5f7`r3pw_a=Z`!0r6r4%OAGYBoq3T7^xI@9xG3prNo>`}k>@VAQk>(=DIy(szD&6@u?YVdC|pJLT@lx{=IZ; zIkO4)YWp*Dpp$`H$Ok#yf;yBmHvTb@)4j)jVNF-O?$nD25z7)I!cWQ|Yt zeS<_C{i|BS4HICD=}T(|)@vd(v!?P4t4>APo7`K5RJvcTpr_KgWeB~zMLknrKMgpx zyN-EI%es5e)FNho=}qGu$`98v(QDPUMUGrY4tq>?x$md>qgNO0@aAQLMLr8XD8z%; z2Osn1D>N^22w4Xb8{~fi^i~SthAo7%ZjNb)ikgj0_AsXqF_0+W6E_doOUi0uV6Lvg z98Xk#>IK|-YHx!XV64==b(nYKMEyqPF?D)yxE=~;LS?LI_0)|1!T3ZtLa?(qd|YlXdI-e$W z(3J*FbOe3cSXvDaTHU^Hqpf2i8aH+ZzqY$cFFIH;fxMtW^(AmiMkBtb9esujw?rte zoo&0%Afb~VBn6A1@R1!OFJ0)6)Fn72x{}7n z+b#5gMommvlyz7c@XE`{ zXj(%~zhQne`$UZ5#&JH0g={XdiEKUyUZwIMH1rZTl%r@(dsvBg5PwEk^<+f_Yd~a@ z%+u%0@?lPzTD>!bR(}RQoc>?JwI|dTEmoL`T?7B zYl^`d{9)rW)|4&_Uc3J=RW25@?ygT$C4l-nsr+B0>HjK~{|+nFYWkm77qP!iX}31a z^$Mj&DlEuh+s(y*%1DHpDT`(sv4|FUgw5IwR_k{lz0o=zIzuCNz|(LMNJwongUHy#|&`T5_TnHLo4d+5bE zo*yU%b=5~wR@CN3YB0To^mV?3SuD~%_?Q{LQ+U){I8r*?&}iWNtji=w&GuF9t~=Q2 z$1cFAw1BTAh23~s$Ht$w!S2!8I;ONwQnAJ;-P4$qOx-7&)dWgIoy-8{>qC8LE?LhJ zR-L4qCha@z*X+j|V<+C(v)-UZmK0CYB?5`xkI)g2KgKl-q&7(tjcrhp5ZaBma4wAd zn`{j>KNPG>Q$xr7zxX}iRo=M#@?>}?F`Sv+j6>G9tN!g@14LUf(YfA4e=z+4f zNpL4g?eJK`S${tcfA{wbn({8i+$wMaLhSJo`-Yp@G2i0Yq~@wdyFxoVH$w9{5Ql2t zFdKG?0$ zV7nmYC@PSsDhnELrvd8}+T=C6ZcR?`uapdWLc2eaww5vKtjQQgbvEr^)ga?IF;@1(?PAE8Xx5`Ej&qg|)5L}yQA1<^}Y zp7WZpk%}L9gMMyB^(mFrl&2Ng$@#Ox3@Z6r%eJ`sGDQbT0a9ruO`T|71C;oCFwTVT zaTnu)eVKURM`1QuvrBhj;1e>1TEZW54sKUfx0Z=N*;Jpdh~Aj-3WB zR|EYVGDxSvnjeA?xxGF41Wj?~loVahklw|zJ=v3pOEVZFJG^TvR z-tJN5m;wZp!E7=z;5J*Oaq%2bc|Jw!{|O+*sja+B(0D2_X`c2)nVkzP1S~LOj~xs!@>aN z3$K2^pW}@R-70K!X&s4DHHoV&BmGWTG4vi9P1H$JxmD|t_V{GlHZv(`yJ234IVuSr z~!;~#ublS8qdL8SJG@XRCwWhkZyg_EKH(sB2}QQSv4W}|CT0ntD_4Eyp519d1%yKvc33|`yW9QzeJ4*XLP7@l=td+bwxSL~jCf-ny)IDC^~u5s)E-y^FdtU?)hkN{82Y{Lo)bCWcBOx;Jbw;)Pg9bWQQTY-3RWehpok!>D>Sa2EcEOS@ua)#G3I+GxL_ra^92Y!}tMX zwAp*Fv-aAarn`ME7N#Uyim%ynre6u?KS15L#$#rKZSgLnXx;g8TP9suMpO055p278 z%o-6eT(3gdIVFN}Gb3k$zbTyrHYel1x6OxETsk&h0E?&}KUA4>2mi0len7~*;{Io~ znf+tX?|;&u^`Bk-KYtx6Rb6!y7F)kP<5OGX(;)+Re0Y;asCLP;3yO#p>BRy*>lC$}LiEEUGJHB!a=&3CddUu?Qw>{{zm)83wYRy%i}UV2s| z9e>ZXHzuMV#R1yJZato0-F|Jl_w2sUjAw@FzM=DxH}vM>dlB&bQ!>51aGc}&WAH`b z6M6iG$AyJIAJ7-c0+(;pf=2=!B=%yoM1i9r==Q+}CK3uW%##U1rP~mwjUb8PLsi8Q zq!aTLLYK4HQ$vN1sU;d3XW{oFA{u@1$tduWmdOqc(~AqWq+`V)G&?YOOwAK20x>{q zOgII2&A_FXPzVtgrD80Y5J+_SEmyUcdM2N%q);|ZF_m z)6PBcOcAAy3kN*`8ac%zPH3^61_zn6_2FT#NCOWYx>ezqZzCC;tzM%pJC^gFAFcTs ze6C3WE-a*=nt8tErPG9zfPRn$QHqB7aHe8x3w&rWT(0F54<2uBJDYtbB}y|@9V6T( zmM!t}T5SuwxyTCma14&l|yiQRw5Pn|OiDBkx z?4tUGrIVsC9zs=F{W>zl9XeknEc+~Mz7zCnefUPUF8iF?A)QJK8=84#-TLLxq?BTM z=VYjYW%TOhrBp>3D@K{vStlEUt%e{HRc=766AQ+s7V_F|1A!)P3?y*=gUgbZO;O39 zX*BC((-XbnoaRGxxhRQRVKCDG9|qC6?7TwCz{A{OZp$Wu(~0DFo(w^P3f>4gr8@P^ zl8`!vA=_fvwTZc%-Z42}m>Q;KQ~&v;ipZzbA2;}Peg*v}TlKRmU%4WNN<%qb!cLo= zoSx;XBrv4}ErykT!)z)Qar4o?(q6!mpWLNFe~Nz0S@yI{1)Lxt<0K=Q$~>*HH+Wbp zQ~fx0aup_lZb|e6*@IJOJjw~Ypiwdq69&Y2vthfGq6u1!Joy%;v;~4`B@B*S(}}i- zmZc^*aHOK(dd(geOKg)P+J4+*eThk;P@wRjvm}e)h|#EpsV9YoqqRW{)ABhRlvGA* zL$&k5w*_-X1ITCwXiH=)=5lzjxY5tQJTBrv<{dM7$98pdK%i;RGZtiJKaSGCji7w)aNrHu_9_IPGHS-mMN5AheTn_ia^YdunCzcp2ap8eI-RQEm zj(q7_CT)o|w_noPm@MVqIjv%H4Bdo6*9*!Zj)bLx!p9POp(`$dj1QW`V=;=|`Gx8QST=OnK5jlJX3!KBz>v7j$&5b5YrhIArRVL)1C^o{@DJ}*mk*s=< zDK{e2f%fG)mK_Mz*x@#ahOO)cQQ#VH+8Wef>NKWcu4J>PIc3iz8y6PwCmY|UQ(O3!B;HtsE&jvyv^XjL7Env5#i zH4-k5GzPr-%36#%+Hvw1*UiOIk3b7F^|1dPi!-i7C^ZWp~_KI%D!sGYb@@zXa?*{XfjZ~%Y^mT!kaK_>K8 z_jL78^ zS0eRdqZ0v~WWow1CE;vDBh#{w9R4JgB!})W9N{{D=p-RMnehZ#pH*ABzDP46ryZkt z4ek|LHS{CDhTTMQa3a5fO9OLg?y$+#Gi2}Fv>QD-+ZEQKX2Fv{jr~miXz1ZpPcXvJ zNvQT@kQbBz_Y4Kg)*`E2t;tPh5_7tSGvL-|-A`lgHX3uVG4jLev9>YCZUeNNzioL? z;OBD{z+=Gs3+*ph)#bO#7IHl|rOFfvpK%cF>W??Q!Nh&B@hByD&}g|>a?GJ4uhX3g zPJXKKAh&zWv&wITO66G{PuGLsxpWSqaadFsv>_vQt?LVslVob7wylsa+O`IYWySoO z$tw#v7=&7ZGZqS}N!c##5-bC%>ze*s0H9J%d|!JgE#uZ|k1_bAn*x(Y%r{c=(HLwNkPZOUT#@j4{YfG#@=49YJ{?7? zddbK}G-@Dod&^Vf`GOo)G|`n@kq?Z=o84x{889+?F*dQz(kr@9lQ-TXhGN`)^-Li1 zb}xO2W(FvB2)EA;%qAkHbDd&#h`iW06N1LYz%)9;A&A25joc!4x+4%D@w1R+doLs= z#@(A@oWJq?1*oT>$+4=V=UnuMvEk;IcEnp4kcC<_>x=Hw9~h+03Og7#DK(3y3ohIp z-gQ$-RQIJTx%0o@PDST|NW41VgAR?CH`Sj-OTS0)?Y*M_wo|92;Oz)aya`^I0@?S{ z<%^epAw!Tw(bvSmU_k~Im^%#|0`Xkcmxj;31jX2Gg?PbzdXp9Dg~P)PW+Xi%iWiCr zV-Vv9IR5guDS2lGV!lfTWxkD8w%yz=UB`2j2Zb0eg~arRA*Q6>`q=8#4&OC|L6O}8 z)!w(idG0yk-BF#~k@Avk>an9z_ibOP*Rb;db_PsakNWYdNoygT?yRG=+5>ud<6Vxhk?P9rk!+8?xMg!x5kD*f2XOd^`O3U zlO;ImEy0SYI_J05cMW{dk@%d@iZFCNhIVtOm8$viM>=zM+EKJG%c0)dZ0D$4*-psQ zW+Fq|WmbYkBh5|^-l$w-`Uy8#T#<+3=}z!(6RadEpFlr1f6OFuQ5sG735YicWaoYR z`wuEZT2dntHGC7G*Kzk$tsm?Fd25LTHJj?Zo2RH;9rW9WY1`;@t_O3NC};dayX;Ib zgq6afb4!50qL-o5%yzgcR-1Xm-l4SE!rE>o!L=E`Jeug(IoZ36piq6d)aek0AV)EJ zaha2uBM!>RkZHRN0#w07A=yf4(DBmy(IN6NdGe$?(7h?5H)*?(Li#GjB!M{nq@C3# z^y{4CK_XQKuO>(88PRb&&8LbRDW1Ib>gl6qu(7g}zSkf<8=nFPXE1~pvmOT3pn^sa z+6oK0Bn$TBMWYTmhJzk_6)$>>W)nF^N$ld9 z8f^Y^MLVz@5b}F0fZID^9%hRL#()Xw*%yhs&~|PK|MGI8zuO!f!FqbmX9icd zXU(JOCwac|Z|=Yr(>Q3)HsXl!^$8VSzsgI#)D2XkpZ2=WOBcFF!2&d;*nF%h0I!`mRHl$91jYzqtLfNHUoYzrMzjR)u zP_|Hti4^){G?Ge6L_T^zVdS@KHwtq^+*+aBNl=hVc6#KB-It()qb&8LhnVW9Yxn&S z&^s^u1OzB(d_ByXz=xm4cpJzNzV+Txh`~H(176n4RGlY6( zg?ed(a!J?4(oL}@UfBpgPL*)KrGtM_hMIdu!RywK@d!b-{YAY?(?w3yB@Fi3g|G)| zho%)<=%Q$Lo7S-BxEjTL;M74{y+`Q^Xg#j}VvF|Y>X7s+Ps~aqT--tJNd9U6;Ej&o zj@|!`{Xy90t_Zdb>+m8tCFJ@X(Y$mR>%)gv4Vt;oGr`idhQ7H1^L3v4<_2}-UoguorcscRfdgumUVa0mK7-Wm~#vbrnX9ro}@82q=9t;lM9nH<} zLL#=1L7*f+mQWfyFnETMi*fe8AI+gdY6BM7CkRS&i4$ZRv$v*=*`oo>TjZ84sYD&T zI!DgZ4ueeJKvjBAmHNu|A?R2>?p{kQCRy zRnGg@C%oB#-;H-o-n##G`wcPWhTviRCjB{?mR20|wE9Kn3m6(%Sf_oNXWP^b;dz7( zb{blETKwpl`AT#W7E6T|0*bl?%r{}-BYdwrn0zN(DZXM1~53hGjjP9xzr$p z>ZH?35!~7LHiD7yo7-zzH18eTSAZjW>7-q5TYzDvJ$$S$Z@q)h)ZnY(3YBl+_ZK~* zd6T1UEKdrzmv2xc>eFj2^eQPu;gqBdB@TLqWgPk|#WAS0c@!t08Ph)b>F3 zGP}9_Pfp;kelV05nUfnb%*Oa{h;3Yi^B5xyDM~1r@o%v#RYi-%EYfSYY&02eW#bGb zu8(H8i9zhyn%?kx5Txx^6 z2i}CK(HeQ_R2_u?PFp#6CK zjr}k8Cx#C?DFgP`uN<;}x*Gd$-JgG3J_i3s>fk@_Po}b|JNz=Dm+<{^51m=mO;n4B&azYm{>+VhB{iyxuW+j>w@>VHcJyoSBQi=hu0;p zPw3Aj?%Ai^UeD{ySPIqsf|v0L&f_fmE7oh(s|jwbkK5^AQ9F|;a5V}EdSE?fyxdgf zHTq!f0;+-V{0oF+l_~>rMGk?f~m^wDXlxqt1@+)6Zv?BNR$+%$i z*NF93f}~4d9H2C7@?IibyqUtLL!XZW2ap4fkkxMqDZuZ>`+AfWJQ%~O2WR}NoA=OP zieg@q!mP z?=qU=EE6L0_UpzXt0qwX2tF~}c|;`#MUY2TMz6k({hpkiSz>Dxt*4-PtkAdAA*0hn zk~CK6#V=*^m5 zg$tB6rSO-=9l>GAl^DjJBHdk0wD0(L!OrcZ?qmtYbl+}s(@rtE-O=RTx*1cZq~u~5 zQPVt(IB=*?Pm;Le%#i1SFxHY|>=Y$^RF-FGAUSkBpn`|+p!4RHyv-Q(XgZ5Xg5W}J z8RcT?+4FdVQ>z~9kP5By8eM95f_LDnsnA%K;i6`OpcuJS=^n|6nH-B2EhH=dLbO@Z zuw=Ug>7gsu33`Pzy3Lji0x8OCH={?VRqFEi;@oDIS<*?dG@9X1*tlYCm4YUIMhyfo zJ~=K@-X$D z<-4dH<-5o#yMj%f@U{nfWYVdrREJ}_o4&|c*_+M6gk z-Up9-i~jM-bwR;Bf0&C5wteli>r7ZjGi+mHk3aC4mS5 zPC^{w+G%menlWun+&<#i&DJ41thvk;OKZEB`S%sZ6 zzYpO2x_Ce@fa0LuIeC=7gRHN#os!MQ7h}m9k3@u68K2$&;_mSe2`>uvV<`RgC)TKX z`J}&Kb%*f{Oznj$%-QafB}Zb$Pi%@D&^ZTcgJ0+Bk6-iOJ-P|Q10)5ie2u0JzKb2r z2C@{f?ZBcPw5%h&aKG+6%Qvhw(t1Y{hZ82YE4(Tlk`2VCgE&1x;AUt+5U*$%>P|iWLeb_PJL!VX=b4#>#QM;TGjFHBNRy+d{v>2cVXFyqaLd300 zFHWrc8lB1KSOH3dkJClJ%A5oE^31WrQZ3^-3`Zk?1GqoV7Wr62=V9C=(;#R zhzXAT03)d z9OdZ|;CjSnqQeqF-CUNR=x9x76JYnpr|T+6u#$y=7cMVG72k4f*BJIG>l1NNvyv6NQzr4U`r;= z&%W1Ri2sI5p|8%q5~zM-AMptHj_eX7FzJN7t(%+2dA)efyFbePBsClxY_yMqWbEdT z+jm?SZgH3mCzU?e^psnyd8UK zfZ$^_^}C1WYB1-$m4qwT@#=wsAq$9Xj=%IRvc#V?1azEi|RSc;M zQn;3%Gjk3D)R+3`gZplB>Pt;g?#EiwRzxON;% z#P5IK*YAh1Md<$o21R}j^8Y#t#`fP`nErnb@&CkI{`XNXulcVIXwLcS%VE4i4-!8a zpj-q)#TqXkFg&z4G9pG45A-$B_Lfacr)H85ge*yqTLAb(oY1$6Xu7Rc%^aVOmzsKd z=WEXA40~hm@7FKD9t14nSRt)m0XWkP1YbAE009nIupf`md=v&J;C}estaY0%^Z;;lf>5AF-y%Xf1QEK(}4n+ zhKsTx^bQSpwM=UWd3WRcpEQfw>P%zuhLeEdY}s%cGitMZa14Ui*Mzm%=(7<#b2gHmJ?kdeymT7H+Z8k8tgd zp-dhC)R!P!)w(n%RgOi%^)LGZX)yxC%@f@d4x@IRbq{elrCHyIuphEE6qd6l6O`;B zi0WQg;j`hcu51uYTBSSYNvY{Lkn$iu=Ae0g6o1cSTRwXmEvNcNI zv;)Z_?g>?aG`Zp}*gY8%LGI}{>J#`x;v=*ykuY@z2Erz>@b*)tMp2>=C20MI8|{Z2 z9hbyDJ7d#MdWK&fyZB>Jdm!#x_uRw%>`OuM!&QMim}baa76{L|VAuq%1UpXVHsClm zPD4}hjj{lj`)aaD;x|PJ9v@?8gZ!t5hER6!b~HJ_l9P|(h&R6js3mAfrC|c+fcH^1 zPF*w*_~+k%_~6|eE;-x}zc%qi-D-UpTcAg|5@FCEbYw6FhECLo+mVn^>@s-RqkhuDbDmM~lo<4sa`|9|$AltN_;g>$|B}Qs zpWVSnKNq69{}?|I`EOT~owb>vzQg|?@OEL`xKtkxLeMnWZ@ejqjJ%orYIs!jq3 zTfqdNelN8sLy2|MAkv`bxx`RN?4Dq{EIvjMbjI57d*`pO?Ns{7jxNsbUp=rF$GCut z7#7Dm#Gvh}E8~2Tyhj2reA%=ji|G6yr%@QV{(90cE{JYOW$0F|2MO+TM^`cAu$B7s zmBV^{IqUIbw5~muv}st`dDdIxSU@Eb>xf3$qwEcg;H+vp1^ArN@A)RtQ4hrid2B{9 zb~pG8?SC3#xctpJXWRGXt=cx6Cw!IqoJrK)kuLL&`UYYB{R6Dw)k9nKy>R#q_X|V* z%zVsST$=d(HozVBc|=9<175^~M$v$hL9azT^)TL7BIA#qt>N2^iWvMQgt;!YZt~cv zn!x^OB!3mOVj>^^{mloGiJhLI4qy3Vt-148>9j~d8coH)q|Cg5P89Xj>>hjtzq5iT z%go41Nhi}x7ZztTWj|deVpj>Oc#IrI{NxIm;qhnuNlvNZ0}d=DVa}=H0}Vi-I+wKK z*1uD=0_)b-!9S^5#(%_>3jcS-mv^;yFtq$1)!wGk2QP%=EbpoW++nvbFgbun1Eqri z<%yp)iPo|>^$*IHm@*O74Jve%nSmDeNGrZ&)N9 z)1rSz4ib+_{4ss2rSXRiDy zgh(descvk^&W|y)Oj#V@#)C658!**J#=ckpxGniX#zs0tA~NG>E#Hn3Q3wdKBfMG& zK}2y#|FLt}E`UQ6t3jK#G&e22bMBc3=C)LyqU706frdCAqa;~Q0L5)KJ4?@h*FFu4 z!s=hOC;G?Q)BRKJ1q_XJ9W5LLejp1L*187&5Bo4Of)k>T=WpQl3v#4iX$574fW`p+ z3m}r-F8Gjv1m3yTia=+2An1+E&psbXKjH2{<1xMb37`|D<%7c`0`~m0r>AQD^%nUJ`%PxS>)*{i zg?VHw)ju!$@$>xGszUyM_BsCF3*%>rxVZ8vrYB?PvDBBHQWz04T&UpxKU7{ zrb~8R4W>e)){FrKo^O5ts8O^r^t70=!se(2-(8&aTdaFU2;SR=dyECLBp|MVU@JIt z)z$TAHMKRnyX*5;O<*xm+(>Fo41G;Tk0w01ilh#uFJa{teQne`QCOHZp`&du5gkAWr@9Ywz%@P@KB0bD{lXo7PmrPC%J!A z%orlB>F}qRa$`XC2Ai_4L56#h2GWm;>sScPxhMO5a*guk2 z+56H}PZnq-sxASPn!B~W#8B1W=OQPf-lEbhOh%>%{AND;w%w;t<8%a%HNk`LQ0GpT z6au2l)=Brql2Fq{Kw316jHdW-WF<{46(Xad0uxi%3aEARVi*dKaR^jjW)$<$7QEiF z0uK-~dQ@|hxT5M|t$pBl+9IJig2o;?4>qY%<|sZ4Rk0Dc{ud;zd`g$&UcwLjY))aV z4jh&lc(;hjQaWB)K9EB@b^I)LQ~N_;SFEEWA&}`)g!E7-wzF%J8)yZaSOeR=igBiM zaU=T>5*oyz3jYaqv-RSC;r$%d^Z(cbLGwTQiT+3KCMt*OBOD@rPZ}8;)1_*l<5aBp zjl{A?HiE$Y6$NWUgPY(x@k^9)A|CC#nqZ?B&q-ceGE;Y7F{@0{lQuPnsj0~YX(VoZ zdJ})6X8821kH4_0vt$gocDeSve(SuROm_bM98&+q72$1m(x?A;;)@TWyuVXQV!{#( z41CN;(vq_a|56Yny*sb>5`lt+>?dvF0++3L!wQ_eJmXi)z_1UAmNi80_bG^|J$GZs zK^|0X@8jq9pyPt$dpiWWAG)mNg7X_BME=&UYoq>nc0gtk_YoXNb5hYb!hG ztf(P(6Bcy6`wroiv-5NLLjVBx&|;W6WwKMmB+ph%7$AJfV95||OktlFlTMqdKP0i#Y*rj`(XeYUz=adk`3hA(LvO`y z|0%R3GMWC#x}RbCNX_Cf;_wEOS}%lqj#-CXQDIpi8Qis%Radz>q0vjbY&8DdR>jXU zmvR%au!=9lMN?P=hzQpNGOJRw?Cn8@B@kEp4r5$bgdM0?Fdua~*H~mGTf}17rZog% z!Kj#>m=l>Po$A`_fcT-pHy*aya+n%rXmG0CJ6a{nF%>TfyzKC2Dit7a;!8r;X^G$~ zS03MClV}lI)S^Py2I2rLnpjR64L!#Fl!mCP0td}~3GFB3?F31>5JCwIC zC~8VAun2Z}@%MZ{PlIWpU@CJ06F_<61le-_Ws+FSmJ@j>XyyV(BH@K!JRR^~iGjAh zQ+NnRD1C)ttcyijf*{xky2tyhTpJvac8m%=FR-LL@s>rN`?kMDGf2yMliwkYj= zwEEJ0wlFp%TmE6|fiti_^wVrxJ#gh7z@f0+P!kS>c>;BHH)N`PW0JHTqA?B~fz6H+ zdQq>iwU2Kne+4kR2e~l2`>(-^qqujX*@|w7k>s=e)Y-lwoI{$Tx_2}&y$9LZzKG-w z{TH06d?a9;01ze%EvqDCEt;qAaOYdf@X)zT)ScQs**7gQ**A5+o9p#P*X5~lMpNl2 z6p=Ecy7#f++P2sk;I2Nd`w-!5Y^3QHV0RVy2<55pqQ z&Q&b+JIKTf&6N(UjwrECT(BwKhkdpc#(Aq= zyG*N2frC~4B2Ko7O)bOHP8(}XKc;_(GP&+{?#dJ;Y$YXT$y<%YZmc>C?Sik?i?6E1 zk~VKGMLlNws0d#wk-11tBrAf?Tbes4F)oqxr_*7R-?Yn4IlyyP_ce6(J&tXSFI~P^ zYG1K1&Y@OY%nE}Gsa8~iq!!=l4a+yi7?Rxi#owl|2CnVfey<;AkI<2^CN^r`;-)ob zX7Ccao0G6Ic0ENcm7#3(8Y>}hb9aL6Gi?llW(Kss_CW07Z*0rgVhbod7+2-z3EC%( zq7QLJy|>bn^fyDVwISg;I%*4-lpnL5wLoe=B5sV^!Vdseg%7piW`#>KU*HD}MZ&J=jCFG;)9zqX;~A15Xsg;+mAtJruykiiD4Qc5$;lWT@^-j>F$$|0*{U zmrM6Kwy7I0>uJ&DC#8>dW7&)!1!_uGQ@Mvr)n^bH?_w|*J_E0?B{C&x%7+%$9&Umb zMv=?f8jwV=X`(6MfQLkyXGt_A~#T^(h~B7+v?~%F6k&ziM^m_Cqb!a zf0y+(L*8N@-&FfWsxPx%V97(F{QW`L&>2NJyB_}HBTWa|xRs*TT-y}_qovhF=%OCJ zf)sDf8#yYtG3ySQ*(qqz9dXI;CfS6yLi>4H9w9ii-!j5NwHL>oEN83>IsEP+V_1~u z`?}q?(o8RjDY5V?z9HC@t*0V_hFqA|HyZ8k)T!UJQ`KEKMLlNlIq<$2s!x;)o#SW0?w*zVYU?yc(v(2qyZg z0(^T!7Qzhpm)`?PLS7z|(>s+ZUO?_>f0y8LjB9{7he}@4-%l99L!vhyLW=yQr!);4vCSd-wC1QX-%H=?#UM-D_Wg8t3W z0*rY0Q4xwb5i(lBSOs^u(IgRSP$j!PkhbcIr^rh}e})V_kU5jW{q)m0CALP$`wKi& z?444cDxl;D;SqSw0^h%eA6Ro@BhxmD!}qpGb6OxRi6;iFai!)ctW|gmF3jQz2*O}Z z*TPvZAxFr1-Dd!53U_WQMQh$aauyVf;O60e>&G;Mg83(TOZt!6;s2KT{}By>k&-_m zA1YA0q3ID6fx`!qxy=@dYO@Rn%rEb~7P_%;Dxvl(WAfiJUtti0?~ah#_1`K#A}P2n z7^D~GQL#`hC}2w`btD`i%)VBWnn*jWF=d!kI*6T5-wBdsT)$EZD=mrn&EhxJQ^3>1 zbLeDA3&BIDAv=kWsp0t6>a3lITA;khMX^(B8Ecb^U%P-|RNGB@XLq*Q5a zR9aZ8RFNDYvD`dcva-5ti*`CcV%ltLG;emYG)5Hvo^Boe6!Fu0ekZ(k<<5G3_4>Mg z-?ILGT9yB`Gy?Cnu(PO#(bsKyf9>@F_MJQFZFaBE?dA7x40K@HNwA20g&JE&q z6&$MUcmsL)Sq;;@a9!*!?ct(XynVCJutm{pZ5w3Xci1lQ!9oB`xCdL! z6i6sX5X8iljX<8L4KC)P_hyjfBo3W=8BfQ5^inG|_NhXI*k)fvrDRq;Mtl#IdM%t^ zo(9yQnnQj}I{C__YBGYykMvG(5)bL%7>X@vm&+vnDMvZ(QMVC;#;@DZ9#6!r74JA`7phVA#`JE` z>BU^K@B>jj8Maz2m^>t$!%J^m)e|Ylem4L>e=OHtOVBCDy{0or$Np^VjdNl=g3xT8 zqsE*&O{Q9{>LhP;F2vpR<1t@fO4^Fbd{cO753U@l zLFAlS*(cze1w03?ZyLxG9S&n_udo?=8ddzgt#cv5fKd+uyogyl;44IK1&z^wj=!YK zzUD&kgK%`pt9A4nks?WMImECKCAt*xUXcPbo9e1&PmWU$X9~!}HO|j@r(`+=V^^Lc zcLMKF*Yj`EaS|pmb1uaDbkZvx6m%4{=z+MdgTuv?mT=4T&n?h7T_tQNFYhz$`~(DF zx4T%9nS-@(gWPm3?tZwJIpHDGWzAJ__zZKP;Hw>~%&n=s$Pn?6CaJ>bJzY?o)(O#~ z1fxWpkgP7ukZGyitR1C364Jp*?#{WzBom;9o=XrY;V#_Y5@5*}T5v*hcW#I;Sb)H; z6^g4&{fOcGP0zWCURc5J$ExdSY5s?r-^r#;|BS)8NjQH2--6b}!Q-Aa$mx_pNnz4q z(1_zCdqOu|4b4oo+-*jjTTV_j3WmL9=u`0(l@>00B5Vg?4f?fqwWRCX*2JwC(Yd+i z5A-Rm0r4e~4ceSJnEmWF6Nk>Q;(7sYyQ<-CgPa1fO8m6_pu=Maf0e2hd92Q#i7j?U z-VR;%F~r=@Xs>J2`Nx))UK=X`Shhg3AWzbwE<#%hM+KSQ)y~F!~7j*2}qu zgT9Z6kE4Z|n9Leb=N0%JnFI$AeNrV+!>E(WT7dyOjN~44BhNVL4(%Eo(1JGjS^)Oc zjSPsu`3wT8k`$>Na;G3pMU(9;+ov}PpiRt6*)WNMy(rEUak-14^(K`73yJ1#LZna? zS)ypsH=xt_ z1V%Pk;E@JqJeE1&xI}|JylZJSsu+mw#r=)G*5DBGv*`Q|1AC+!MW979QEZ{H5*8ZW z_U8EI1(M1LDjG^#yy~(OGH)?SdmR~=ma_^2Q#k>)`v#$t=~Ih|79!ZutXQTK^S&w` z1)ONotPDL(cz!_@bFBBOo6W@;7Zz--d9JaOs{)ss4P|Mr%>FaiMR=(fn-Y3SA->6~ zp`5h}dOcY_YfweZB*^el7qqa$&_r-Lg-I+9~U z`JxVCD<$VmoiR$g^3dU%7Sij)XYi*?$#ihSxCBHGOaRRr|Lo9+E}O~M>I}tnokI`}F32Aty#b8rpABEKl|B;*o8ge^^)Kyk z0!(>gFV=c)Q2Y%>gz+sa3xYTUy_X`rK5ca{{erC9WJ3EPKG{|Nng_-78kAD{oh_=K zn*wopK3cG}MBJf%6=}9YouD;zyWbjRt%A#pWc1zb3@FB`_Q~~UI!uvse(FQfl zUt=Qy2DSjwpzAUJ048~^;@Yo{C56R_8nZEeF}vm)0xoYe0y|tYI!>Y(d}mSro0`z; zeb6Eg*(a2{5Ypj8S$-_~L)+IlozZn|Iak`$jQKd63hldhts0=m>k~HC&`@|~;XaG6 zLVxC))8>^?13P*mV#ydlkC0V6AWK(BjWpqu| zbh7#bkKuL<kv5;Emm4zkF;X>rfbzAc7!Z)i};f=*bypYUD zho5-B5n;)FP(nzq8FG3TH?7l0vS{G}G9@~zxY>CqbX^mb$|JncS3I_2RD@?I9bz>LbX13A0N_LQmd(!3AxqmR_;3bJavc81%v z)Q~pDm0d1VrVe~>X?GOUOz94e6Nbt|fe6(S@cN64Gy6{i*TPukTmfvgPR>+qe>)@w z8mS6=rvR0~cqVfEWFsL|kZ3t~m-iV}va(IjJ;Hh4R9uISa6;@9d{D+7CwskGx!7MGZ6|rdE_I{cMD}-` zoi0%doDSznN-Evavf!_d@UNJt*Fl;hNrnVT2Fal8iBh(LU^l>8I1%x!q=6A@zO6O} zs0R@~z(6E;t~6L7tclb6A}zwwIvS;W`?F>>P)INWt6N9r4JbH*;&^6B!lHNAY+v3R zwCVoTTSL`1XtRZ_9vWH*(HcV?PImcNBOtbC4{U(v-HA~xMdpP8<);Xv0y_e1i%t|f zdyL`MtgjoC^Z-wGt@&6(9Wx>;qYcYwopK7H4iejT?T|>BSm)-fV&7yB;ANW4ZRzzc z?^;uh#-bDq@QjjBiIf-00TSw~)V;r?BHNEpDb(dLsJ_Z!zT7<{oC-V^NTEs|MeD0- zzuH~jmz>@&JaYIW>X&?~S>~+R!;wQOq|+{tI&#vV^n%|7ksh!vXzONlSb4zc!X;}> zMaUjix==sr4oMiHxL@~MPL%PrMzU{DPuz`9zWln9XnqKqNo3TZc;22OZ{ zy(90FLmd!qHIv!b-q){c(0@VYnzE(k5#rf~N5m{u-X za_J$`vM`7Bh@_`N%&n~35!O^m^pyWGR65?W@EH_fG}veT4I>@L72iny$1yuwBopv> zsSxe4Htw2+2f`M-+7|iva$OjEp*e=6r{J`{W_IyMTo#x0Yayp+V8z~17Hx&~6G%t? zN=#7bc$BWFl&qzMvU^iRl>Rvj(_`fR9T%ZBYX1?fg((%9FgbGrBl_7^rRQW9GA*@E zLN~c4F@W|oNmH$kHZ)4U$u(P4S;GSPDy671d;6L8z}?RfSb0PHN)PsKViOm_PLB-7 z+-+jjpC&oGWj(BQ{|L#DFOC3+-%fvGOOx^u^Ysxsq)Ox4^;}rM$!;(?`m@wtkXb~%u$Zx% za#IBD9hq=no-2H90jB}1^>TfWp)=Sb1v9w#UAHvYbn1PpHFbB+hwSXWK(ta=^8VN< z^j!PhT^ZXf#;?$ZWkn?(vJ20u-_SsGO1os)z;s=hI)d6iN-4mC9>EtcU@Mybflo@| z82lRHB)FEu4k@P9W+a)>t{^Jl;)gL&tWZBy(gWmfXX8XiUdnU>LtbceRd2RogiprV zK3KHRpSd5n#Hy5wQ!-Fg;{(9?K%pRuAEZwPR-E)JGeljq?MUmP=K$zkEO46*td&DL z%C4c|+^C204zq3rsTdE?%Y;lc1vKitClZ79P)GU-k`VCL5(kX_>5D{)C18r$^duj) zab$~pZ#$FLi^ihhytr80x6p2DsA3IsHPguaQ&s4izcL;7qGj1rPQM)4uc!I=d^j7S zs{`eqUlX0}s<8@_Iij-NBLD<2BE3VJ&k4Z6H;z?!7!7-XeeC-aX{Tl6ml!93m*cFJ z#Z5Q7fr}UC|2wXN*{|KEWPZ(V^*agnsVlrYkAd651IAl&yHxt9OnMCJBht5xn*lR2&NabYN zSWC^|d16K9!d@LjLiX4uEhz;%>2G#@i;bdI;t=8bK>y@P)WT!mDr~z}pG- zRg0M$Qpz0mbKF!xENTw8!Wwu{`9|04Gou}nTQ_L@`rl58B6UT^4~-?*}V`fYfKSaDIH zavlsK6XsL9-WmdH$C72oMpwJp)?;)Z4K6Es0B$SXP*QhM!gvpdUyI?}p1c2yYhY~r z_VvRqI~hi$_97U@cE5#Z{Zhy&EqB*`vAMpf?Ya?h{;uuk-}E1T!ah4kx_Q*9mOjl* zv62c1x-eMCSfQ*b3b|P6*~#_2>fN2y=iJQy-I$q_TIV>AHLGvxzY#v#{w}OBR>mny zZ+4AXVq%F7d*h&{U!c8&&KUXS@X->Bu@pTF71|eeQVYw8ns~h`7|n?)2@d35c_1Jn zeG)5*kFZ<}MejgYN(?7Nw?Mod)k5v*wm{$@osr)Ywv-QvXpeI;3Qku^T}zo`go?co z|65!$tORilITCe4GfhNoqaj~NtO|@obiA%Tub@&qQ)*Sn14oz#=<2osGcxe*+@PL< zyx=_nR&*Un8g$Iu#el1FV8xS6kKlqt6Q_nLmsoyCCicctlpM=xVMApO3V7u00mxNJ zn8H5H7~1cY0)_}KJSfc2QSG+HDoQlkX^Iwi_%Qb4&1XPlDw$%cwf-dlhzTK+<_D-) z&P@=34aLr)@%x%0WcLNFBZ4im4biAYc zX48#WytT#YP@@jEfGgaR&J#HZzJa@HjxyMYHe{pLPnxkn;~Nj*Rk*wS5*frI0o^@# z&G3U*-hF=Y_v1Euf&ZeY$+hsoi~%M`iq}OU5nnKjI6qCo7#tk{_f3pIO(8(pMmgCr#+;(8d(-5n@oY{gBKSFB;sfY zEGd8%M6}wgw88w$*dURSw+YzI2N!gycd}~V$*T@AlPt*-f=web80-YsRGL; zIurEoITNgt(oy6p0G%)TAq})jmI~qDOTd#8SWUAuE(*k}kk&NIGfR#?MWZ&@WgOiL z>$#C7>im5ft}NgVUz#o-;GS~3h`u>vuPTQ6J_?slXE&+uSm7V8X2xqGN*g32wQVF? z60uDVd}|BtzXW}IHl+O9$Y${gL@oN<={bc5POfF*UaM4*ulAX=jeCFG9716kCF{ap z+Aa!D*;gIqFWp_D0@7TOln&`G=|&m}X{5WP1i2vScNypR7x`wGaTX8H zJ@~rx)5+w$k^uMixVE%C0WLCO~Q+tBA;H0@eFG) z9eC{^DN&Wg*!QSPZ&6UQTXd8o&~Nom);LFsVoC&=vbu|xNN`s-1=AH*8)z4To#%#y zdd$@UB#=RyuU6;>-mgB-YAnr|4VG~L%5Zu?2?e8cV@hX1%$C z-Y!`@^OUFtA7Pe=$M(LJiXU=J1!QUEtKOP0NQ3X zL0EH2;5m@t@SxuG%G+4`P52~ZYSYtf<5_!E_05F>!Og3NVhP<3((hbndMVWA>MlDv zn$&G-7+NQ3%TTa+SwC{12rdHZ(>d@r=%m6}QzK^c#Jf1mYV4ihwfN65H)@P8$MxDc zTjl)d2R0#MAxtC@z=02~@CN4)F3cc@}c$eNk#9s}m0 zCQU1m>8KltX-7??Rz`KAa9O`78vwc z96b`^On^}8Uq2X$nJstj(oDH3I)|mIuLz zwkCtM6CN9f((dN*4jqG4{_r(Wh z2u?7~;PfTgKZy`BNs+soV7l`vUoj0Zs59#tk&2GGS#}^vM~n9_o1()DH&=e+1J8g6 z?KqAZE{5+wu z^h1JTDHbTO>mUG#C?;6@CZ1@94=<&=#wE65{;Up>sTq@gJ?nsNSa6zE7ZoR|eSK`& ziwVJeio-qK&1`}djVaTPBHAtX-iedlv!W}@HqzoQ&gu~oM(#ZleNhagi2S^z0$`*2 zvXv*_l*3vp7N$6SniJ6keA;%N);Z;F2X+yzHXEKK>|!l-K+oBIB9Rg(r?T)}`0nwz zW>J5H2T!yBBQv!CV3wS!?e?ao$JZGHB3>?^p;I0oEq1rFbn-K-z1;UX^Zco(t|y{F z&aaht8|ducgto&gzsFOSGgDA6d{NN+DwNR7IvD2_ztxv{`PTvRQAD{R>ii;bqI6H$ zi~7*gkXL6sk*D( zRfRn^T)TGZOa5H8)%KL|b$feS+tmm`x=ir7xA_SFtXdrfwMW*l6LlqDsdN9czC4LZ zxQ1hx2G%}RlTH8PFjxmCx{XLh9X)5F)BD@x`3Yu(w&|MQ@Wn))MQ5P40oe6lq zj6&YQ)Y$fsl?yoMn2DRKmBXL&;#5@wIec)ey+_r)wLWKQ$%Nl|=)1S>2v2Br1GB0z z{26J4KqT_fthh6KL4A_nUGh|M?rQeB3d2M>f>?eF=%>&KBi ztb~177I8YO@8HV-(xw2pP4vCgNM_ODMc*XT)Vb84bZ$(aRZCi0SD4Vb5~0yzn-7uD z8&6`h4|PfG#@4O=sM;eev2gieyH}I*Rnq8!MO>k8@S&aMNX9c!hpUjKeRDUN*M<4& z`yP541rMR2;EXAYLf51%0hfLwoLO*VT(v!KEHyrD(8{a*@p_=xOtG6Ck0QfS>k&u_69rGu_Jt&YG97L`S7&3_{l%EQ)VAjX z2UV7D9)#I1Jv#8Fd6X+dOxjZTXFW0vpAv0)rZ!Ck6!Fz&&ZCezKS|5 z__!pv3>!#(zZ}MQfb=Bz4!aBypX`XnE#6B?yfTCmP8;tZVe#%QC2|cSbs$Q7mx9Wk zrhgq}S`lflHu@AX)_|0m0Dgy%FGt|ZP!H;(BN8Ff)p``6P$lT2Z4~=eFDFmYJt6Yd zs+IG46y)X4Cg=VU%>5u$6hq|9hlX$~MPeX{3SWik%ZBMETV^`}7l|$=T9oPv=>MfAuVpVuT?xQI-5MnhAwB~WKF3p#jb^%x)hgQ5w zEYy^HY%m(3qgTb0>_xhyGy49WgkavN*iwr9){qxmZ}0h)}ji`R&Z0sEAcs4@JVrXS$uNXI67&^So5DE z_wSSV)|hizP*Za+cCTn0^tCx`&1B`kM^^O^qqM)Or4WgFyEKhu_AWCV(8q?&7iiv8?d=$)b z1MCx)Px;%)v~QO*(UKzoMpj-f68L&<9G&jy%k26a6l~xWa27d=0zy9Y?Knv>uTy3B z#R4dYL0;(wG{B!VU<) zL0dQ}cE7}kSnh!@UA2Nn@KkO8%G$oaXs^?*bXW`@IS`edO zPr)lZK}u7D_99TTzwi<#blDq<%z2HzF#{9rVJal40r))tDNA4@UK9YkbOz5og)RphDfLoH8TaTJ5@i1x@Ntowsmz3c5mldGTpqbAC8z+-y z3YUgK2;tdm95YQ4$o=gR_I;ot|JG0jq~!w!JryDgGKTgLd#SK)h0Z1kh907bO~U(% zT6jiFnX@TWSv@xNo`&z|2;9Rf1$ArDtzSTk!BFYr;&ymtj4Bt1vK|q*ut&Efy?Wd; zk}_qM;ziWm-`?rC{al#%^wRcw6wOCC6Gv|Oa7>zIK{tOroHE9p3-q;DwTZq9(y|SP zOB|hi75t%%z@ZErp@owZiI?H$xHMR7h2k#XwmQmT>7xof5gx@XC`fVWVA~cioSE&K zoAYasmf;04$arj zg1&eL7=I?+WRf^o3qFw^#Y?d9v=-_zeL94x2|usB_;~yo&#*;J>I2Yf+qzIM|Bzwn zf!lXOXQspLmvN-cJ7Fy^Z9K-=NwWY4W8RL-q!b82mgurWTar+^3SwpU*Swg_MY|-s469h*lM(kJ74z%e#v1B%~p6k+k`Zr4M;9Y)5 zrQ#%yC8mb5QdUfV#)WRwxc!2-9CA{=B zX*|`We_=f<%xhLdJy`#KbR#+lj|R6pJG@ZTcZtr=Ff(n00MTQyi<~xkl6_QIxuYG4 zAn6QsfWJSaT0)kmDQ#9{(H8{k;(F3zbIvl5oA9MZn}6VxAW4VEuDJQJ_tvW3^8<=i zgp3DjuXDefv#|&0?0j(&4lc6i2+%kQ@a&fm9)1GxAuGZrRy#lIac(Y6!xvAGHrz|( z)4AuuEkq7`w4@FDUqah3+{y7xTbMo!P#&kbRy-1zFRXRTL}Q62x?q@Ltwnr zqyF|*{ZdFu!MG|}fKcf)Jk0y#Qk3t&@IZLWry+1U{!CF4(R_B8fZnVnvN#y`yJk&8 z5o|-I$t$7DEs@z0(ie7=MpaKrn9UfAR;(N*a)J1eej0*KIXkIFx?K6bYtjN0RG<87MN5Ph zVo*0Xd;_STda7fc?U{jG%U9FOdo7NOGFCBEBwR&j;4Q&)m*JVsL7mSZgs;+{K}z*uLldQDk~pDMMpTRSMayDpW3jXcP-aFaK4SRwhOg43SAApaG6v=#1q zJc}I6RObkNMZVE@gW2>|4+xVVmeNu`#F_MzWq24w2tz{n%bb;&u07(#9!N=hc`@qKm@EtkN&lDJr;L zvk}HQSsd&o7#d_Yb%Py=9{clqy|F19S81|cMmz<+n!5J&3Ck5~Y}=}arb30r5}^V2 zwD^K-=syNKf8H+4r==Oz7M~|D34$w9WiTg+r6;uognB=hj*}U3^eWO|j0up?kWWmA zbEER8t!`eQ+ApRkQmsrzPN32!_e#P_Bfh6aGOTD3gOGBH=Ob&R+Zi30Sc%Aea9H~7 zEB4j%17ym*rkGd>UA_HLZ^3@`9`Eu;NC;;HEL3An;iEgR+j-;5@XGL#4o02(SG@?! zmNW>y;+PQTA_i>3r%-PIQ`x*!@b_24mk5(I-0 zzIJW*ZBIgn{B;FFhh;m=5q`WK>P;)21@!H0ON)E1P2mW93!PsfiMK!~#1#~LLfyQC z=}TF_5|H{5J7GF~A2vvJiJs7KH5%w}$Y@iz%2sMQefiYTC#VW!XWSEusTc6L|ImO) zFuc>MCylPg;Rn_By}7kLshEh9A0guK0m6Y_KKvx}_MX5@{;8^|M4lHz59q-^n>s3N%P-)wu*Apy1c*uY%ls6{?1UoxSMsVN7r!vmY$4U1ZpCFZp zSB*$nRK#ut<0W7!D`6u+bGR?I9e<3Zx6iW5FM1YNJ5roEjQwT4gD$elG@b7S?XgGj z6?8Gv(sGLkkFv-Bz!vs_FSNi1>W-{uoLZyfxL5}8Z{yqaEK9mx*?8EyKbB&|oe3nO z8VPv6K-BGik_oh;MUxzP=SHYz+sWoU*_Pc|ZAp%rEG2OgkyA{O@|sV48aj}*$c=#ReFzE9^##pCm4G| z2ExX>|7BshOX&F%0r(Syy*@UGUX!?ky}6Zz8#t5q|1GZL;`G!$N@DbUPo4((w_%ge zvSuqV7dVNPK^Ue9v@t}A{2cJ=Vt!H6_jWRDXA_0fHLnagK+aM{WcrW(C(d1S@nS3RlL zUYh7&54coZVswV%&><$802)Ds6(5Ty!)=(|2PPPUY}b*5H@uVe7@L=Qb0@q9St`u+ zN_!X`!fP90I@Pzd3+=S%-p@UT)RD36;vT`l)y>59$+Nk(IHfmD3&VHLW5m_Y`<9v9=7o^jo4Lz36MNl!%1 z3c{>#C-z6vmYddm?8F5!nukB?&9Qdzs!KMBj{!#L!8zi1kBIRuP=&b|uHG%D0++Ww zKF=0w;?gq+M!;#eX^_}Pr4<(R>gE(Ur;1)gwTux=f1IQG>fb4lRG zauq6JTk=W;nN0r%g|iMMZts2#+~Kw1kA-3nBBM<2&r;0npESg~K6u!!V7Y-zgy%jr z!=09xB~ev~Jcp)_SGwX7G$-j)q(48uz%aSH{(e4l252lUj``uz&I8@A_=KdyUZ?@Q(rXR552h$Wp&%Sm$b-Okpa9CMXW*$|8A3#-)8|R{nX6* zrI}P?wPY7piep=yrIXLRu5>57uq2UvzR<1~NwK~f8JrI9srnbs2UA;5UgdfyLRR&X zAXqb}GL2YZjX`a)UZ~1kU9Bst!uiUq9|M?TT{2V70AVJ|-z~5F6{)i=C=%eGKF6%Y z7Ft=6dZdWTXx8KXRhtxFSRyM*AuF=@3GUfDy+`L!cV z`(^xDDBY+K4#OC;>}DddEs8FK>ce{#!e2#ud;xxKyt5wP;!mD`4l^XIWLkqgMWo%f zaflwyB3@QC!jweeSK)r;DGG-cCu&bG3U3{ikLdi;H(v7DU?2%M?3qCC8b93Hb2PJ8 z@QeX-JYCs{mGVMLlFvfm&_dn3r$3Xx;jR^+ts(ChilDJchx+!Diue#c4B z*?P;?K7WLbI!9T{JovmNd>w<{$E!;H66`ObfV*qFGyRM4F5w9=Avky7CqrbX!vrp)1mkD1rC#mdLXdN5pFSJ z*(*Zoh!M$6Z&r2Qz%JRl;UnMd*_o@|;^NH2X#LxwMlEsQulGJjB@VuxX*cV4`Lws> zjl|ByKhtDk-fUo=Yh_xY^aZC}aF!_|(lIkA7TzQRY(t0p>Gd&tc> zes@Omai_pyi@$|MbZVE&ERRd{jvv1`xy40nO-yXFC#y+=4&S)Sp)+(Djck1bYeH4! zm3cZ@u`K`0Js)Lp=f+iJs`n|0M3vE<8>IBf1WpRk4Sn<9nsijK^v9}F8FXx52olT* z%Rek&eO%wFlj3mYQhb}!v=YZXUUOO=$D~YwDZ#~m7 z44|QAFF^b`OSw!ZP+^L^zK)1>UerWGO_E%p^2sP({CtErlFQfrt$O>4 zcuslow^_3ri0HuWcigZz2w%Q*7cm;>40)1o@kz}pysE50TzoIPQwuXFW}elhNffQq ztZ)$Oz@XwhOmbLQ@ zHdq2g<@TQ%lSARCV#zL2X2O~fLkuTD81 z;n(NWjoQXwD1@m_!wBJ5PzLd0<=A+CCKTW<`dnOI=yAmO5HaW9zyjJ<0ws*rHnyd_&^78n&clLII+-hONNCDg>?d-5cWDLC_b)9n6o{P1CU-$7L407s-_ z-pN>_?^HhHRDQmVX3NRF#4(=Jdi27iXbVZSm@Te&4UHIPDSbLIRgksrcMi!}LH8kx zi1kkV?^GlM!Caxc9^)p1vBDD=F(&PD^l79>spQ`#vz{QD@ z9VQiviBfRP&y$x0E-FU?(j7DNYgz5FnO9-1U7Fj10D;J3`ywYGRtdNp5Y>Qo+1-P@|$#4vrd!{It&D4(5 z88MK>t&(M*q{{bk+gKz8BV8NoUls7#Pa(Gk7HG*!WO1MnoAKw=-;D)9T2XpobRN@;R9$ zdDZ*TNdMDRe3pcxxWT#?Gvz6$N>L_At8M<_Nu!G9BUfJBQ zeod4i4j8la+F6~Ch&@o#a%JWXtFx6-@5vSL5;@>X>|ze$N=4Jovjt5>8c*=P)os?J z=UlsoH#$Jz7vfg0g=+%Jf)w{Z(Z%^d5W}1#^0}%BgEhRzNs8I2&P7V?GtK0o$CS>y zS%AH91idyPyNX-#5}K5@2VRQ>?Da%6Q(1)*NzRxW9-2LG&+L zW9v~&N*UPrd!ao6TTvM1O*2z1?grU81wdZsv-2#9){B=Yo58FPq{90cNRy?PdBzqr zbXR&i)#}mnzKE|yj_#pCV$njDr<`4a;0d&q@G_^+74Q(M$6rW^ZRcZS?r=zYm%#Gj z!Sc1I-ZxAVPnlVmU2ukuW86&QC4@4nDGZNmY%^`PdC5+u~%7?p{5Ihg@E{qe%G7|%$x8>B2lP60{y^WAi!)2f5_jj zyAZ&Czma_OcZ!1f$!-?4yN(KE{v8Flf2F|VM_l1=DI&Z}(RBvZ-?=MJurdV+bx}qc zMM>r#Mp-#9xf(Dlj7$ur%9-=K=m+1QT9ro_U?#&Wv%M{`+o5WT)8b>jv9 z{(W;{+`KsjQAHU^2{m;l1<5DCcK8k!lt%~8FU9>xGEa>%xpxcvNwk|}rEBVH6gs&y zcc%2{>C}&E29pz0OWd`^u-ES8cTVPzX`)(qt=d?&K@&=Rotx78SlqgrEVG_qUo)_mC$8U`F#qlHOCD&RSroexT?YJLzvne^0W z@;=|QRR6AVW@n3W0fEJOGM5gbEhzW#FFa{0FL+k>kgt~r3DnajgxZvn2mk*LWvgsJNdYFw~S!X4cFe+Q;Q-_W%N z9+%cg5D+rIfU$v>NB;`!-|$Y|w(+s#2VpgER|yU}|IL~d1DHEF1OAnnMj?dmwqP?|!Tm)27hExl-^LX;b^(CT z!UODGtX!?!0czl=9(xOLEjt>6{g40iN!)JVBc;&q!{D7LBTNX0>kPC%g@yXJ??CR3 z^oF;AH}dO}OTni1fx&;Ra!+t5|8G{gf|ZL4*w`O!41NfJAE&N>zi#R(&V#)+FzyN% z_g90{z|?BLiTfv@hp{u@$1u7B_-1N#iJ#RBzM2BR!2c8QKQ->n9NpJB+kXlz_@(`y zApg-W%GVs=-$=u6Jp_Mfr34rf;5=qxnT`lG`0>Z&B#n)_ODW`1+jPPicN} zhgOBZJau)7R=(j9e&@_!Y{d>iX#+|6|i>`&Q={(}Kji+O zpFcjFOMd9Ss|3O?C362PVeDvZY6)PztKhZE=cg?HTJXn${I25H4xgVwR(eM*+@Z8Irh^0H1^@(vM%fLB8x9<0IcS*cf20Th OJOEd-=rxTO#Qy`$*1Hh^ literal 0 HcmV?d00001 diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 000000000..eb9194764 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.3/apache-maven-3.3.3-bin.zip \ No newline at end of file diff --git a/.settings.xml b/.settings.xml new file mode 100644 index 000000000..8b234ce74 --- /dev/null +++ b/.settings.xml @@ -0,0 +1,66 @@ + + + + + repo.spring.io + ${env.CI_DEPLOY_USERNAME} + ${env.CI_DEPLOY_PASSWORD} + + + + + + spring + true + + + spring-snapshots + Spring Snapshots + http://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + http://repo.spring.io/release + + false + + + + + + spring-snapshots + Spring Snapshots + http://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.adoc b/README.adoc index bc38a291f..05aec3f55 100644 --- a/README.adoc +++ b/README.adoc @@ -1 +1,3 @@ # spring-cloud-stream-binder-kafka + +Spring Cloud Stream Binder implementation for Kafka diff --git a/mvnw b/mvnw new file mode 100755 index 000000000..e3511489f --- /dev/null +++ b/mvnw @@ -0,0 +1,234 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # + # Look for the Apple JDKs first to preserve the existing behaviour, and then look + # for the new JDKs provided by Oracle. + # + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then + # + # Oracle JDKs + # + export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then + # + # Apple JDKs + # + export JAVA_HOME=`/usr/libexec/java_home` + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + local basedir=$(pwd) + local wdir=$(pwd) + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + wdir=$(cd "$wdir/.."; pwd) + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} "$@" + diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 000000000..e3511489f --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,234 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # + # Look for the Apple JDKs first to preserve the existing behaviour, and then look + # for the new JDKs provided by Oracle. + # + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then + # + # Oracle JDKs + # + export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then + # + # Apple JDKs + # + export JAVA_HOME=`/usr/libexec/java_home` + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + local basedir=$(pwd) + local wdir=$(pwd) + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + wdir=$(cd "$wdir/.."; pwd) + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} "$@" + diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000..7af25d3ce --- /dev/null +++ b/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + spring-cloud-stream-binder-kafka-parent + 1.1.0.BUILD-SNAPSHOT + pom + + org.springframework.cloud + spring-cloud-build + 1.1.1.RELEASE + + + + 1.4.0.BUILD-SNAPSHOT + + + + + org.springframework.cloud + spring-cloud-stream-dependencies + 1.1.0.BUILD-SNAPSHOT + pom + import + + + + + spring-cloud-stream-binder-kafka + spring-cloud-starter-stream-kafka + + + + spring + + + spring-snapshots + Spring Snapshots + http://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + http://repo.spring.io/release + + false + + + + + + spring-snapshots + Spring Snapshots + http://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + http://repo.spring.io/libs-release-local + + false + + + + + + diff --git a/spring-cloud-starter-stream-kafka/pom.xml b/spring-cloud-starter-stream-kafka/pom.xml new file mode 100644 index 000000000..caaf900ba --- /dev/null +++ b/spring-cloud-starter-stream-kafka/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 1.1.0.BUILD-SNAPSHOT + + spring-cloud-starter-stream-kafka + Spring Cloud Starter Stream Kafka + http://projects.spring.io/spring-cloud + + Pivotal Software, Inc. + http://www.spring.io + + + ${basedir}/../.. + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + diff --git a/spring-cloud-starter-stream-kafka/src/main/resources/META-INF/spring.provides b/spring-cloud-starter-stream-kafka/src/main/resources/META-INF/spring.provides new file mode 100644 index 000000000..0aa482658 --- /dev/null +++ b/spring-cloud-starter-stream-kafka/src/main/resources/META-INF/spring.provides @@ -0,0 +1 @@ +provides: spring-cloud-stream-binder-kafka \ No newline at end of file diff --git a/spring-cloud-stream-binder-kafka/pom.xml b/spring-cloud-stream-binder-kafka/pom.xml new file mode 100644 index 000000000..3426dc64e --- /dev/null +++ b/spring-cloud-stream-binder-kafka/pom.xml @@ -0,0 +1,143 @@ + + + 4.0.0 + + spring-cloud-stream-binder-kafka + jar + spring-cloud-stream-binder-kafka + Kafka binder implementation + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-parent + 1.1.0.BUILD-SNAPSHOT + + + + 0.8.2.2 + 2.6.0 + 1.3.1.BUILD-SNAPSHOT + 1.0.0 + + + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.cloud + spring-cloud-stream-codec + + + org.springframework.boot + spring-boot-autoconfigure + true + + + org.springframework.cloud + spring-cloud-stream-binder-test + test + + + org.springframework.cloud + spring-cloud-stream-test-support-internal + test + + + org.springframework.integration + spring-integration-kafka + ${spring-integration-kafka.version} + + + org.apache.avro + avro-compiler + + + + + org.apache.kafka + kafka_2.10 + + + org.apache.kafka + kafka-clients + + + io.reactivex + rxjava + + + io.reactivex + rxjava-math + ${rxjava-math.version} + + + org.apache.curator + curator-recipes + test + + + org.apache.kafka + kafka_2.10 + test + test + + + org.apache.curator + curator-test + test + + + + + + + org.apache.kafka + kafka_2.10 + ${kafka.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + org.apache.kafka + kafka_2.10 + test + ${kafka.version} + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.curator + curator-test + ${curator.version} + + + + diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java new file mode 100644 index 000000000..c523f0854 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealthIndicator.java @@ -0,0 +1,95 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import kafka.cluster.Broker; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils$; +import org.I0Itec.zkclient.ZkClient; +import scala.collection.JavaConversions; +import scala.collection.Seq; + +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfigurationProperties; +import org.springframework.integration.kafka.core.BrokerAddress; +import org.springframework.integration.kafka.core.Partition; + +/** + * Health indicator for Kafka. + * + * @author Ilayaperumal Gopinathan + */ +public class KafkaBinderHealthIndicator implements HealthIndicator { + + private final KafkaMessageChannelBinder binder; + + private final KafkaBinderConfigurationProperties configurationProperties; + + public KafkaBinderHealthIndicator(KafkaMessageChannelBinder binder, + KafkaBinderConfigurationProperties configurationProperties) { + this.binder = binder; + this.configurationProperties = configurationProperties; + } + + @Override + public Health health() { + ZkClient zkClient = null; + try { + zkClient = new ZkClient(configurationProperties.getZkConnectionString(), + configurationProperties.getZkSessionTimeout(), + configurationProperties.getZkConnectionTimeout(), ZKStringSerializer$.MODULE$); + Set brokersInClusterSet = new HashSet<>(); + Seq allBrokersInCluster = ZkUtils$.MODULE$.getAllBrokersInCluster(zkClient); + Collection brokersInCluster = JavaConversions.asJavaCollection(allBrokersInCluster); + for (Broker broker : brokersInCluster) { + brokersInClusterSet.add(broker.connectionString()); + } + Set downMessages = new HashSet<>(); + for (Map.Entry> entry : binder.getTopicsInUse().entrySet()) { + for (Partition partition : entry.getValue()) { + BrokerAddress address = binder.getConnectionFactory().getLeader(partition); + if (!brokersInClusterSet.contains(address.toString())) { + downMessages.add(address.toString()); + } + } + } + if (downMessages.isEmpty()) { + return Health.up().build(); + } + return Health.down().withDetail("Following brokers are down: ", downMessages.toString()).build(); + } + catch (Exception e) { + return Health.down(e).build(); + } + finally { + if (zkClient != null) { + try { + zkClient.close(); + } + catch (Exception e) { + // ignore + } + } + } + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingProperties.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingProperties.java new file mode 100644 index 000000000..6bd233d63 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +/** + * @author Marius Bogoevici + */ +public class KafkaBindingProperties { + + private KafkaConsumerProperties consumer = new KafkaConsumerProperties(); + + private KafkaProducerProperties producer = new KafkaProducerProperties(); + + public KafkaConsumerProperties getConsumer() { + return consumer; + } + + public void setConsumer(KafkaConsumerProperties consumer) { + this.consumer = consumer; + } + + public KafkaProducerProperties getProducer() { + return producer; + } + + public void setProducer(KafkaProducerProperties producer) { + this.producer = producer; + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaConsumerProperties.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaConsumerProperties.java new file mode 100644 index 000000000..c52a38221 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaConsumerProperties.java @@ -0,0 +1,83 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +/** + * @author Marius Bogoevici + */ +public class KafkaConsumerProperties { + + private boolean autoCommitOffset = true; + + private Boolean autoCommitOnError; + + private boolean resetOffsets; + + private KafkaMessageChannelBinder.StartOffset startOffset; + + private boolean enableDlq; + + private int recoveryInterval = 5000; + + public boolean isAutoCommitOffset() { + return autoCommitOffset; + } + + public void setAutoCommitOffset(boolean autoCommitOffset) { + this.autoCommitOffset = autoCommitOffset; + } + + public boolean isResetOffsets() { + return resetOffsets; + } + + public void setResetOffsets(boolean resetOffsets) { + this.resetOffsets = resetOffsets; + } + + public KafkaMessageChannelBinder.StartOffset getStartOffset() { + return startOffset; + } + + public void setStartOffset(KafkaMessageChannelBinder.StartOffset startOffset) { + this.startOffset = startOffset; + } + + public boolean isEnableDlq() { + return enableDlq; + } + + public void setEnableDlq(boolean enableDlq) { + this.enableDlq = enableDlq; + } + + public Boolean getAutoCommitOnError() { + return autoCommitOnError; + } + + public void setAutoCommitOnError(Boolean autoCommitOnError) { + this.autoCommitOnError = autoCommitOnError; + } + + public int getRecoveryInterval() { + return recoveryInterval; + } + + public void setRecoveryInterval(int recoveryInterval) { + this.recoveryInterval = recoveryInterval; + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExtendedBindingProperties.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExtendedBindingProperties.java new file mode 100644 index 000000000..637d14a35 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExtendedBindingProperties.java @@ -0,0 +1,60 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.ExtendedBindingProperties; + +/** + * @author Marius Bogoevici + */ +@ConfigurationProperties("spring.cloud.stream.kafka") +public class KafkaExtendedBindingProperties implements ExtendedBindingProperties { + + private Map bindings = new HashMap<>(); + + public Map getBindings() { + return bindings; + } + + public void setBindings(Map bindings) { + this.bindings = bindings; + } + + @Override + public KafkaConsumerProperties getExtendedConsumerProperties(String channelName) { + if (bindings.containsKey(channelName) && bindings.get(channelName).getConsumer() != null) { + return bindings.get(channelName).getConsumer(); + } + else { + return new KafkaConsumerProperties(); + } + } + + @Override + public KafkaProducerProperties getExtendedProducerProperties(String channelName) { + if (bindings.containsKey(channelName) && bindings.get(channelName).getProducer() != null) { + return bindings.get(channelName).getProducer(); + } + else { + return new KafkaProducerProperties(); + } + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java new file mode 100644 index 000000000..359fd4d5c --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaMessageChannelBinder.java @@ -0,0 +1,800 @@ +/* + * Copyright 2014-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import kafka.admin.AdminUtils; +import kafka.api.OffsetRequest; +import kafka.api.TopicMetadata; +import kafka.common.ErrorMapping; +import kafka.serializer.Decoder; +import kafka.serializer.DefaultDecoder; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.utils.Utils; +import scala.collection.Seq; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.cloud.stream.binder.AbstractBinder; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.BinderException; +import org.springframework.cloud.stream.binder.BinderHeaders; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.DefaultBinding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.HeaderMode; +import org.springframework.cloud.stream.binder.MessageValues; +import org.springframework.cloud.stream.binder.PartitionHandler; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfigurationProperties; +import org.springframework.http.MediaType; +import org.springframework.integration.channel.FixedSubscriberChannel; +import org.springframework.integration.endpoint.EventDrivenConsumer; +import org.springframework.integration.handler.AbstractMessageHandler; +import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.listener.AcknowledgingMessageListener; +import org.springframework.integration.kafka.listener.Acknowledgment; +import org.springframework.integration.kafka.listener.ErrorHandler; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.listener.KafkaNativeOffsetManager; +import org.springframework.integration.kafka.listener.MessageListener; +import org.springframework.integration.kafka.listener.OffsetManager; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.integration.kafka.support.ProducerConfiguration; +import org.springframework.integration.kafka.support.ProducerFactoryBean; +import org.springframework.integration.kafka.support.ProducerListener; +import org.springframework.integration.kafka.support.ProducerMetadata; +import org.springframework.integration.kafka.support.ZookeeperConnect; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.retry.RetryCallback; +import org.springframework.retry.RetryContext; +import org.springframework.retry.RetryOperations; +import org.springframework.retry.backoff.ExponentialBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * A {@link Binder} that uses Kafka as the underlying middleware. + * + * @author Eric Bottard + * @author Marius Bogoevici + * @author Ilayaperumal Gopinathan + * @author David Turanski + * @author Gary Russell + * @author Mark Fisher + * @author Soby Chacko + */ +public class KafkaMessageChannelBinder extends + AbstractBinder, ExtendedProducerProperties> + implements ExtendedPropertiesBinder, + DisposableBean { + + public static final ByteArraySerializer BYTE_ARRAY_SERIALIZER = new ByteArraySerializer(); + + public static final ThreadFactory DAEMON_THREAD_FACTORY; + + static { + CustomizableThreadFactory threadFactory = new CustomizableThreadFactory("kafka-binder-"); + threadFactory.setDaemon(true); + DAEMON_THREAD_FACTORY = threadFactory; + } + + private final KafkaBinderConfigurationProperties configurationProperties; + + private final String[] headersToMap; + + private RetryOperations metadataRetryOperations; + + private final Map> topicsInUse = new HashMap<>(); + + // -------- Default values for properties ------- + + private ConnectionFactory connectionFactory; + + private ProducerListener producerListener; + + private volatile Producer dlqProducer; + + private KafkaExtendedBindingProperties extendedBindingProperties = new KafkaExtendedBindingProperties(); + + public KafkaMessageChannelBinder(KafkaBinderConfigurationProperties configurationProperties) { + this.configurationProperties = configurationProperties; + String[] configuredHeaders = configurationProperties.getHeaders(); + if (ObjectUtils.isEmpty(configuredHeaders)) { + this.headersToMap = BinderHeaders.STANDARD_HEADERS; + } + else { + String[] combinedHeadersToMap = Arrays.copyOfRange(BinderHeaders.STANDARD_HEADERS, 0, + BinderHeaders.STANDARD_HEADERS.length + configuredHeaders.length); + System.arraycopy(configuredHeaders, 0, combinedHeadersToMap, BinderHeaders.STANDARD_HEADERS.length, + configuredHeaders.length); + this.headersToMap = combinedHeadersToMap; + } + } + + String getZkAddress() { + return this.configurationProperties.getZkConnectionString(); + } + + public ConnectionFactory getConnectionFactory() { + return connectionFactory; + } + + public void setProducerListener(ProducerListener producerListener) { + this.producerListener = producerListener; + } + + /** + * Retry configuration for operations such as validating topic creation + * @param metadataRetryOperations the retry configuration + */ + public void setMetadataRetryOperations(RetryOperations metadataRetryOperations) { + this.metadataRetryOperations = metadataRetryOperations; + } + + public void setExtendedBindingProperties(KafkaExtendedBindingProperties extendedBindingProperties) { + this.extendedBindingProperties = extendedBindingProperties; + } + + @Override + public void onInit() throws Exception { + ZookeeperConfiguration configuration = new ZookeeperConfiguration( + new ZookeeperConnect(configurationProperties.getZkConnectionString())); + configuration.setBufferSize(configurationProperties.getSocketBufferSize()); + configuration.setMaxWait(configurationProperties.getMaxWait()); + DefaultConnectionFactory defaultConnectionFactory = new DefaultConnectionFactory(configuration); + defaultConnectionFactory.afterPropertiesSet(); + this.connectionFactory = defaultConnectionFactory; + if (metadataRetryOperations == null) { + RetryTemplate retryTemplate = new RetryTemplate(); + + SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(); + simpleRetryPolicy.setMaxAttempts(10); + retryTemplate.setRetryPolicy(simpleRetryPolicy); + + ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); + backOffPolicy.setInitialInterval(100); + backOffPolicy.setMultiplier(2); + backOffPolicy.setMaxInterval(1000); + retryTemplate.setBackOffPolicy(backOffPolicy); + metadataRetryOperations = retryTemplate; + } + } + + @Override + public void destroy() throws Exception { + if (dlqProducer != null) { + dlqProducer.close(); + dlqProducer = null; + } + } + + /** + * Allowed chars are ASCII alphanumerics, '.', '_' and '-'. + */ + public static void validateTopicName(String topicName) { + try { + byte[] utf8 = topicName.getBytes("UTF-8"); + for (byte b : utf8) { + if (!((b >= 'a') && (b <= 'z') || (b >= 'A') && (b <= 'Z') || (b >= '0') && (b <= '9') || (b == '.') + || (b == '-') || (b == '_'))) { + throw new IllegalArgumentException( + "Topic name can only have ASCII alphanumerics, '.', '_' and '-'"); + } + } + } + catch (UnsupportedEncodingException e) { + throw new AssertionError(e); // Can't happen + } + } + + @Override + public KafkaConsumerProperties getExtendedConsumerProperties(String channelName) { + return extendedBindingProperties.getExtendedConsumerProperties(channelName); + } + + @Override + public KafkaProducerProperties getExtendedProducerProperties(String channelName) { + return extendedBindingProperties.getExtendedProducerProperties(channelName); + } + + Map> getTopicsInUse() { + return this.topicsInUse; + } + + @Override + protected Binding doBindConsumer(String name, String group, MessageChannel inputChannel, + ExtendedConsumerProperties properties) { + // If the caller provides a consumer group, use it; otherwise an anonymous + // consumer group + // is generated each time, such that each anonymous binding will receive all + // messages. + // Consumers reset offsets at the latest time by default, which allows them to + // receive only + // messages sent after they've been bound. That behavior can be changed with the + // "resetOffsets" and "startOffset" properties. + boolean anonymous = !StringUtils.hasText(group); + Assert.isTrue(!anonymous || !properties.getExtension().isEnableDlq(), + "DLQ support is not available for anonymous subscriptions"); + String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID().toString() : group; + // The reference point, if not set explicitly is the latest time for anonymous + // subscriptions and the + // earliest time for group subscriptions. This allows the latter to receive + // messages published before the group + // has been created. + long referencePoint = properties.getExtension().getStartOffset() != null + ? properties.getExtension().getStartOffset().getReferencePoint() + : (anonymous ? OffsetRequest.LatestTime() : OffsetRequest.EarliestTime()); + return createKafkaConsumer(name, inputChannel, properties, consumerGroup, referencePoint); + } + + @Override + public Binding doBindProducer(String name, MessageChannel moduleOutputChannel, + ExtendedProducerProperties properties) { + + Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel); + if (logger.isInfoEnabled()) { + logger.info("Using kafka topic for outbound: " + name); + } + + validateTopicName(name); + + Collection partitions = ensureTopicCreated(name, properties.getPartitionCount()); + + if (properties.getPartitionCount() < partitions.size()) { + if (logger.isInfoEnabled()) { + logger.info("The `partitionCount` of the producer for topic " + name + " is " + + properties.getPartitionCount() + ", smaller than the actual partition count of " + + partitions.size() + " of the topic. The larger number will be used instead."); + } + } + + topicsInUse.put(name, partitions); + + ProducerMetadata producerMetadata = new ProducerMetadata<>(name, byte[].class, byte[].class, + BYTE_ARRAY_SERIALIZER, BYTE_ARRAY_SERIALIZER); + producerMetadata.setSync(properties.getExtension().isSync()); + producerMetadata.setCompressionType(properties.getExtension().getCompressionType()); + producerMetadata.setBatchBytes(properties.getExtension().getBufferSize()); + Properties additionalProps = new Properties(); + additionalProps.put(ProducerConfig.ACKS_CONFIG, String.valueOf(configurationProperties.getRequiredAcks())); + additionalProps.put(ProducerConfig.LINGER_MS_CONFIG, + String.valueOf(properties.getExtension().getBatchTimeout())); + ProducerFactoryBean producerFB = new ProducerFactoryBean<>(producerMetadata, + configurationProperties.getKafkaConnectionString(), additionalProps); + + try { + final ProducerConfiguration producerConfiguration = new ProducerConfiguration<>( + producerMetadata, producerFB.getObject()); + producerConfiguration.setProducerListener(producerListener); + + MessageHandler handler = new SendingHandler(name, properties, partitions.size(), producerConfiguration); + EventDrivenConsumer consumer = new EventDrivenConsumer((SubscribableChannel) moduleOutputChannel, handler) { + + @Override + protected void doStop() { + super.doStop(); + producerConfiguration.stop(); + } + }; + consumer.setBeanFactory(this.getBeanFactory()); + consumer.setBeanName("outbound." + name); + consumer.afterPropertiesSet(); + DefaultBinding producerBinding = new DefaultBinding<>(name, null, moduleOutputChannel, + consumer); + consumer.start(); + return producerBinding; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Creates a Kafka topic if needed, or try to increase its partition count to the + * desired number. + */ + private Collection ensureTopicCreated(final String topicName, final int partitionCount) { + + final ZkClient zkClient = new ZkClient(configurationProperties.getZkConnectionString(), + configurationProperties.getZkSessionTimeout(), configurationProperties.getZkConnectionTimeout(), + ZKStringSerializer$.MODULE$); + try { + final Properties topicConfig = new Properties(); + TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topicName, zkClient); + if (topicMetadata.errorCode() == ErrorMapping.NoError()) { + // only consider minPartitionCount for resizing if autoAddPartitions is + // true + int effectivePartitionCount = configurationProperties.isAutoAddPartitions() + ? Math.max(configurationProperties.getMinPartitionCount(), partitionCount) : partitionCount; + if (topicMetadata.partitionsMetadata().size() < effectivePartitionCount) { + if (configurationProperties.isAutoAddPartitions()) { + AdminUtils.addPartitions(zkClient, topicName, effectivePartitionCount, null, false, + new Properties()); + } + else { + int topicSize = topicMetadata.partitionsMetadata().size(); + throw new BinderException("The number of expected partitions was: " + partitionCount + ", but " + + topicSize + (topicSize > 1 ? " have " : " has ") + "been found instead." + + "Consider either increasing the partition count of the topic or enabling `autoAddPartitions`"); + } + } + } + else if (topicMetadata.errorCode() == ErrorMapping.UnknownTopicOrPartitionCode()) { + if (configurationProperties.isAutoCreateTopics()) { + Seq brokerList = ZkUtils.getSortedBrokerList(zkClient); + // always consider minPartitionCount for topic creation + int effectivePartitionCount = Math.max(configurationProperties.getMinPartitionCount(), + partitionCount); + final scala.collection.Map> replicaAssignment = AdminUtils + .assignReplicasToBrokers(brokerList, effectivePartitionCount, + configurationProperties.getReplicationFactor(), -1, -1); + metadataRetryOperations.execute(new RetryCallback() { + @Override + public Object doWithRetry(RetryContext context) throws RuntimeException { + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topicName, + replicaAssignment, topicConfig, true); + return null; + } + }); + } + else { + throw new BinderException("Topic " + topicName + " does not exist"); + } + } + else { + throw new BinderException("Error fetching Kafka topic metadata: ", + ErrorMapping.exceptionFor(topicMetadata.errorCode())); + } + try { + Collection partitions = metadataRetryOperations + .execute(new RetryCallback, Exception>() { + + @Override + public Collection doWithRetry(RetryContext context) throws Exception { + connectionFactory.refreshMetadata(Collections.singleton(topicName)); + Collection partitions = connectionFactory.getPartitions(topicName); + // do a sanity check on the partition set + if (partitions.size() < partitionCount) { + throw new IllegalStateException("The number of expected partitions was: " + + partitionCount + ", but " + partitions.size() + + (partitions.size() > 1 ? " have " : " has ") + "been found instead"); + } + connectionFactory.getLeaders(partitions); + return partitions; + } + }); + return partitions; + } + catch (Exception e) { + logger.error("Cannot initialize Binder", e); + throw new BinderException("Cannot initialize binder:", e); + } + + } + finally { + zkClient.close(); + } + } + + private Binding createKafkaConsumer(String name, final MessageChannel moduleInputChannel, + ExtendedConsumerProperties properties, String group, long referencePoint) { + + validateTopicName(name); + if (properties.getInstanceCount() == 0) { + throw new IllegalArgumentException("Instance count cannot be zero"); + } + Collection allPartitions = ensureTopicCreated(name, + properties.getInstanceCount() * properties.getConcurrency()); + + Decoder valueDecoder = new DefaultDecoder(null); + Decoder keyDecoder = new DefaultDecoder(null); + + Collection listenedPartitions; + + if (properties.getInstanceCount() == 1) { + listenedPartitions = allPartitions; + } + else { + listenedPartitions = new ArrayList<>(); + for (Partition partition : allPartitions) { + // divide partitions across modules + if ((partition.getId() % properties.getInstanceCount()) == properties.getInstanceIndex()) { + listenedPartitions.add(partition); + } + } + } + topicsInUse.put(name, listenedPartitions); + ReceivingHandler rh = new ReceivingHandler(properties); + rh.setOutputChannel(moduleInputChannel); + + final FixedSubscriberChannel bridge = new FixedSubscriberChannel(rh); + bridge.setBeanName("bridge." + name); + + Assert.isTrue(!CollectionUtils.isEmpty(listenedPartitions), "A list of partitions must be provided"); + final KafkaMessageListenerContainer messageListenerContainer = new KafkaMessageListenerContainer( + connectionFactory, listenedPartitions.toArray(new Partition[listenedPartitions.size()])); + + if (logger.isDebugEnabled()) { + logger.debug("Listened partitions: " + StringUtils.collectionToCommaDelimitedString(listenedPartitions)); + } + + OffsetManager offsetManager = createOffsetManager(group, referencePoint); + if (properties.getExtension().isResetOffsets()) { + offsetManager.resetOffsets(listenedPartitions); + } + messageListenerContainer.setOffsetManager(offsetManager); + messageListenerContainer.setQueueSize(configurationProperties.getQueueSize()); + messageListenerContainer.setMaxFetch(configurationProperties.getFetchSize()); + boolean autoCommitOnError = properties.getExtension().getAutoCommitOnError() != null + ? properties.getExtension().getAutoCommitOnError() + : properties.getExtension().isAutoCommitOffset() && properties.getExtension().isEnableDlq(); + messageListenerContainer.setAutoCommitOnError(autoCommitOnError); + messageListenerContainer.setRecoveryInterval(properties.getExtension().getRecoveryInterval()); + + int concurrency = Math.min(properties.getConcurrency(), listenedPartitions.size()); + messageListenerContainer.setConcurrency(concurrency); + final ExecutorService dispatcherTaskExecutor = Executors.newFixedThreadPool(concurrency, DAEMON_THREAD_FACTORY); + messageListenerContainer.setDispatcherTaskExecutor(dispatcherTaskExecutor); + final KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = new KafkaMessageDrivenChannelAdapter( + messageListenerContainer); + kafkaMessageDrivenChannelAdapter.setBeanFactory(this.getBeanFactory()); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(keyDecoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(valueDecoder); + kafkaMessageDrivenChannelAdapter.setOutputChannel(bridge); + kafkaMessageDrivenChannelAdapter.setAutoCommitOffset(properties.getExtension().isAutoCommitOffset()); + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + + // we need to wrap the adapter listener into a retrying listener so that the retry + // logic is applied before the ErrorHandler is executed + final RetryTemplate retryTemplate = buildRetryTemplateIfRetryEnabled(properties); + if (retryTemplate != null) { + if (properties.getExtension().isAutoCommitOffset()) { + final MessageListener originalMessageListener = (MessageListener) messageListenerContainer + .getMessageListener(); + messageListenerContainer.setMessageListener(new MessageListener() { + @Override + public void onMessage(final KafkaMessage message) { + try { + retryTemplate.execute(new RetryCallback() { + @Override + public Object doWithRetry(RetryContext context) { + originalMessageListener.onMessage(message); + return null; + } + }); + } + catch (Throwable throwable) { + if (throwable instanceof RuntimeException) { + throw (RuntimeException) throwable; + } + else { + throw new RuntimeException(throwable); + } + } + } + }); + } + else { + messageListenerContainer.setMessageListener(new AcknowledgingMessageListener() { + final AcknowledgingMessageListener originalMessageListener = (AcknowledgingMessageListener) messageListenerContainer + .getMessageListener(); + + @Override + public void onMessage(final KafkaMessage message, final Acknowledgment acknowledgment) { + retryTemplate.execute(new RetryCallback() { + @Override + public Object doWithRetry(RetryContext context) { + originalMessageListener.onMessage(message, acknowledgment); + return null; + } + }); + } + }); + } + } + + if (properties.getExtension().isEnableDlq()) { + final String dlqTopic = "error." + name + "." + group; + initDlqProducer(); + messageListenerContainer.setErrorHandler(new ErrorHandler() { + @Override + public void handle(Exception thrownException, final KafkaMessage message) { + final byte[] key = message.getMessage().key() != null ? Utils.toArray(message.getMessage().key()) + : null; + final byte[] payload = message.getMessage().payload() != null + ? Utils.toArray(message.getMessage().payload()) : null; + dlqProducer.send(new ProducerRecord<>(dlqTopic, key, payload), new Callback() { + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + StringBuffer messageLog = new StringBuffer(); + messageLog.append(" a message with key='" + + toDisplayString(ObjectUtils.nullSafeToString(key), 50) + "'"); + messageLog.append(" and payload='" + + toDisplayString(ObjectUtils.nullSafeToString(payload), 50) + "'"); + messageLog.append(" received from " + message.getMetadata().getPartition()); + if (exception != null) { + logger.error("Error sending to DLQ" + messageLog.toString(), exception); + } + else { + if (logger.isDebugEnabled()) { + logger.debug("Sent to DLQ " + messageLog.toString()); + } + } + } + }); + } + }); + } + kafkaMessageDrivenChannelAdapter.start(); + + EventDrivenConsumer edc = new EventDrivenConsumer(bridge, rh) { + + @Override + protected void doStop() { + // stop the offset manager and the channel adapter before unbinding + // this means that the upstream channel adapter has a chance to stop + kafkaMessageDrivenChannelAdapter.stop(); + if (messageListenerContainer.getOffsetManager() instanceof DisposableBean) { + try { + ((DisposableBean) messageListenerContainer.getOffsetManager()).destroy(); + } + catch (Exception e) { + logger.error("Error while closing the offset manager", e); + } + } + super.doStop(); + } + }; + String groupedName = groupedName(name, group); + edc.setBeanName("inbound." + groupedName); + + DefaultBinding consumerBinding = new DefaultBinding(name, group, + moduleInputChannel, edc) { + @Override + protected void afterUnbind() { + dispatcherTaskExecutor.shutdown(); + } + }; + edc.start(); + return consumerBinding; + } + + private synchronized void initDlqProducer() { + try { + if (dlqProducer == null) { + synchronized (this) { + if (dlqProducer == null) { + // we can use the producer defaults as we do not need to tune + // performance + ProducerMetadata producerMetadata = new ProducerMetadata<>("dlqKafkaProducer", + byte[].class, byte[].class, BYTE_ARRAY_SERIALIZER, BYTE_ARRAY_SERIALIZER); + producerMetadata.setSync(false); + producerMetadata.setCompressionType(ProducerMetadata.CompressionType.none); + producerMetadata.setBatchBytes(16384); + Properties additionalProps = new Properties(); + additionalProps.put(ProducerConfig.ACKS_CONFIG, + String.valueOf(configurationProperties.getRequiredAcks())); + additionalProps.put(ProducerConfig.LINGER_MS_CONFIG, String.valueOf(0)); + ProducerFactoryBean producerFactoryBean = new ProducerFactoryBean<>( + producerMetadata, configurationProperties.getKafkaConnectionString(), additionalProps); + dlqProducer = producerFactoryBean.getObject(); + } + } + } + } + catch (Exception e) { + throw new RuntimeException("Cannot initialize DLQ producer:", e); + } + } + + private OffsetManager createOffsetManager(String group, long referencePoint) { + try { + + KafkaNativeOffsetManager kafkaOffsetManager = new KafkaNativeOffsetManager(connectionFactory, + new ZookeeperConnect(configurationProperties.getZkConnectionString()), + Collections.emptyMap()); + kafkaOffsetManager.setConsumerId(group); + kafkaOffsetManager.setReferenceTimestamp(referencePoint); + kafkaOffsetManager.afterPropertiesSet(); + + WindowingOffsetManager windowingOffsetManager = new WindowingOffsetManager(kafkaOffsetManager); + windowingOffsetManager.setTimespan(configurationProperties.getOffsetUpdateTimeWindow()); + windowingOffsetManager.setCount(configurationProperties.getOffsetUpdateCount()); + windowingOffsetManager.setShutdownTimeout(configurationProperties.getOffsetUpdateShutdownTimeout()); + + windowingOffsetManager.afterPropertiesSet(); + return windowingOffsetManager; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String toDisplayString(String original, int maxCharacters) { + if (original.length() <= maxCharacters) { + return original; + } + return original.substring(0, maxCharacters) + "..."; + } + + @Override + public void doManualAck(LinkedList messageHeadersList) { + Iterator iterator = messageHeadersList.iterator(); + while (iterator.hasNext()) { + MessageHeaders headers = iterator.next(); + Acknowledgment acknowledgment = (Acknowledgment) headers.get(KafkaHeaders.ACKNOWLEDGMENT); + Assert.notNull(acknowledgment, + "Acknowledgement shouldn't be null when acknowledging kafka message " + "manually."); + acknowledgment.acknowledge(); + } + } + + private final class ReceivingHandler extends AbstractReplyProducingMessageHandler { + + private final ExtendedConsumerProperties consumerProperties; + + private ReceivingHandler(ExtendedConsumerProperties consumerProperties) { + this.consumerProperties = consumerProperties; + } + + @Override + protected Object handleRequestMessage(Message requestMessage) { + if (HeaderMode.embeddedHeaders.equals(consumerProperties.getHeaderMode())) { + MessageValues messageValues = extractMessageValues(requestMessage); + return MessageBuilder.createMessage(messageValues.getPayload(), new KafkaBinderHeaders(messageValues)); + } + else { + return requestMessage; + } + } + + @Override + protected boolean shouldCopyRequestHeaders() { + // prevent the message from being copied again in superclass + return false; + } + + @SuppressWarnings("serial") + private final class KafkaBinderHeaders extends MessageHeaders { + KafkaBinderHeaders(Map headers) { + super(headers, MessageHeaders.ID_VALUE_NONE, -1L); + } + } + } + + private final class SendingHandler extends AbstractMessageHandler { + + private final AtomicInteger roundRobinCount = new AtomicInteger(); + + private final String topicName; + + private final ExtendedProducerProperties producerProperties; + + private final int numberOfKafkaPartitions; + + private final ProducerConfiguration producerConfiguration; + + private final PartitionHandler partitionHandler; + + private SendingHandler(String topicName, ExtendedProducerProperties properties, + int numberOfPartitions, ProducerConfiguration producerConfiguration) { + this.topicName = topicName; + producerProperties = properties; + this.numberOfKafkaPartitions = numberOfPartitions; + ConfigurableListableBeanFactory beanFactory = KafkaMessageChannelBinder.this.getBeanFactory(); + this.setBeanFactory(beanFactory); + this.producerConfiguration = producerConfiguration; + this.partitionHandler = new PartitionHandler(beanFactory, evaluationContext, partitionSelector, properties); + } + + @Override + protected void handleMessageInternal(Message message) throws Exception { + int targetPartition; + if (producerProperties.isPartitioned()) { + targetPartition = this.partitionHandler.determinePartition(message); + } + else { + targetPartition = roundRobin() % numberOfKafkaPartitions; + } + if (HeaderMode.embeddedHeaders.equals(producerProperties.getHeaderMode())) { + MessageValues transformed = serializePayloadIfNecessary(message); + byte[] messageToSend = embeddedHeadersMessageConverter.embedHeaders(transformed, + KafkaMessageChannelBinder.this.headersToMap); + producerConfiguration.send(topicName, targetPartition, null, messageToSend); + } + else if (HeaderMode.raw.equals(producerProperties.getHeaderMode())) { + Object contentType = message.getHeaders().get(MessageHeaders.CONTENT_TYPE); + if (contentType != null && !contentType.equals(MediaType.APPLICATION_OCTET_STREAM_VALUE)) { + logger.error("Raw mode supports only " + MediaType.APPLICATION_OCTET_STREAM_VALUE + " content type" + + message.getPayload().getClass()); + } + if (message.getPayload() instanceof byte[]) { + producerConfiguration.send(topicName, targetPartition, null, (byte[]) message.getPayload()); + } + else { + logger.error("Raw mode supports only byte[] payloads but value sent was of type " + + message.getPayload().getClass()); + } + } + } + + private int roundRobin() { + int result = roundRobinCount.incrementAndGet(); + if (result == Integer.MAX_VALUE) { + roundRobinCount.set(0); + } + return result; + } + + } + + public enum StartOffset { + earliest(OffsetRequest.EarliestTime()), latest(OffsetRequest.LatestTime()); + + private final long referencePoint; + + StartOffset(long referencePoint) { + this.referencePoint = referencePoint; + } + + public long getReferencePoint() { + return referencePoint; + } + } + +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaProducerProperties.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaProducerProperties.java new file mode 100644 index 000000000..99f2486a1 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaProducerProperties.java @@ -0,0 +1,68 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import javax.validation.constraints.NotNull; + +import org.springframework.integration.kafka.support.ProducerMetadata; + +/** + * @author Marius Bogoevici + */ +public class KafkaProducerProperties { + + private int bufferSize = 16384; + + private ProducerMetadata.CompressionType compressionType = ProducerMetadata.CompressionType.none; + + private boolean sync; + + private int batchTimeout; + + public int getBufferSize() { + return bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + @NotNull + public ProducerMetadata.CompressionType getCompressionType() { + return compressionType; + } + + public void setCompressionType(ProducerMetadata.CompressionType compressionType) { + this.compressionType = compressionType; + } + + public boolean isSync() { + return sync; + } + + public void setSync(boolean sync) { + this.sync = sync; + } + + public int getBatchTimeout() { + return batchTimeout; + } + + public void setBatchTimeout(int batchTimeout) { + this.batchTimeout = batchTimeout; + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/WindowingOffsetManager.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/WindowingOffsetManager.java new file mode 100644 index 000000000..44fa9792d --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/WindowingOffsetManager.java @@ -0,0 +1,262 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import rx.Observable; +import rx.Subscription; +import rx.functions.Action0; +import rx.functions.Action1; +import rx.functions.Func1; +import rx.functions.Func2; +import rx.observables.GroupedObservable; +import rx.observables.MathObservable; +import rx.subjects.PublishSubject; +import rx.subjects.SerializedSubject; +import rx.subjects.Subject; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.listener.OffsetManager; +import org.springframework.util.Assert; + +/** + * An {@link OffsetManager} that aggregates writes over a time or count window, using an underlying delegate to + * do the actual operations. Its purpose is to reduce the performance impact of writing operations + * wherever this is desirable. + * + * Either a time window or a number of writes can be specified, but not both. + * + * @author Marius Bogoevici + */ +public class WindowingOffsetManager implements OffsetManager, InitializingBean, DisposableBean { + + private final CreatePartitionAndOffsetFunction createPartitionAndOffsetFunction = new CreatePartitionAndOffsetFunction(); + + private final GetOffsetFunction getOffsetFunction = new GetOffsetFunction(); + + private final ComputeMaximumOffsetByPartitionFunction findHighestOffsetInPartitionGroup = new ComputeMaximumOffsetByPartitionFunction(); + + private final GetPartitionFunction getPartition = new GetPartitionFunction(); + + private final FindHighestOffsetsByPartitionFunction findHighestOffsetsByPartition = new FindHighestOffsetsByPartitionFunction(); + + private final DelegateUpdateOffsetAction delegateUpdateOffsetAction = new DelegateUpdateOffsetAction(); + + private final NotifyObservableClosedAction notifyObservableClosed = new NotifyObservableClosedAction(); + + private final OffsetManager delegate; + + private long timespan = 10 * 1000; + + private int count; + + private Subject offsets; + + private Subscription subscription; + + private int shutdownTimeout = 2000; + + private CountDownLatch shutdownLatch; + + public WindowingOffsetManager(OffsetManager offsetManager) { + this.delegate = offsetManager; + } + + /** + * The timespan for aggregating write operations, before invoking the underlying {@link OffsetManager}. + * + * @param timespan duration in milliseconds + */ + public void setTimespan(long timespan) { + Assert.isTrue(timespan >= 0, "Timespan must be a positive value"); + this.timespan = timespan; + } + + /** + * How many writes should be aggregated, before invoking the underlying {@link OffsetManager}. Setting this value + * to 1 effectively disables windowing. + * + * @param count number of writes + */ + public void setCount(int count) { + Assert.isTrue(count >= 0, "Count must be a positive value"); + this.count = count; + } + + /** + * The timeout that {@link #close()} and {@link #destroy()} operations will wait for receving a confirmation that the + * underlying writes have been processed. + * + * @param shutdownTimeout duration in milliseconds + */ + public void setShutdownTimeout(int shutdownTimeout) { + this.shutdownTimeout = shutdownTimeout; + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.isTrue(timespan > 0 ^ count > 0, "Only one of the timespan or count must be set"); + // create the stream if windowing is set, and count is higher than 1 + if (timespan > 0 || count > 1) { + offsets = new SerializedSubject<>(PublishSubject.create()); + // window by either count or time + Observable> window = + timespan > 0 ? offsets.window(timespan, TimeUnit.MILLISECONDS) : offsets.window(count); + Observable maximumOffsetsByWindow = window + .flatMap(findHighestOffsetsByPartition) + .doOnCompleted(notifyObservableClosed); + subscription = maximumOffsetsByWindow.subscribe(delegateUpdateOffsetAction); + } + else { + offsets = null; + } + } + + @Override + public void destroy() throws Exception { + this.flush(); + this.close(); + if (delegate instanceof DisposableBean) { + ((DisposableBean) delegate).destroy(); + } + } + + @Override + public void updateOffset(Partition partition, long offset) { + if (offsets != null) { + offsets.onNext(new PartitionAndOffset(partition, offset)); + } + else { + delegate.updateOffset(partition, offset); + } + } + + @Override + public long getOffset(Partition partition) { + return delegate.getOffset(partition); + } + + @Override + public void deleteOffset(Partition partition) { + delegate.deleteOffset(partition); + } + + @Override + public void resetOffsets(Collection partition) { + delegate.resetOffsets(partition); + } + + @Override + public void close() throws IOException { + if (offsets != null) { + shutdownLatch = new CountDownLatch(1); + offsets.onCompleted(); + try { + shutdownLatch.await(shutdownTimeout, TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + // ignore + } + subscription.unsubscribe(); + } + delegate.close(); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + private final class PartitionAndOffset { + + private final Partition partition; + + private final Long offset; + + private PartitionAndOffset(Partition partition, Long offset) { + this.partition = partition; + this.offset = offset; + } + + public Partition getPartition() { + return partition; + } + + public Long getOffset() { + return offset; + } + } + + private class DelegateUpdateOffsetAction implements Action1 { + @Override + public void call(PartitionAndOffset partitionAndOffset) { + delegate.updateOffset(partitionAndOffset.getPartition(), partitionAndOffset.getOffset()); + } + } + + private class NotifyObservableClosedAction implements Action0 { + @Override + public void call() { + if (shutdownLatch != null) { + shutdownLatch.countDown(); + } + } + } + + private class CreatePartitionAndOffsetFunction implements Func2 { + @Override + public PartitionAndOffset call(Partition partition, Long offset) { + return new PartitionAndOffset(partition, offset); + } + } + + private class GetOffsetFunction implements Func1 { + @Override + public Long call(PartitionAndOffset partitionAndOffset) { + return partitionAndOffset.getOffset(); + } + } + + private class ComputeMaximumOffsetByPartitionFunction implements Func1, Observable> { + @Override + public Observable call(GroupedObservable group) { + return Observable.zip(Observable.just(group.getKey()), + MathObservable.max(group.map(getOffsetFunction)), + createPartitionAndOffsetFunction); + } + } + + private class GetPartitionFunction implements Func1 { + @Override + public Partition call(PartitionAndOffset partitionAndOffset) { + return partitionAndOffset.getPartition(); + } + } + + private class FindHighestOffsetsByPartitionFunction implements Func1, Observable> { + @Override + public Observable call(Observable windowBuffer) { + return windowBuffer.groupBy(getPartition).flatMap(findHighestOffsetInPartitionGroup); + } + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java new file mode 100644 index 000000000..623bbf35d --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfiguration.java @@ -0,0 +1,79 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.PropertyPlaceholderAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.stream.binder.Binder; +import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; +import org.springframework.cloud.stream.binder.kafka.KafkaExtendedBindingProperties; +import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; +import org.springframework.cloud.stream.config.codec.kryo.KryoCodecAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.integration.codec.Codec; +import org.springframework.integration.kafka.support.LoggingProducerListener; +import org.springframework.integration.kafka.support.ProducerListener; + +/** + * @author David Turanski + * @author Marius Bogoevici + * @author Soby Chacko + * @author Mark Fisher + * @author Ilayaperumal Gopinathan + */ +@Configuration +@ConditionalOnMissingBean(Binder.class) +@Import({ KryoCodecAutoConfiguration.class, PropertyPlaceholderAutoConfiguration.class }) +@EnableConfigurationProperties({ KafkaBinderConfigurationProperties.class, KafkaExtendedBindingProperties.class }) +public class KafkaBinderConfiguration { + + @Autowired + private Codec codec; + + @Autowired + private KafkaBinderConfigurationProperties configurationProperties; + + @Autowired + private KafkaExtendedBindingProperties kafkaExtendedBindingProperties; + + @Autowired + private ProducerListener producerListener; + + @Bean + KafkaMessageChannelBinder kafkaMessageChannelBinder() { + KafkaMessageChannelBinder kafkaMessageChannelBinder = new KafkaMessageChannelBinder(configurationProperties); + kafkaMessageChannelBinder.setCodec(codec); + kafkaMessageChannelBinder.setProducerListener(producerListener); + kafkaMessageChannelBinder.setExtendedBindingProperties(kafkaExtendedBindingProperties); + return kafkaMessageChannelBinder; + } + + @Bean + @ConditionalOnMissingBean(ProducerListener.class) + ProducerListener producerListener() { + return new LoggingProducerListener(); + } + + @Bean + KafkaBinderHealthIndicator healthIndicator(KafkaMessageChannelBinder kafkaMessageChannelBinder) { + return new KafkaBinderHealthIndicator(kafkaMessageChannelBinder, configurationProperties); + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfigurationProperties.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfigurationProperties.java new file mode 100644 index 000000000..98ee963f2 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderConfigurationProperties.java @@ -0,0 +1,243 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; + +/** + * @author David Turanski + * @author Ilayaperumal Gopinathan + * @author Marius Bogoevici + */ +@ConfigurationProperties(prefix = "spring.cloud.stream.kafka.binder") +public class KafkaBinderConfigurationProperties { + + private String[] zkNodes = new String[] { "localhost" }; + + private String defaultZkPort = "2181"; + + private String[] brokers = new String[] { "localhost" }; + + private String defaultBrokerPort = "9092"; + + private String[] headers = new String[] {}; + + private int offsetUpdateTimeWindow = 10000; + + private int offsetUpdateCount; + + private int offsetUpdateShutdownTimeout = 2000; + + private int maxWait = 100; + + private boolean autoCreateTopics = true; + + private boolean autoAddPartitions; + + private int socketBufferSize = 2097152; + + /** + * ZK session timeout in milliseconds. + */ + private int zkSessionTimeout = 10000; + + /** + * ZK Connection timeout in milliseconds. + */ + private int zkConnectionTimeout = 10000; + + private int requiredAcks = 1; + + private int replicationFactor = 1; + + private int fetchSize = 1024 * 1024; + + private int minPartitionCount = 1; + + private int queueSize = 8192; + + public String getZkConnectionString() { + return toConnectionString(this.zkNodes, this.defaultZkPort); + } + + public String getKafkaConnectionString() { + return toConnectionString(this.brokers, this.defaultBrokerPort); + } + + public String[] getHeaders() { + return headers; + } + + public int getOffsetUpdateTimeWindow() { + return this.offsetUpdateTimeWindow; + } + + public int getOffsetUpdateCount() { + return this.offsetUpdateCount; + } + + public int getOffsetUpdateShutdownTimeout() { + return this.offsetUpdateShutdownTimeout; + } + + public String[] getZkNodes() { + return zkNodes; + } + + public void setZkNodes(String... zkNodes) { + this.zkNodes = zkNodes; + } + + public void setDefaultZkPort(String defaultZkPort) { + this.defaultZkPort = defaultZkPort; + } + + public String[] getBrokers() { + return brokers; + } + + public void setBrokers(String... brokers) { + this.brokers = brokers; + } + + public void setDefaultBrokerPort(String defaultBrokerPort) { + this.defaultBrokerPort = defaultBrokerPort; + } + + public void setHeaders(String... headers) { + this.headers = headers; + } + + public void setOffsetUpdateTimeWindow(int offsetUpdateTimeWindow) { + this.offsetUpdateTimeWindow = offsetUpdateTimeWindow; + } + + public void setOffsetUpdateCount(int offsetUpdateCount) { + this.offsetUpdateCount = offsetUpdateCount; + } + + public void setOffsetUpdateShutdownTimeout(int offsetUpdateShutdownTimeout) { + this.offsetUpdateShutdownTimeout = offsetUpdateShutdownTimeout; + } + + public int getZkSessionTimeout() { + return this.zkSessionTimeout; + } + + public void setZkSessionTimeout(int zkSessionTimeout) { + this.zkSessionTimeout = zkSessionTimeout; + } + + public int getZkConnectionTimeout() { + return this.zkConnectionTimeout; + } + + public void setZkConnectionTimeout(int zkConnectionTimeout) { + this.zkConnectionTimeout = zkConnectionTimeout; + } + + /** + * Converts an array of host values to a comma-separated String. + * + * It will append the default port value, if not already specified. + */ + private String toConnectionString(String[] hosts, String defaultPort) { + String[] fullyFormattedHosts = new String[hosts.length]; + for (int i = 0; i < hosts.length; i++) { + if (hosts[i].contains(":") || StringUtils.isEmpty(defaultPort)) { + fullyFormattedHosts[i] = hosts[i]; + } + else { + fullyFormattedHosts[i] = hosts[i] + ":" + defaultPort; + } + } + return StringUtils.arrayToCommaDelimitedString(fullyFormattedHosts); + } + + public int getMaxWait() { + return maxWait; + } + + public void setMaxWait(int maxWait) { + this.maxWait = maxWait; + } + + public int getRequiredAcks() { + return requiredAcks; + } + + public void setRequiredAcks(int requiredAcks) { + this.requiredAcks = requiredAcks; + } + + public int getReplicationFactor() { + return replicationFactor; + } + + public void setReplicationFactor(int replicationFactor) { + this.replicationFactor = replicationFactor; + } + + public int getFetchSize() { + return fetchSize; + } + + public void setFetchSize(int fetchSize) { + this.fetchSize = fetchSize; + } + + public int getMinPartitionCount() { + return minPartitionCount; + } + + public void setMinPartitionCount(int minPartitionCount) { + this.minPartitionCount = minPartitionCount; + } + + public int getQueueSize() { + return queueSize; + } + + public void setQueueSize(int queueSize) { + this.queueSize = queueSize; + } + + public boolean isAutoCreateTopics() { + return autoCreateTopics; + } + + public void setAutoCreateTopics(boolean autoCreateTopics) { + this.autoCreateTopics = autoCreateTopics; + } + + public boolean isAutoAddPartitions() { + return autoAddPartitions; + } + + public void setAutoAddPartitions(boolean autoAddPartitions) { + this.autoAddPartitions = autoAddPartitions; + } + + public int getSocketBufferSize() { + return socketBufferSize; + } + + public void setSocketBufferSize(int socketBufferSize) { + this.socketBufferSize = socketBufferSize; + } +} diff --git a/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/package-info.java b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/package-info.java new file mode 100644 index 000000000..a95c7dea1 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains an implementation of the {@link org.springframework.cloud.stream.binder.Binder} for Kafka. + */ +package org.springframework.cloud.stream.binder.kafka; diff --git a/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders b/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders new file mode 100644 index 000000000..063a7400f --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders @@ -0,0 +1,2 @@ +kafka:\ +org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java new file mode 100644 index 000000000..08ca966f1 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderTests.java @@ -0,0 +1,808 @@ +/* + * Copyright 2014-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import kafka.admin.AdminUtils; +import kafka.api.TopicMetadata; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.cloud.stream.binder.BinderException; +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.PartitionCapableBinderTests; +import org.springframework.cloud.stream.binder.Spy; +import org.springframework.cloud.stream.binder.TestUtils; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.test.junit.kafka.KafkaTestSupport; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.TopicNotFoundException; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.integration.kafka.support.ProducerConfiguration; +import org.springframework.integration.kafka.support.ProducerMetadata; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.retry.backoff.FixedBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +/** + * Integration tests for the {@link KafkaMessageChannelBinder}. + * @author Eric Bottard + * @author Marius Bogoevici + * @author Mark Fisher + * @author Ilayaperumal Gopinathan + */ +public class KafkaBinderTests extends + PartitionCapableBinderTests, ExtendedProducerProperties> { + + private final String CLASS_UNDER_TEST_NAME = KafkaMessageChannelBinder.class.getSimpleName(); + + @ClassRule + public static KafkaTestSupport kafkaTestSupport = new KafkaTestSupport(); + + private KafkaTestBinder binder; + + @Override + protected void binderBindUnbindLatency() throws InterruptedException { + Thread.sleep(500); + } + + @Override + protected KafkaTestBinder getBinder() { + if (binder == null) { + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binder = new KafkaTestBinder(binderConfiguration); + } + return binder; + } + + private KafkaBinderConfigurationProperties createConfigurationProperties() { + KafkaBinderConfigurationProperties binderConfiguration = new KafkaBinderConfigurationProperties(); + binderConfiguration.setBrokers(kafkaTestSupport.getBrokerAddress()); + binderConfiguration.setZkNodes(kafkaTestSupport.getZkConnectString()); + return binderConfiguration; + } + + @Override + protected ExtendedConsumerProperties createConsumerProperties() { + return new ExtendedConsumerProperties<>(new KafkaConsumerProperties()); + } + + @Override + protected ExtendedProducerProperties createProducerProperties() { + return new ExtendedProducerProperties<>(new KafkaProducerProperties()); + } + + @Before + public void init() { + String multiplier = System.getenv("KAFKA_TIMEOUT_MULTIPLIER"); + if (multiplier != null) { + timeoutMultiplier = Double.parseDouble(multiplier); + } + } + + @Override + protected boolean usesExplicitRouting() { + return false; + } + + @Override + protected String getClassUnderTestName() { + return CLASS_UNDER_TEST_NAME; + } + + @Override + public Spy spyOn(final String name) { + throw new UnsupportedOperationException("'spyOn' is not used by Kafka tests"); + } + + @Test + public void testDlqAndRetry() { + KafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = new DirectChannel(); + DirectChannel moduleInputChannel = new DirectChannel(); + QueueChannel dlqChannel = new QueueChannel(); + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(10); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(3); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + consumerProperties.getExtension().setEnableDlq(true); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer("retryTest." + uniqueBindingId + ".0", + moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", + "testGroup", moduleInputChannel, consumerProperties); + + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + + Binding dlqConsumerBinding = binder.bindConsumer( + "error.retryTest." + uniqueBindingId + ".0.testGroup", null, dlqChannel, dlqConsumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder.withPayload(testMessagePayload).build(); + moduleOutputChannel.send(testMessage); + + Message receivedMessage = receive(dlqChannel, 3); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getPayload()).isEqualTo(testMessagePayload); + assertThat(handler.getInvocationCount()).isEqualTo(consumerProperties.getMaxAttempts()); + dlqConsumerBinding.unbind(); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + public void testDefaultAutoCommitOnErrorWithoutDlq() throws Exception { + KafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = new DirectChannel(); + DirectChannel moduleInputChannel = new DirectChannel(); + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(10); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(1); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer("retryTest." + uniqueBindingId + ".0", + moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", + "testGroup", moduleInputChannel, consumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder.withPayload(testMessagePayload).build(); + moduleOutputChannel.send(testMessage); + + assertThat(handler.getLatch().await((int) (timeoutMultiplier * 1000), TimeUnit.MILLISECONDS)); + // first attempt fails + assertThat(handler.getReceivedMessages().entrySet()).hasSize(1); + Message receivedMessage = handler.getReceivedMessages().entrySet().iterator().next().getValue(); + assertThat(receivedMessage).isNotNull(); + assertThat(receivedMessage.getPayload()).isEqualTo(testMessagePayload); + assertThat(handler.getInvocationCount()).isEqualTo(consumerProperties.getMaxAttempts()); + consumerBinding.unbind(); + + // on the second attempt the message is redelivered + QueueChannel successfulInputChannel = new QueueChannel(); + consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", "testGroup", + successfulInputChannel, consumerProperties); + String testMessage2Payload = "test." + UUID.randomUUID().toString(); + Message testMessage2 = MessageBuilder.withPayload(testMessage2Payload).build(); + moduleOutputChannel.send(testMessage2); + + Message firstReceived = receive(successfulInputChannel); + assertThat(firstReceived.getPayload()).isEqualTo(testMessagePayload); + Message secondReceived = receive(successfulInputChannel); + assertThat(secondReceived.getPayload()).isEqualTo(testMessage2Payload); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + public void testDefaultAutoCommitOnErrorWithDlq() throws Exception { + KafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = new DirectChannel(); + DirectChannel moduleInputChannel = new DirectChannel(); + FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler(); + moduleInputChannel.subscribe(handler); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(10); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setMaxAttempts(3); + consumerProperties.setBackOffInitialInterval(100); + consumerProperties.setBackOffMaxInterval(150); + consumerProperties.getExtension().setEnableDlq(true); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer("retryTest." + uniqueBindingId + ".0", + moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", + "testGroup", moduleInputChannel, consumerProperties); + ExtendedConsumerProperties dlqConsumerProperties = createConsumerProperties(); + dlqConsumerProperties.setMaxAttempts(1); + QueueChannel dlqChannel = new QueueChannel(); + Binding dlqConsumerBinding = binder.bindConsumer( + "error.retryTest." + uniqueBindingId + ".0.testGroup", null, dlqChannel, dlqConsumerProperties); + + String testMessagePayload = "test." + UUID.randomUUID().toString(); + Message testMessage = MessageBuilder.withPayload(testMessagePayload).build(); + moduleOutputChannel.send(testMessage); + + Message dlqMessage = receive(dlqChannel, 3); + assertThat(dlqMessage).isNotNull(); + assertThat(dlqMessage.getPayload()).isEqualTo(testMessagePayload); + + // first attempt fails + assertThat(handler.getReceivedMessages().entrySet()).hasSize(1); + Message handledMessage = handler.getReceivedMessages().entrySet().iterator().next().getValue(); + assertThat(handledMessage).isNotNull(); + assertThat(handledMessage.getPayload()).isEqualTo(testMessagePayload); + assertThat(handler.getInvocationCount()).isEqualTo(consumerProperties.getMaxAttempts()); + + dlqConsumerBinding.unbind(); + consumerBinding.unbind(); + + // on the second attempt the message is not redelivered because the DLQ is set + QueueChannel successfulInputChannel = new QueueChannel(); + consumerBinding = binder.bindConsumer("retryTest." + uniqueBindingId + ".0", "testGroup", + successfulInputChannel, consumerProperties); + String testMessage2Payload = "test." + UUID.randomUUID().toString(); + Message testMessage2 = MessageBuilder.withPayload(testMessage2Payload).build(); + moduleOutputChannel.send(testMessage2); + + Message receivedMessage = receive(successfulInputChannel); + assertThat(receivedMessage.getPayload()).isEqualTo(testMessage2Payload); + + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test(expected = IllegalArgumentException.class) + public void testValidateKafkaTopicName() { + KafkaMessageChannelBinder.validateTopicName("foo:bar"); + } + + @Test + public void testCompression() throws Exception { + final ProducerMetadata.CompressionType[] codecs = new ProducerMetadata.CompressionType[] { + ProducerMetadata.CompressionType.none, ProducerMetadata.CompressionType.gzip, + ProducerMetadata.CompressionType.snappy }; + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaTestBinder binder = getBinder(); + for (ProducerMetadata.CompressionType codec : codecs) { + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.getExtension().setCompressionType(codec); + Binding producerBinding = binder.bindProducer("foo.0", moduleOutputChannel, + producerProperties); + Binding consumerBinding = binder.bindConsumer("foo.0", "test", moduleInputChannel, + createConsumerProperties()); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload) + .build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + producerBinding.unbind(); + consumerBinding.unbind(); + } + } + + @Test + public void testCustomPartitionCountOverridesDefaultIfLarger() throws Exception { + + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setMinPartitionCount(10); + KafkaTestBinder binder = new KafkaTestBinder(binderConfiguration); + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(10); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer("foo" + uniqueBindingId + ".0", + moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer("foo" + uniqueBindingId + ".0", null, + moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload) + .build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + Collection partitions = binder.getCoreBinder().getConnectionFactory() + .getPartitions("foo" + uniqueBindingId + ".0"); + assertThat(partitions).hasSize(10); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + public void testCustomPartitionCountDoesNotOverridePartitioningIfSmaller() throws Exception { + + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setMinPartitionCount(6); + KafkaTestBinder binder = new KafkaTestBinder(binderConfiguration); + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(5); + producerProperties.setPartitionKeyExpression(spelExpressionParser.parseExpression("payload")); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer("foo" + uniqueBindingId + ".0", + moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer("foo" + uniqueBindingId + ".0", null, + moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload) + .build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + Collection partitions = binder.getCoreBinder().getConnectionFactory() + .getPartitions("foo" + uniqueBindingId + ".0"); + assertThat(partitions).hasSize(6); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + public void testCustomPartitionCountOverridesPartitioningIfLarger() throws Exception { + + byte[] testPayload = new byte[2048]; + Arrays.fill(testPayload, (byte) 65); + KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties(); + binderConfiguration.setMinPartitionCount(4); + KafkaTestBinder binder = new KafkaTestBinder(binderConfiguration); + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setPartitionCount(5); + producerProperties.setPartitionKeyExpression(spelExpressionParser.parseExpression("payload")); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + long uniqueBindingId = System.currentTimeMillis(); + Binding producerBinding = binder.bindProducer("foo" + uniqueBindingId + ".0", + moduleOutputChannel, producerProperties); + Binding consumerBinding = binder.bindConsumer("foo" + uniqueBindingId + ".0", null, + moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload) + .build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload); + Collection partitions = binder.getCoreBinder().getConnectionFactory() + .getPartitions("foo" + uniqueBindingId + ".0"); + assertThat(partitions).hasSize(5); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testDefaultConsumerStartsAtEarliest() throws Exception { + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(createConfigurationProperties()); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + DirectChannel output = new DirectChannel(); + QueueChannel input1 = new QueueChannel(); + + String testTopicName = UUID.randomUUID().toString(); + binder.bindProducer(testTopicName, output, createProducerProperties()); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + binder.bindConsumer(testTopicName, "startOffsets", input1, createConsumerProperties()); + Message receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + assertThat(new String(receivedMessage1.getPayload())).isEqualTo(testPayload1); + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload2.getBytes())); + Message receivedMessage2 = (Message) receive(input1); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload2); + } + + @Test + @SuppressWarnings("unchecked") + public void testEarliest() throws Exception { + KafkaTestBinder binder = getBinder(); + DirectChannel output = new DirectChannel(); + QueueChannel input1 = new QueueChannel(); + + String testTopicName = UUID.randomUUID().toString(); + binder.bindProducer(testTopicName, output, createProducerProperties()); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + ExtendedConsumerProperties properties = createConsumerProperties(); + properties.getExtension().setStartOffset(KafkaMessageChannelBinder.StartOffset.earliest); + binder.bindConsumer(testTopicName, "startOffsets", input1, properties); + Message receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload2.getBytes())); + Message receivedMessage2 = (Message) receive(input1); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload2); + } + + @Test + @SuppressWarnings("unchecked") + public void testReset() throws Exception { + KafkaTestBinder binder = getBinder(); + DirectChannel output = new DirectChannel(); + QueueChannel input1 = new QueueChannel(); + + String testTopicName = UUID.randomUUID().toString(); + + Binding producerBinding = binder.bindProducer(testTopicName, output, + createProducerProperties()); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + ExtendedConsumerProperties properties = createConsumerProperties(); + properties.getExtension().setResetOffsets(true); + properties.getExtension().setStartOffset(KafkaMessageChannelBinder.StartOffset.earliest); + Binding consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, + properties); + Message receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload2.getBytes())); + Message receivedMessage2 = (Message) receive(input1); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String(receivedMessage2.getPayload())).isEqualTo(testPayload2); + consumerBinding.unbind(); + + String testPayload3 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload3.getBytes())); + + ExtendedConsumerProperties properties2 = createConsumerProperties(); + properties2.getExtension().setResetOffsets(true); + properties2.getExtension().setStartOffset(KafkaMessageChannelBinder.StartOffset.earliest); + consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, properties2); + Message receivedMessage4 = (Message) receive(input1); + assertThat(receivedMessage4).isNotNull(); + assertThat(new String(receivedMessage4.getPayload())).isEqualTo(testPayload1); + Message receivedMessage5 = (Message) receive(input1); + assertThat(receivedMessage5).isNotNull(); + assertThat(new String(receivedMessage5.getPayload())).isEqualTo(testPayload2); + Message receivedMessage6 = (Message) receive(input1); + assertThat(receivedMessage6).isNotNull(); + assertThat(new String(receivedMessage6.getPayload())).isEqualTo(testPayload3); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + @SuppressWarnings("unchecked") + public void testResume() throws Exception { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + DirectChannel output = new DirectChannel(); + QueueChannel input1 = new QueueChannel(); + + String testTopicName = UUID.randomUUID().toString(); + Binding producerBinding = binder.bindProducer(testTopicName, output, + createProducerProperties()); + String testPayload1 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload1.getBytes())); + ExtendedConsumerProperties firstConsumerProperties = createConsumerProperties(); + Binding consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, + firstConsumerProperties); + Message receivedMessage1 = (Message) receive(input1); + assertThat(receivedMessage1).isNotNull(); + String testPayload2 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload2.getBytes())); + Message receivedMessage2 = (Message) receive(input1); + assertThat(receivedMessage2).isNotNull(); + assertThat(new String(receivedMessage2.getPayload())).isNotNull(); + consumerBinding.unbind(); + + String testPayload3 = "foo-" + UUID.randomUUID().toString(); + output.send(new GenericMessage<>(testPayload3.getBytes())); + + consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1, createConsumerProperties()); + Message receivedMessage3 = (Message) receive(input1); + assertThat(receivedMessage3).isNotNull(); + assertThat(new String(receivedMessage3.getPayload())).isEqualTo(testPayload3); + consumerBinding.unbind(); + producerBinding.unbind(); + } + + @Test + public void testSyncProducerMetadata() throws Exception { + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(createConfigurationProperties()); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + DirectChannel output = new DirectChannel(); + String testTopicName = UUID.randomUUID().toString(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.getExtension().setSync(true); + Binding producerBinding = binder.bindProducer(testTopicName, output, properties); + DirectFieldAccessor accessor = new DirectFieldAccessor(extractEndpoint(producerBinding)); + MessageHandler handler = (MessageHandler) accessor.getPropertyValue("handler"); + DirectFieldAccessor accessor1 = new DirectFieldAccessor(handler); + ProducerConfiguration producerConfiguration = (ProducerConfiguration) accessor1 + .getPropertyValue("producerConfiguration"); + assertThat(producerConfiguration.getProducerMetadata().isSync()) + .withFailMessage("Kafka Sync Producer should have been enabled."); + producerBinding.unbind(); + } + + @Test + public void testAutoCreateTopicsDisabledFailsIfTopicMissing() throws Exception { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoCreateTopics(false); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + RetryTemplate metatadataRetrievalRetryOperations = new RetryTemplate(); + metatadataRetrievalRetryOperations.setRetryPolicy(new SimpleRetryPolicy()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1000); + metatadataRetrievalRetryOperations.setBackOffPolicy(backOffPolicy); + binder.setMetadataRetryOperations(metatadataRetrievalRetryOperations); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + String testTopicName = "nonexisting" + System.currentTimeMillis(); + try { + binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + fail(); + } + catch (Exception e) { + assertThat(e).isInstanceOf(BinderException.class); + assertThat(e).hasMessageContaining("Topic " + testTopicName + " does not exist"); + } + + try { + binder.getConnectionFactory().getPartitions(testTopicName); + fail(); + } + catch (Exception e) { + assertThat(e).isInstanceOf(TopicNotFoundException.class); + } + } + + @Test + public void testAutoConfigureTopicsDisabledSucceedsIfTopicExisting() throws Exception { + String testTopicName = "existing" + System.currentTimeMillis(); + AdminUtils.createTopic(kafkaTestSupport.getZkClient(), testTopicName, 5, 1, new Properties()); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoCreateTopics(false); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + Binding binding = binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + binding.unbind(); + } + + @Test + public void testAutoAddPartitionsDisabledFailsIfTopicUnderpartitioned() throws Exception { + String testTopicName = "existing" + System.currentTimeMillis(); + AdminUtils.createTopic(kafkaTestSupport.getZkClient(), testTopicName, 1, 1, new Properties()); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(false); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + // this consumer must consume from partition 2 + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(2); + try { + binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + } + catch (Exception e) { + assertThat(e).isInstanceOf(BinderException.class); + assertThat(e) + .hasMessageContaining("The number of expected partitions was: 3, but 1 has been found instead"); + } + } + + @Test + public void testAutoAddPartitionsDisabledSucceedsIfTopicPartitionedCorrectly() throws Exception { + + String testTopicName = "existing" + System.currentTimeMillis(); + AdminUtils.createTopic(kafkaTestSupport.getZkClient(), testTopicName, 6, 1, new Properties()); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(false); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + RetryTemplate metatadataRetrievalRetryOperations = new RetryTemplate(); + metatadataRetrievalRetryOperations.setRetryPolicy(new SimpleRetryPolicy()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1000); + metatadataRetrievalRetryOperations.setBackOffPolicy(backOffPolicy); + binder.setMetadataRetryOperations(metatadataRetrievalRetryOperations); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + // this consumer must consume from partition 2 + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(2); + + Binding binding = binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + + Partition[] listenedPartitions = TestUtils.getPropertyValue(binding, + "endpoint.val$messageListenerContainer.partitions", Partition[].class); + + assertThat(listenedPartitions).hasSize(2); + assertThat(listenedPartitions).contains(new Partition(testTopicName, 2), new Partition(testTopicName, 5)); + Collection partitions = binder.getConnectionFactory().getPartitions(testTopicName); + assertThat(partitions).hasSize(6); + binding.unbind(); + } + + @Test + public void testAutoCreateTopicsEnabledSucceeds() throws Exception { + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoCreateTopics(true); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + RetryTemplate metatadataRetrievalRetryOperations = new RetryTemplate(); + metatadataRetrievalRetryOperations.setRetryPolicy(new SimpleRetryPolicy()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1000); + metatadataRetrievalRetryOperations.setBackOffPolicy(backOffPolicy); + binder.setMetadataRetryOperations(metatadataRetrievalRetryOperations); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + String testTopicName = "nonexisting" + System.currentTimeMillis(); + Binding binding = binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + binding.unbind(); + } + + @Test + public void testPartitionCountNotReduced() throws Exception { + String testTopicName = "existing" + System.currentTimeMillis(); + AdminUtils.createTopic(kafkaTestSupport.getZkClient(), testTopicName, 6, 1, new Properties()); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setAutoAddPartitions(true); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + RetryTemplate metatadataRetrievalRetryOperations = new RetryTemplate(); + metatadataRetrievalRetryOperations.setRetryPolicy(new SimpleRetryPolicy()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1000); + metatadataRetrievalRetryOperations.setBackOffPolicy(backOffPolicy); + binder.setMetadataRetryOperations(metatadataRetrievalRetryOperations); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + Binding binding = binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + binding.unbind(); + TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(testTopicName, + kafkaTestSupport.getZkClient()); + assertThat(topicMetadata.partitionsMetadata().size()).isEqualTo(6); + } + + @Test + public void testPartitionCountIncreasedIfAutoAddPartitionsSet() throws Exception { + String testTopicName = "existing" + System.currentTimeMillis(); + AdminUtils.createTopic(kafkaTestSupport.getZkClient(), testTopicName, 1, 1, new Properties()); + KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties(); + configurationProperties.setMinPartitionCount(6); + configurationProperties.setAutoAddPartitions(true); + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(configurationProperties); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + RetryTemplate metatadataRetrievalRetryOperations = new RetryTemplate(); + metatadataRetrievalRetryOperations.setRetryPolicy(new SimpleRetryPolicy()); + FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy(); + backOffPolicy.setBackOffPeriod(1000); + metatadataRetrievalRetryOperations.setBackOffPolicy(backOffPolicy); + binder.setMetadataRetryOperations(metatadataRetrievalRetryOperations); + DirectChannel output = new DirectChannel(); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + Binding binding = binder.doBindConsumer(testTopicName, "test", output, consumerProperties); + binding.unbind(); + TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(testTopicName, + kafkaTestSupport.getZkClient()); + assertThat(topicMetadata.partitionsMetadata().size()).isEqualTo(6); + } + + private static final class FailingInvocationCountingMessageHandler implements MessageHandler { + + private int invocationCount; + + private final LinkedHashMap> receivedMessages = new LinkedHashMap<>(); + + private final CountDownLatch latch; + + private FailingInvocationCountingMessageHandler(int latchSize) { + latch = new CountDownLatch(latchSize); + } + + private FailingInvocationCountingMessageHandler() { + this(1); + } + + @Override + public void handleMessage(Message message) throws MessagingException { + invocationCount++; + Long offset = message.getHeaders().get(KafkaHeaders.OFFSET, Long.class); + // using the offset as key allows to ensure that we don't store duplicate + // messages on retry + if (!receivedMessages.containsKey(offset)) { + receivedMessages.put(offset, message); + latch.countDown(); + } + throw new RuntimeException(); + } + + public LinkedHashMap> getReceivedMessages() { + return receivedMessages; + } + + public int getInvocationCount() { + return invocationCount; + } + + public CountDownLatch getLatch() { + return latch; + } + } +} diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java new file mode 100644 index 000000000..b58fd7fc2 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java @@ -0,0 +1,89 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.List; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Registration; + +import org.springframework.cloud.stream.binder.AbstractTestBinder; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfigurationProperties; +import org.springframework.cloud.stream.test.junit.kafka.TestKafkaCluster; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.integration.codec.Codec; +import org.springframework.integration.codec.kryo.KryoRegistrar; +import org.springframework.integration.codec.kryo.PojoCodec; +import org.springframework.integration.kafka.support.LoggingProducerListener; +import org.springframework.integration.kafka.support.ProducerListener; +import org.springframework.integration.tuple.TupleKryoRegistrar; + +/** + * Test support class for {@link KafkaMessageChannelBinder}. Creates a binder that uses a + * test {@link TestKafkaCluster kafka cluster}. + * @author Eric Bottard + * @author Marius Bogoevici + * @author David Turanski + * @author Gary Russell + * @author Soby Chacko + */ +public class KafkaTestBinder extends + AbstractTestBinder, ExtendedProducerProperties> { + + public KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration) { + try { + KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder(binderConfiguration); + binder.setCodec(getCodec()); + ProducerListener producerListener = new LoggingProducerListener(); + binder.setProducerListener(producerListener); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + binder.setApplicationContext(context); + binder.afterPropertiesSet(); + this.setBinder(binder); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void cleanup() { + // do nothing - the rule will take care of that + } + + private static Codec getCodec() { + return new PojoCodec(new TupleRegistrar()); + } + + private static class TupleRegistrar implements KryoRegistrar { + private final TupleKryoRegistrar delegate = new TupleKryoRegistrar(); + + @Override + public void registerTypes(Kryo kryo) { + this.delegate.registerTypes(kryo); + } + + @Override + public List getRegistrations() { + return this.delegate.getRegistrations(); + } + } + +} diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java new file mode 100644 index 000000000..b0f8da029 --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java @@ -0,0 +1,38 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import org.springframework.cloud.stream.binder.PartitionKeyExtractorStrategy; +import org.springframework.cloud.stream.binder.PartitionSelectorStrategy; +import org.springframework.messaging.Message; + +/** + * @author Marius Bogoevici + */ +public class RawKafkaPartitionTestSupport implements PartitionKeyExtractorStrategy, PartitionSelectorStrategy { + + @Override + public int selectPartition(Object key, int divisor) { + return ((byte[]) key)[0] % divisor; + } + + @Override + public Object extractKey(Message message) { + return message.getPayload(); + } + +} diff --git a/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawModeKafkaBinderTests.java b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawModeKafkaBinderTests.java new file mode 100644 index 000000000..775d2428e --- /dev/null +++ b/spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawModeKafkaBinderTests.java @@ -0,0 +1,262 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.kafka; + +import java.util.Arrays; + +import org.junit.Ignore; +import org.junit.Test; + +import org.springframework.cloud.stream.binder.Binding; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.HeaderMode; +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.endpoint.AbstractEndpoint; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Marius Bogoevici + * @author David Turanski + * @author Gary Russell + * @author Mark Fisher + */ +public class RawModeKafkaBinderTests extends KafkaBinderTests { + + @Test + @Override + public void testPartitionedModuleJava() throws Exception { + KafkaTestBinder binder = getBinder(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.setHeaderMode(HeaderMode.raw); + properties.setPartitionKeyExtractorClass(RawKafkaPartitionTestSupport.class); + properties.setPartitionSelectorClass(RawKafkaPartitionTestSupport.class); + properties.setPartitionCount(6); + + DirectChannel output = new DirectChannel(); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("partJ.0", output, properties); + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceCount(3); + consumerProperties.setInstanceIndex(0); + consumerProperties.setPartitioned(true); + consumerProperties.setHeaderMode(HeaderMode.raw); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0J"); + Binding input0Binding = binder.bindConsumer("partJ.0", "test", input0, consumerProperties); + consumerProperties.setInstanceIndex(1); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1J"); + Binding input1Binding = binder.bindConsumer("partJ.0", "test", input1, consumerProperties); + consumerProperties.setInstanceIndex(2); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2J"); + Binding input2Binding = binder.bindConsumer("partJ.0", "test", input2, consumerProperties); + + output.send(new GenericMessage<>(new byte[] { (byte) 0 })); + output.send(new GenericMessage<>(new byte[] { (byte) 1 })); + output.send(new GenericMessage<>(new byte[] { (byte) 2 })); + + Message receive0 = receive(input0); + assertThat(receive0).isNotNull(); + Message receive1 = receive(input1); + assertThat(receive1).isNotNull(); + Message receive2 = receive(input2); + assertThat(receive2).isNotNull(); + + assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0], ((byte[]) receive1.getPayload())[0], + ((byte[]) receive2.getPayload())[0])).containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2); + + input0Binding.unbind(); + input1Binding.unbind(); + input2Binding.unbind(); + outputBinding.unbind(); + } + + @Test + @Override + public void testPartitionedModuleSpEL() throws Exception { + KafkaTestBinder binder = getBinder(); + ExtendedProducerProperties properties = createProducerProperties(); + properties.setPartitionKeyExpression(spelExpressionParser.parseExpression("payload[0]")); + properties.setPartitionSelectorExpression(spelExpressionParser.parseExpression("hashCode()")); + properties.setPartitionCount(6); + properties.setHeaderMode(HeaderMode.raw); + + DirectChannel output = new DirectChannel(); + output.setBeanName("test.output"); + Binding outputBinding = binder.bindProducer("part.0", output, properties); + try { + AbstractEndpoint endpoint = extractEndpoint(outputBinding); + assertThat(getEndpointRouting(endpoint)).contains("part.0-' + headers['partition']"); + } + catch (UnsupportedOperationException ignored) { + + } + + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setConcurrency(2); + consumerProperties.setInstanceIndex(0); + consumerProperties.setInstanceCount(3); + consumerProperties.setPartitioned(true); + consumerProperties.setHeaderMode(HeaderMode.raw); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0S"); + Binding input0Binding = binder.bindConsumer("part.0", "test", input0, consumerProperties); + consumerProperties.setInstanceIndex(1); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1S"); + Binding input1Binding = binder.bindConsumer("part.0", "test", input1, consumerProperties); + consumerProperties.setInstanceIndex(2); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2S"); + Binding input2Binding = binder.bindConsumer("part.0", "test", input2, consumerProperties); + + Message message2 = MessageBuilder.withPayload(new byte[] { 2 }) + .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "foo") + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42) + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43).build(); + output.send(message2); + output.send(new GenericMessage<>(new byte[] { 1 })); + output.send(new GenericMessage<>(new byte[] { 0 })); + Message receive0 = receive(input0); + assertThat(receive0).isNotNull(); + Message receive1 = receive(input1); + assertThat(receive1).isNotNull(); + Message receive2 = receive(input2); + assertThat(receive2).isNotNull(); + assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0], ((byte[]) receive1.getPayload())[0], + ((byte[]) receive2.getPayload())[0])).containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2); + input0Binding.unbind(); + input1Binding.unbind(); + input2Binding.unbind(); + outputBinding.unbind(); + } + + @Test + @Override + public void testSendAndReceive() throws Exception { + KafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setHeaderMode(HeaderMode.raw); + Binding producerBinding = binder.bindProducer("foo.0", moduleOutputChannel, producerProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setHeaderMode(HeaderMode.raw); + Binding consumerBinding = binder.bindConsumer("foo.0", "test", moduleInputChannel, + consumerProperties); + Message message = MessageBuilder.withPayload("foo".getBytes()).build(); + // Let the consumer actually bind to the producer before sending a msg + binderBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = receive(moduleInputChannel); + assertThat(inbound).isNotNull(); + assertThat(new String((byte[]) inbound.getPayload())).isEqualTo("foo"); + producerBinding.unbind(); + consumerBinding.unbind(); + } + + // Ignored, since raw mode does not support headers + @Test + @Override + @Ignore + public void testSendAndReceiveNoOriginalContentType() throws Exception { + + } + + @Test + public void testSendAndReceiveWithExplicitConsumerGroup() { + KafkaTestBinder binder = getBinder(); + DirectChannel moduleOutputChannel = new DirectChannel(); + // Test pub/sub by emulating how StreamPlugin handles taps + QueueChannel module1InputChannel = new QueueChannel(); + QueueChannel module2InputChannel = new QueueChannel(); + QueueChannel module3InputChannel = new QueueChannel(); + ExtendedProducerProperties producerProperties = createProducerProperties(); + producerProperties.setHeaderMode(HeaderMode.raw); + Binding producerBinding = binder.bindProducer("baz.0", moduleOutputChannel, producerProperties); + ExtendedConsumerProperties consumerProperties = createConsumerProperties(); + consumerProperties.setHeaderMode(HeaderMode.raw); + Binding input1Binding = binder.bindConsumer("baz.0", "test", module1InputChannel, + consumerProperties); + // A new module is using the tap as an input channel + String fooTapName = "baz.0"; + Binding input2Binding = binder.bindConsumer(fooTapName, "tap1", module2InputChannel, + consumerProperties); + // Another new module is using tap as an input channel + String barTapName = "baz.0"; + Binding input3Binding = binder.bindConsumer(barTapName, "tap2", module3InputChannel, + consumerProperties); + + Message message = MessageBuilder.withPayload("foo".getBytes()).build(); + boolean success = false; + boolean retried = false; + while (!success) { + moduleOutputChannel.send(message); + Message inbound = receive(module1InputChannel); + assertThat(inbound).isNotNull(); + assertThat(new String((byte[]) inbound.getPayload())).isEqualTo("foo"); + + Message tapped1 = receive(module2InputChannel); + Message tapped2 = receive(module3InputChannel); + if (tapped1 == null || tapped2 == null) { + // listener may not have started + assertThat(retried).isFalse().withFailMessage("Failed to receive tap after retry"); + retried = true; + continue; + } + success = true; + assertThat(new String((byte[]) tapped1.getPayload())).isEqualTo("foo"); + assertThat(new String((byte[]) tapped2.getPayload())).isEqualTo("foo"); + } + // delete one tap stream is deleted + input3Binding.unbind(); + Message message2 = MessageBuilder.withPayload("bar".getBytes()).build(); + moduleOutputChannel.send(message2); + + // other tap still receives messages + Message tapped = receive(module2InputChannel); + assertThat(tapped).isNotNull(); + + // removed tap does not + assertThat(receive(module3InputChannel)).isNull(); + + // re-subscribed tap does receive the message + input3Binding = binder.bindConsumer(barTapName, "tap2", module3InputChannel, createConsumerProperties()); + assertThat(receive(module3InputChannel)).isNotNull(); + + // clean up + input1Binding.unbind(); + input2Binding.unbind(); + input3Binding.unbind(); + producerBinding.unbind(); + assertThat(extractEndpoint(input1Binding).isRunning()).isFalse(); + assertThat(extractEndpoint(input2Binding).isRunning()).isFalse(); + assertThat(extractEndpoint(input3Binding).isRunning()).isFalse(); + assertThat(extractEndpoint(producerBinding).isRunning()).isFalse(); + } +}