From 209082665637bef45691f3c0390f08cd405a22ba Mon Sep 17 00:00:00 2001 From: Mudabir Hussain Date: Sun, 6 Oct 2024 15:34:33 +0400 Subject: [PATCH] Fix NullPointerException with Ollama + tool calls Fixed #1459 --- .../ai/ollama/OllamaChatModel.java | 19 +++++++++++------- .../ollama/OllamaChatModelMultimodalIT.java | 13 ++++++++++++ .../src/test/resources/norway.webp | Bin 0 -> 10474 bytes 3 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 models/spring-ai-ollama/src/test/resources/norway.webp diff --git a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java index c6d689e68..6168ea55c 100644 --- a/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java +++ b/models/spring-ai-ollama/src/main/java/org/springframework/ai/ollama/OllamaChatModel.java @@ -211,13 +211,18 @@ public class OllamaChatModel extends AbstractToolCallSupport implements ChatMode Flux chatResponse = ollamaResponse.map(chunk -> { String content = (chunk.message() != null) ? chunk.message().content() : ""; - List toolCalls = chunk.message().toolCalls() == null ? List.of() - : chunk.message() - .toolCalls() - .stream() - .map(toolCall -> new AssistantMessage.ToolCall("", "function", toolCall.function().name(), - ModelOptionsUtils.toJsonString(toolCall.function().arguments()))) - .toList(); + + List toolCalls = List.of(); + + // Added null checks to prevent NPE when accessing tool calls + if (chunk.message() != null && chunk.message().toolCalls() != null) { + toolCalls = chunk.message() + .toolCalls() + .stream() + .map(toolCall -> new AssistantMessage.ToolCall("", "function", toolCall.function().name(), + ModelOptionsUtils.toJsonString(toolCall.function().arguments()))) + .toList(); + } var assistantMessage = new AssistantMessage(content, Map.of(), toolCalls); diff --git a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelMultimodalIT.java b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelMultimodalIT.java index f58552f86..4dffc7d2f 100644 --- a/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelMultimodalIT.java +++ b/models/spring-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelMultimodalIT.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThrows; @SpringBootTest @Testcontainers @@ -67,6 +68,18 @@ class OllamaChatModelMultimodalIT extends BaseOllamaIT { @Autowired private OllamaChatModel chatModel; + @Test + void unsupportedMediaType() throws IOException { + + var imageData = new ClassPathResource("/norway.webp"); + + var userMessage = new UserMessage("Explain what do you see on this picture?", + List.of(new Media(MimeTypeUtils.IMAGE_PNG, imageData))); + + assertThrows(RuntimeException.class, () -> chatModel.call(new Prompt(List.of(userMessage)))); + + } + @Test void multiModalityTest() throws IOException { diff --git a/models/spring-ai-ollama/src/test/resources/norway.webp b/models/spring-ai-ollama/src/test/resources/norway.webp new file mode 100644 index 0000000000000000000000000000000000000000..0da983e2ce5335eb13a7b793a1403588617dde3c GIT binary patch literal 10474 zcmV7UINk&HEC;$LgMM6+kP&gpgC;$MGvjCj|DnJ3&06vjAl}Dr^3{6ivAOwO~ z+eST}*jM{(wJ+6cBmBd+9~^!R8h;bM$G=+!KA*Y2dco(n^PJzG-Hf&KYnPvWv&*9T z|1aRL_HNr>qAtMwG5( zaZc)n6Q5~UfSUzhs=x-_&u0so=_GdYM6n|qccd(=ZwQQU!0e3BA~NZ!H?=L2^!-A+ zPx)e6k{A*}Jb?mhq_bkOtrQHeS#muC&9^f@pCHDBxXT&D{)tC=MTk?&kbCl!_N z?XBxLkCIBGj7S^s5LL!&r4+AWN7N`YVzoQ_>!P`v&OQ!%j5ws?tGxx>S%^7;s1q*v zo)KRN@%RF(3*nxarCBhB^2xDkNFdwcl`Tw|els65!A|we&!A^e@86_URtQB02>imS zDBIGYs5;TM$&j!u&sKtR;Y{hUn7;x=`H}x0UnUSjp}4&)nDtGOb<2I_(9(u*9`3G{ zYt_iS)AL<@k{{%+eBUgwyl6Ha?|o zS~(-2e8dTG@YCyIeDuq@0qYa&qAND8$dZ1^KhhXb&$VSG=*JmLEwpSWj_<6Lt9dM%HgB+=Y|)Lj}m=nNd`5=>z5!pUbi zHp!_;FYex}3aR9VDGneH6t2mVPg7X?nb)>r>2|xh=AJa&fc=SZ6ts)UJi4f|arcsN zgeHG9>gK@G>Lp(v)Hvu`S*C@5+07Va^}{{hbu?Ga?`^&=Aa@lI>PmmBSMOGx7CV{$ z8IL+?q~ztet9xWa`_l&-9MZIhj%h}O=b9G5Q{$0(+JNUId`4GE%Gj{}kLPWm<4~7Z zBJM~#chEMQxFmxSVaaXc_J#-998g8#o`T71Iaq_REQhFBdguy%*(=Clj$#7*iwduu zD~S2yv|40)H=F#DY=co{3+7{;PkrY{jd-Rp4pw?Z!LbjzRhlcK_kiMh`D7{z>JHtn zLC5C-vN^|iTU>Zz;0wYl%o#dN*2Del$13gSNC|d8Dut^*%wGmHlx(Jz(-$kkE?A8p z#>YuV4Um37R|?W$!*c080~j4WEM@z5`BCrK#3cJ_x@CC zDfL=79hA!?ei>-BNflmPAVg%*TdRwYX$8d0|)x_ujXI@$RKwa86l;O@`#*_#rz?Ylk5pNw&<<@c*3jG}9Xb^wP<>)mv7c{Gizf zueq|o%#|61X27G5XW_)&1k&T9KT9Uh}&S=t^a4#($5t&C8lva{cy@fZeSS_gO>-^WKlXlECuQH8*Ri zfTEEmfPPq~g!T(;4M0p_Fzw~8Z^nV43?*@3h1GR#Q)rGK_okxz8ab$k5D1)`_ZQ+^ zV?&d%G>0G0HRTwRK)v2VXM!!XV)r@HYH59UHLs1if7So>%=~(?#u#f^3qQEW1vYC) z?8?@(LR>RjZ_A=!^}m1ZfJnLVCv#^0!vgPm6HUejm;aX5_eGv;wSV%Rj@LStJECi9-$E3;p@2im(1dP5;yVZ1T zqVecO;<9$5;g4$1`R@EIYRi2tkq%vK6hrNZH%G&0N6LvC@dR|wd{==Mx;!3gbZl4Y zm35z`7}s>*hoG`a(HRH!~&peEnbGUK2T;p$W0zQ)E z>#I=MTR~r5Jd^UY8iU7$zj{SDiC~YLGsr>OX>!P}Otvk@gE^L}GO~7OZO-yN ztoy*b>k&bBypdf*zb$JxgF{1K#5Bu-@@$#M7^c$+qKSeiNl!F(tf7ne98C*XnS%eu zK!K3aePN}njsfLQ*!IEXWhGd7l>JU%9d$MW#;OVk3D9tnGVvit^$0>A7U_t(Zz@tk zn@QUpl9xeMV5Svd6pwoAnQ1T^vF_ z{*}B5$3oxxYXlYg8k`Dksg#a`4${E&q;uwgQ)qj1=l`!!c7P+Bo*S1STcdI>RnvM> zkX%KC{sS_@%}bXZfYxm$$p=rwg2BTJB{CypPgpk|lw}GHX3MZ|rn`o^%I zYORGGlM$oS1+#K@VB2;0#^A<4#r6$XGTC0B3VMLi63V^D^kbFY-sn zYF4Ifl5vW3Yt}zCh1{`z{0G?({%7})08pu~6W9kOz?sl8{ToYKEZ3v6R@_+2xn+&6 zd`8(rkASd;YPZCpXzr6QBR(L!bGq#xXA(99=QX(WOGxp$fAvVZk?PkgX zl3}tqj{S`n=y=R9y8oNjO!ewCh9xvH8!_G!HdiZ@*Ea$Z`y)8}54PQz>%f%yV}=D- z)i^|2KrziT46__ly5(fM=xyLxlHqG|Wr4zZQ$pv^LVi=~pWI(yjLQq(j9QXVjJ-by_ zTeYozIwlTMz{D$A-=8J8*T2}yb0c&*%D~^PG{GT!9Knl=#%^ldG2mY@PpoK zGd(QUSqSWX{3vyg(Vq^C3Ms493D0x9GX*)93}l{|I_TyKxU`+h0=?Ux$B=QQZCgT| z9HEq!Xbj2 zZ^U8@e#f?3V3aZX1mEDUZf0S)r!}^#f8F zT#QNMGt0{v7;~J+G&ItwWaHk(px$_+hQSwenF<40Eo#*_NI)`f`b=1eR?_omge@B^ zw?JH;Pe&5yUf?F{Zy*AFoob?vV)uAwWnz6l^R5%jn|u8P9vqPH;Xv`i^snXcO=MgG zrrV*Yfn5H{(66I8YoTtWD_Mn0`b+?o$1nTHe|h5@qfq7zI?Y8y^saXaLsPs3;c4FQ z0hb=Q*WILtwFXH)K6d7S=}6QI(R_al1yMZF=qbgqrX#HW?6qEamMDgd)&x7ZeWI_PD~()-&~vr7F6l|CDYny6I~jS>?T|lEBiuo8|C%aIEN6jKc{b!XVHqw>7G&pwDG2V6qb zI2QUdtVj5rki}K7<}c?25-fQvEHl?BIT0iY2Gp3Ub?ADqvkDK9PjoZ__Ya+gzgCDE zqZgFezxJ6T{8oSL6c&;ICN7u9hgn!-YZm!9hAw)pE3xG3LIugx)kyT;GdQTFWJ#jk z_?wDpMy}85Sc^6emdayOHMzTaUzs%d2wFWu2c9uA3ih$G#thYBca*)cK~nFjgDNxm zK-MNBwlWj$;8su|Omw7DcVH&jtndy$>;6IJc>E4M`UPy$iQ6o$k(N4!i65lp6*T)% zR_8&%%c)QZKv0z0-1nDtQzdW$;T=|n&&av}_9(ZuLRFt*gx#XUPK8cGjH2)~|AY29 zs~>?|DepKdfGxbo8+#W* zd8+CpZ@HRYsoq!Z`&x-!wm>BWblFNiqK*DZ%O&$! zwnZ4<-~T+)r&e#5w;XuRU*ILWto_<)7D1eSu&`$`BHQ`Pe3@oH(kyB-)?_yb11O-a zuk1(UIs}4H+62U8y%S>T_^6=rV?+xW8fa%w2fHkiQ4?Pn-!dKns6ql2b_39;K)-loY4G%-+NGlMs zQ+sVFGh{zcbJ(o`Q8}rJ>rW@W*z)FwXECRp?0)%Xr`nq}wnVdMN@Igw-A2(v!BgIB z=vk=^sInH4-OHhAwwYNjy_>kwWwFrGTTd|d5+SK-ki=Wz0(3bi>i$3ol@U$Tt9T?k zPv`(NJjo>y$g}m2q}74zAccw~2Pe`zoi1lKc4X=ZDm28Ph1rEW+}h;z7~PG+j6;aq z8urvS^Qbd%m+eCy2hkNbvb>NbnWS^}oL0VFM=cE4p0dc7 zUY!*e zgfg`IT%uq;f@p%QVnNfQ6RMW`3w>GAx02WdoP&wWAd*U`m>*neBaYzB2Wvl_-K`jJ zu`C*7mO`|^)qL5E^l&!Z?dtXWwBxv9&cAf@T?aXsUnCeS>@tmS7T! zu@ZJHUx>g07w2z(tVA=^XSTYKAHJRX#n32I$dbn0S)m5`T$LWij`tM35q1BAUbdcz zbn2nk_i)sjaTRGV_D&%}r4;EM55*!29k$Zs20_rHJR`}KJs#=+kbBP3mdD{&<5bm& z!ivdk2DV)Nj=tadP;820aLK00z0EUPl&-e4NifKkEMo&+Uw@D7mkCK>Yg{Pw1Dg|Y zu2nz=5)fmpIoga@?vP6p6FiZZAUcU$a9G!hYZ=J3qK@yCZ*U@ce6n?S2Ch1T4Qc2- zT*1{2;$$fZjsz8O?sZ>WoZ*lA5W8>v3&QiJ67Zrg+_0+n)d_+<<&U|H z4_g;RA(3h|_{T;#t>5vRh z0vIpi!22VEJbpj;ic3hou_&a8BVm6Bsk+Ob-%()BwwwR$B#@k(r>No9Vc4pTy-c|^6llM24@EHAjJCEv|@>Hd=e|wIbYvZ8xJ4Q^t!0tF_`+h8$ z+Sj_gH*=I5JLMo$0T~+1K~?f%^?-m@;;XQ?)Ui>Hi&o z@@CRlXGu)@cCCpnovLdZ$hxe-9Uqy7i zh)dSrpJ=5-_;mo-Jxj5VmDV~vq!(f!pU49G-S}b@E;g9hh(R|Q#kMfz195a`nC(-Q zQ9J#BG1PiYiE37zY^aOBq0RcxjXNn&Eph`|w-dxWYu`0WhWVTT>0sffb-qaa4Z4w} z(Az(l@zm*v50t{pf6dsRD&#y3(w3nwL+)sacT9xd`VA&xEc?jYOj5bg)A**ZVRc>= zax-_f_KQ5_*qF-rfv-^-Bx&;NK9UoVf|Mq6?i$ZKl4@R}O;9~P=ZRS1;jDXEc|?vh zteFu!M`M{D^H#s8>sS2lV3n$3g3|GMp$_y{S(=A!ohGp7cSZVW5T7}9aj zE7HfXrKJ{5JD&;h1FJHY%7iAq5AAWE|1;2Fq{mzx^tK0_-c8o<%C&(-++H>P30El! zw82JhOGHO;8-awuyt#dZUiskrYJ-=4bbtN1&mQu|-%ct=d44#hx-$D^Gqb$g)^ZnM zx+f-ROt4F43SY3;c3l-{7|BPxxkswsX*|bMxh5k$Rpl zYjNJ}-C!-95!@UmOYNqfPUE2IyJN*OoAkzJg@fo%2C&7`UEW2fu#207^u4C|vy*gu zC!F=>kL}7*_CUO6`6(jA6#pYeid45YdRJ^}D{bqsUczKtzAK!^HoZ=M02ODdqyafc zPm4?1r_wWmtx6O{1ji&@Swos97Q3XOa^bhZMd1PfD0hmc2=Qes@MNg`dB}a6SIC@&k-l$DZr>)63#|a{6 z7dQS9FaN3tkyQ4|m)3QNP@bgoQ!c9opa3Ar6%~e08s_jbY`3Bd%BAk$e6=Qwb1|%4 z$~ug=y~w4HGdkI86Q4ccR?pEj+=@I_iKzlt&5Xc-fbT^fa}E$dFLw?b?(T%r@U(A6v9)I!F#eCK{G z0!x$le!&$yUp2(5EUM*^C>lHV`X(i^L_05$N)Jg}vqn0qs({_+C;WKSO2&*v`T}v! zMhB(U6Zr%{$rdZdXjEf zS+_N7%XG1N`|`+NvRzh3+12IJ-G{mrmxP7KR3QJRm~Aky5PrS$N8mgoIEMMQg3PAa z7H@rDRC;nWKKd{7n3gYti$|0cAOg~$Lap)G^d>Bb(-B`>|GvC*JNo=11|qD7W73Bx zX7aSWG}2l?Ld@ITK97^fDQ*`W9f4{i5l=#1G`+Bytq5rU7vC7s!0+_=Up_*Rh;`&B zety<0xwVGCJPwDS<$P8CaL!GLHmq_ga33@6dS)tr>zjQg3Ea-I!DU&(bPYS&7*Mkp zQUe)nj9Uptl^a;y3}o6Kj_=2H+pn7)Q;M^_vVup65T`z>R$|TRYJ9#$)BSd0_T0ec1_xS>hZ1{Z<}Pm&u@n=g_-M@D)f7Js5okk5>RN=K@tlY+EdE<2u< z?+L-OI1@#34@I&Yn(~GhB+^NYPjN$*X5|NF2MXUV+Vu!iaza4_IB?@fHVt7*wX^Z_+;%`rQmJ?fTv^Q*9-2)e@mc34e@UA}KIa zDpZN8u*-*3PC>e+zgs&z|AY7XI?NfO174TN2D~$A_*E019jL$alLnGwP#`Z-#Cq4+ zSlRWgDLj24<+;S==qqgZUTT2=2miUGp(bB8JmFrXqfguLt`MEerxQ-#t`=^P#+slL z5a;8JmASWds6xQGhRag3E>Gas4YT7pM(5m?^&t|7joEI+H>6&Jm@&J(| zRt~>N$eLAu}rOeT+7qU|+~jKph&MyfB)y8Vit?p* z%PTS()VVw3m?7>>m>a!%M#kVPX{Hfs`6Eq%3Lw2}FUatycfBI6lGfzm7X=5Xq7CxH z>*U{SKDPvy!fHTrl2|v z6?X!o{}+Q@4YI4l01CJ+tLc{#j-0xpUXftR2#A7j?)e}zRlXFV zx@YxUr&Y&VTJiC5wkpn>2(KMq8d8zi!ZEf=WPW_qW-EPspq~Pev6${Tn;Ntd3}j*Jgp>7qyW?d(8bkWPhV0uT87xZo1~`YGBjbJycYO z%g#2MG?yOjJnv0}J2N&ohvLCc>R^&jQO?YbIAl*LB_o2_Qs|b{PtO>@0}UWNIA_mU zgT^qmSD2)xOEavy_7MAlFCT&QA*jWjx_rqRc=F|_WAkQlSMhAfgM@j#hM3S`MW6#- zVwK%~oF-Yq>{pT;X(^P6_#YXg3B&)dvbJ3B?ljX+7N<<_XD;P{(soH*_z3i+I(cN> zBZ2t-)cOgm`o6!zk?FUOfg_k2+%sJTDv7ih=5`@Q#c|d$$SI+*ohmN;DJz|xBR6%_ zCc_y2Pu1bg5ePxbrr>`Ref+V~7SG=KX1TJbiLfv=!9ljzxbH8zDnoD;-&vo_`~mKu zXqP?62Ovv8@$&cfeA6>5l9N{5LDSac9D}u+O0|-A9kYKapl8aca|bIA0wxba8xode zq(1D1$h|qSr{%TxGj01kY@@$sWNcwYILOKy9ngPkY8&M1tgKHt$q|3~MwAwXXci8- z_a*C-wp91e+$C<@Msc1R8$uj5+Tp!n-bX!3*-m28d;N_xiq}JlxN{9Id0sp3Zg@6D zey$f(urx*0M>C$1ed<`zQ=p*2g2);R)Qt7Sia~q+oyZhWeVo3I-?9#zHv4&b*lQDm zcUpek8818z-Z>Xto!-y=I^onqM22$v0L;A15cX&@X>Y))Oq_KG_D6gjinYSo5Y|0;tK&CZSwp87e9TdZ|0aS{Rg%!lBdng_s7d8bZ2&l5#svC1%@kKlFYOPp#S#H10_j z%ey`%p?T+9;QvkoxATrLzsa332&0w(79!#!%;28-t&&dDWx1-!LvCo?*=cVaYkx+I6RwvPl2Mz%}-19R-r;QV*7vTe)icM*j+ z6)wC7%IuJxq}@$w6$O)E~Nu{ZgWu!{ZT70Cp-Mygf)QE$4 z2s#8My$x*0M65$-D+s7@F$+m}HhGX?l6~Ak|7vDjz!PsmK+}__So%<7l`DD>fDCIH z8e)Jw%?FK=TF=d%3HBN^pW>X{d!aRlE*Udsrpe@ET`n(whFBV8!$ef@mcigoT4z9y zt11ECtv~>BgSqx(j*U(rQ4~%G_W=b}&QTu)FYB+($7<|2q#K6b5@j9&&IH3bgQqPc zcA&@Bvhb6>AIA@Q9rQ6Oq0Y)){p1P2buD03NTPTkRc2=Er%gMwXjA-)`d1hkXUJtU z#)x-Wqfe?Mt)|=JK`=*gpe)qL4o=7nEj*QE*u^mFm8mL!b1~EOnW&3;6l@|+m~~0= zup)L|Of0hPaR1|A%Mc9gW%%|Eh{De9eRl=ipY)+W@>YJ={VY{)Q1YH&imyM$aSPkq zyj$6~Mk^%jyk7)twe;h}!r`_^@RJyk#z4<<_C4>x&N0GM=i&f}CHlup)OI72k(!{^ zS8N6zXD-*u69hWbdeYMIbhU}zBG3rE6~?sx11Y}m*rE6|IYjSm)Ki+3W@QmWU-iyc zp@9cdjPqpTqdDdeIl`|gCCihrVz@Py#&B zBK}U!7gQ@*2`@i(Oz8OGq7KWqZ{44op61iePp#U}eOE?{lRAa9k&KLOG4I_>dyIpx z7w06lGb;I^MHL^?=e~PN)Ghuv?oVh=0YCdy;Z;DjC#zwzA4&YNd^hkYrV;>!iyE*` zMa!Tr#2T!TQEbQG5hb+3h_*@9fZLb`AKAD0kx55i^X3YAn&IQRmdTw0;{2&42=mtb zJc<$g(zof(+zhKS>5X;LIgkMLVS?XVL)t!>=>udxKwFp>0a8KI%svdZS-_5+)q@_~W(jqayX60m8u#WWb#nb&7$WPTD^erA!~zWdr5 zhXE(}F)7Xn>O{xbPNAZvRN3m2a!Y@hNW0NnhkrX2i|)?uWKz&wd;kO(e9U$xlPg2P zB0EQvx`G}Fo8|zF+BCZ)E<<1Sxz6MQEJ<_E``ej;1|C4ez)$s_=!vlsPhAY}e(Qx@ zFbiU93uI@B*#@%&e{{6mUo!|>Eh?%2x;J4^99f~ZuC51upqj{MTE{JvlX%!a6P{Z9 z#GdDAwinewV{u?Z+SQA6-$W)Yw15J~aHn8o*V(_M$&~z>+fp{s3Nh|t`|XM--S)B3%RGx>tHf<6 z)g5$Tpw`QWEVK1wOBjU}^kTcpwPWpG_+9y7MWhcUUz6_dGi!xcK<5HUVNuEljI}{i zB#f>=pgq^r#RO(A|J58j>aa+U^ZPwQJCpaZ+F*tZ!a_-pfN zY5i#mJ;k>E_XQtjK*o#xDOVHU<-F^?Vw%~|l5gep6 z5&iCqY?Q*QlSHh_Y99UqD(G9-o9}r58W3CR&vG_ktuK!W3ue-!KeS%!6@}KWJC48y giLlbD?3du$IMk23MD+EeZ|&DCACFU1$3TDp02cv?f&c&j literal 0 HcmV?d00001